From ec68eb088b846b716fdac14c601ee073450890a2 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Thu, 9 Jul 2026 22:26:04 -0700 Subject: [PATCH 01/22] Add internal TYP_HALF JIT type for FP16 acceleration Introduce TYP_HALF as an internal JIT SIMD base type only, used to drive FP16 scalar instruction selection (vaddsh, etc.). It is never the type of a local, argument, or return, so the ABI is unchanged. Update the WASM var_types mapping tables for the shifted enum. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/registeropswasm.cpp | 2 ++ src/coreclr/jit/typelist.h | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/coreclr/jit/registeropswasm.cpp b/src/coreclr/jit/registeropswasm.cpp index 112520dd9928b1..87d6f1b7463883 100644 --- a/src/coreclr/jit/registeropswasm.cpp +++ b/src/coreclr/jit/registeropswasm.cpp @@ -79,6 +79,7 @@ WasmValueType TypeToWasmValueType(var_types type) WasmValueType::Invalid, // TYP_ULONG, WasmValueType::F32, // TYP_FLOAT, WasmValueType::F64, // TYP_DOUBLE, + WasmValueType::Invalid, // TYP_HALF, WasmValueType::I, // TYP_REF, WasmValueType::I, // TYP_BYREF, WasmValueType::Invalid, // TYP_STRUCT, @@ -122,6 +123,7 @@ WasmValueType ActualTypeToWasmValueType(var_types type) WasmValueType::Invalid, // TYP_ULONG, WasmValueType::F32, // TYP_FLOAT, WasmValueType::F64, // TYP_DOUBLE, + WasmValueType::Invalid, // TYP_HALF, WasmValueType::I, // TYP_REF, WasmValueType::I, // TYP_BYREF, WasmValueType::Invalid, // TYP_STRUCT, diff --git a/src/coreclr/jit/typelist.h b/src/coreclr/jit/typelist.h index b2cf2ffa1390f4..3b374c1cdb7feb 100644 --- a/src/coreclr/jit/typelist.h +++ b/src/coreclr/jit/typelist.h @@ -54,6 +54,8 @@ DEF_TP(ULONG ,"ulong" , TYP_LONG, 8,EPS,EPS, 2, 8, VTR_INT, available DEF_TP(FLOAT ,"float" , TYP_FLOAT, 4, 4, 4, 1, 4, VTR_FLOAT, availableFloatRegs, RBM_FLT_CALLEE_SAVED, RBM_FLT_CALLEE_TRASH, VTF_FLT) DEF_TP(DOUBLE ,"double" , TYP_DOUBLE, 8, 8, 8, 2, 8, VTR_FLOAT, availableDoubleRegs, RBM_FLT_CALLEE_SAVED, RBM_FLT_CALLEE_TRASH, VTF_FLT) +DEF_TP(HALF ,"half" , TYP_HALF, 2, 2, 2, 1, 2, VTR_FLOAT, availableFloatRegs, RBM_FLT_CALLEE_SAVED, RBM_FLT_CALLEE_TRASH, VTF_S) + DEF_TP(REF ,"ref" , TYP_REF, PS,GCS,GCS, PST,PS,VTR_INT, availableIntRegs, RBM_INT_CALLEE_SAVED, RBM_INT_CALLEE_TRASH, VTF_ANY|VTF_GCR|VTF_I) DEF_TP(BYREF ,"byref" , TYP_BYREF, PS,BRS,BRS, PST,PS,VTR_INT, availableIntRegs, RBM_INT_CALLEE_SAVED, RBM_INT_CALLEE_TRASH, VTF_ANY|VTF_BYR|VTF_I) DEF_TP(STRUCT ,"struct" , TYP_STRUCT, 0, 0, 0, 1, 4, VTR_INT, availableIntRegs, RBM_INT_CALLEE_SAVED, RBM_INT_CALLEE_TRASH, VTF_S) From 09529f8b9a14c87275b10e4413d5a3d9ac7faa1c Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Thu, 9 Jul 2026 22:40:50 -0700 Subject: [PATCH 02/22] Add AVX10v1 FP16 scalar hwintrinsic table entries and TYP_HALF ins slot Extend the HARDWARE_INTRINSIC ins array with an 11th slot for TYP_HALF across all targets, add the internal NI_AVX10v1_* FP16 scalar intrinsic entries used to drive vaddsh/vsubsh/etc. selection, and declare the NI_System_Half_* named intrinsics. No behavior change yet; these tables are wired up in later commits. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/hwintrinsic.cpp | 8 +- src/coreclr/jit/hwintrinsic.h | 2 +- src/coreclr/jit/hwintrinsiclist.h | 348 +++--- src/coreclr/jit/hwintrinsiclistarm64.h | 994 ++++++++--------- src/coreclr/jit/hwintrinsiclistwasm.h | 128 +-- src/coreclr/jit/hwintrinsiclistxarch.h | 1354 ++++++++++++------------ src/coreclr/jit/namedintrinsiclist.h | 35 +- src/coreclr/jit/valuenumfuncs.h | 4 +- 8 files changed, 1473 insertions(+), 1400 deletions(-) diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index 421b6adffd2d27..eb39ffe945ba48 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -8,12 +8,12 @@ static const HWIntrinsicInfo hwIntrinsicInfoArray[] = { // clang-format off -#define HARDWARE_INTRINSIC(isa, name, simdSize, numArgs, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, intCost, fltCost, category, flag) \ +#define HARDWARE_INTRINSIC(isa, name, simdSize, numArgs, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, intCost, fltCost, category, flag) \ { \ /* name */ #name, \ /* flags */ static_cast(flag), \ /* id */ NI_##isa##_##name, \ - /* ins */ {t1, t2, t3, t4, t5, t6, t7, t8, t9, t10}, \ + /* ins */ {t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11}, \ /* isa */ InstructionSet_##isa, \ /* simdSize */ simdSize, \ /* numArgs */ numArgs, \ @@ -55,7 +55,7 @@ const HWIntrinsicInfo& HWIntrinsicInfo::lookup(NamedIntrinsic id) // The instruction for id and type instruction HWIntrinsicInfo::lookupIns(NamedIntrinsic id, var_types type, Compiler* comp) { - if ((type < TYP_BYTE) || (type > TYP_DOUBLE)) + if ((type < TYP_BYTE) || (type > TYP_HALF)) { assert(!"Unexpected type"); return INS_invalid; @@ -932,7 +932,7 @@ static const HWIntrinsicIsaRange hwintrinsicIsaRangeArray[] = { { FIRST_NI_AVX512, LAST_NI_AVX512 }, // AVX512 { FIRST_NI_AVX512v2, LAST_NI_AVX512v2 }, // AVX512v2 { FIRST_NI_AVX512v3, LAST_NI_AVX512v3 }, // AVX512v3 - { NI_Illegal, NI_Illegal }, // AVX10v1 + { FIRST_NI_AVX10v1, LAST_NI_AVX10v1 }, // AVX10v1 { FIRST_NI_AVX10v2, LAST_NI_AVX10v2 }, // AVX10v2 { NI_Illegal, NI_Illegal }, // APX { FIRST_NI_AES, LAST_NI_AES }, // AES diff --git a/src/coreclr/jit/hwintrinsic.h b/src/coreclr/jit/hwintrinsic.h index 7effb01b75f84b..6119138f9fa743 100644 --- a/src/coreclr/jit/hwintrinsic.h +++ b/src/coreclr/jit/hwintrinsic.h @@ -549,7 +549,7 @@ struct HWIntrinsicInfo const char* name; // 4 or 8-bytes HWIntrinsicFlag flags; // 8-bytes NamedIntrinsic id; // 2-bytes - uint16_t ins[10]; // 10 * 2-bytes + uint16_t ins[11]; // 11 * 2-bytes uint8_t isa; // 1-byte int8_t simdSize; // 1-byte int8_t numArgs; // 1-byte diff --git a/src/coreclr/jit/hwintrinsiclist.h b/src/coreclr/jit/hwintrinsiclist.h index e9ffeba17c0411..7148426957c853 100644 --- a/src/coreclr/jit/hwintrinsiclist.h +++ b/src/coreclr/jit/hwintrinsiclist.h @@ -33,232 +33,232 @@ // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Intrinsics for Vector64/128/256/512 and Vector #define FIRST_NI_Vector NI_Vector_Abs -HARDWARE_INTRINSIC(Vector, Abs, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AddSaturate, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AndNot, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, As, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsByte, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsDouble, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsInt16, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsInt64, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsNInt, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsNUInt, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsSByte, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsSingle, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsUInt16, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsUInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsUInt64, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsVector, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsVector128, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, Abs, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AddSaturate, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AndNot, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, As, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsByte, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsDouble, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsInt16, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsInt64, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsNInt, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsNUInt, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsSByte, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsSingle, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsUInt16, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsUInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsUInt64, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsVector, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsVector128, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) #if defined(TARGET_XARCH) -HARDWARE_INTRINSIC(Vector, AsVector128Unsafe, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movups, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics|HW_Flag_NoContainment|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsVector2, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movsd_simd, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsVector256, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsVector3, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movups, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsVector128Unsafe, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movups, INS_invalid, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics|HW_Flag_NoContainment|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsVector2, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movsd_simd, INS_invalid, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsVector256, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsVector3, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movups, INS_invalid, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_AvxOnlyCompatible) #elif defined(TARGET_ARM64) -HARDWARE_INTRINSIC(Vector, AsVector128Unsafe, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mov, INS_invalid, -1, 1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Vector, AsVector2, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, 1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, AsVector3, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mov, INS_invalid, -1, 1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Vector, AsVector128Unsafe, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mov, INS_invalid, INS_invalid, -1, 1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Vector, AsVector2, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, 1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, AsVector3, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mov, INS_invalid, INS_invalid, -1, 1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) #else -HARDWARE_INTRINSIC(Vector, AsVector128Unsafe, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, 1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, AsVector2, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, 1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, AsVector3, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, 1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, AsVector128Unsafe, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, 1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, AsVector2, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, 1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, AsVector3, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, 1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) #endif -HARDWARE_INTRINSIC(Vector, AsVector4, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsVector4, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) #if defined(TARGET_XARCH) -HARDWARE_INTRINSIC(Vector, AsVector512, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, AsVector512, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) #endif -HARDWARE_INTRINSIC(Vector, Ceiling, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, ConcatLowerLower, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, ConcatLowerUpper, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, ConcatUpperLower, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, ConcatUpperUpper, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, Ceiling, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, ConcatLowerLower, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, ConcatLowerUpper, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, ConcatUpperLower, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, ConcatUpperUpper, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) #if defined(TARGET_XARCH) -HARDWARE_INTRINSIC(Vector, ConditionalSelect, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, ConditionalSelect, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) #else -HARDWARE_INTRINSIC(Vector, ConditionalSelect, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, ConditionalSelect, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) #endif -HARDWARE_INTRINSIC(Vector, ConvertToDouble, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, ConvertToInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, ConvertToInt32Native, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, ConvertToInt64, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, ConvertToInt64Native, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, ConvertToSingle, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, ConvertToUInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, ConvertToUInt32Native, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, ConvertToUInt64, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, ConvertToUInt64Native, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, Create, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, CreateAlternatingSequence, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, CreateGeometricSequence, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, ConvertToDouble, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, ConvertToInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, ConvertToInt32Native, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, ConvertToInt64, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, ConvertToInt64Native, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, ConvertToSingle, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, ConvertToUInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, ConvertToUInt32Native, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, ConvertToUInt64, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, ConvertToUInt64Native, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, Create, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, CreateAlternatingSequence, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, CreateGeometricSequence, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) #if defined(TARGET_XARCH) -HARDWARE_INTRINSIC(Vector, CreateScalar, -1, 1, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd64, INS_movd64, INS_movss, INS_movsd_simd, -1, -1, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, CreateScalarUnsafe, -1, 1, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd64, INS_movd64, INS_movss, INS_movsd_simd, -1, -1, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, CreateScalar, -1, 1, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd64, INS_movd64, INS_movss, INS_movsd_simd, INS_invalid, -1, -1, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, CreateScalarUnsafe, -1, 1, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd64, INS_movd64, INS_movss, INS_movsd_simd, INS_invalid, -1, -1, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics|HW_Flag_AvxOnlyCompatible) #elif defined(TARGET_ARM64) -HARDWARE_INTRINSIC(Vector, CreateScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) -HARDWARE_INTRINSIC(Vector, CreateScalarUnsafe, -1, 1, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_fmov, INS_fmov, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment) +HARDWARE_INTRINSIC(Vector, CreateScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) +HARDWARE_INTRINSIC(Vector, CreateScalarUnsafe, -1, 1, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_fmov, INS_fmov, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment) #else -HARDWARE_INTRINSIC(Vector, CreateScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) -HARDWARE_INTRINSIC(Vector, CreateScalarUnsafe, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Vector, CreateScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) +HARDWARE_INTRINSIC(Vector, CreateScalarUnsafe, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) #endif -HARDWARE_INTRINSIC(Vector, CreateSequence, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, Dot, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, Equals, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, EqualsAny, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, ExtractMostSignificantBits, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, Floor, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, FusedMultiplyAdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, CreateSequence, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, Dot, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, Equals, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, EqualsAny, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, ExtractMostSignificantBits, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, Floor, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, FusedMultiplyAdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) #if defined(TARGET_XARCH) -HARDWARE_INTRINSIC(Vector, GetElement, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_extractps, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsScalarT|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, GetElement, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_extractps, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsScalarT|HW_Flag_AvxOnlyCompatible) #elif defined(TARGET_ARM64) -HARDWARE_INTRINSIC(Vector, GetElement, -1, 2, INS_smov, INS_umov, INS_smov, INS_umov, INS_smov, INS_umov, INS_umov, INS_umov, INS_dup, INS_dup, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsScalarT|HW_Flag_SupportsContainment) +HARDWARE_INTRINSIC(Vector, GetElement, -1, 2, INS_smov, INS_umov, INS_smov, INS_umov, INS_smov, INS_umov, INS_umov, INS_umov, INS_dup, INS_dup, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsScalarT|HW_Flag_SupportsContainment) #else -HARDWARE_INTRINSIC(Vector, GetElement, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsScalarT) +HARDWARE_INTRINSIC(Vector, GetElement, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsScalarT) #endif #if defined(TARGET_XARCH) -HARDWARE_INTRINSIC(Vector, GetLower, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_movdqu32, INS_movdqu32, INS_movups, INS_movupd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, GetLower128, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_vmovdqu64, INS_vmovdqu64, INS_movups, INS_movupd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, GetUpper, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, GetLower, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_movdqu32, INS_movdqu32, INS_movups, INS_movupd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, GetLower128, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_vmovdqu64, INS_vmovdqu64, INS_movups, INS_movupd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, GetUpper, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) #elif defined(TARGET_ARM64) -HARDWARE_INTRINSIC(Vector, GetLower, -1, 1, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Vector, GetUpper, -1, 1, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Vector, GetLower, -1, 1, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Vector, GetUpper, -1, 1, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) #endif -HARDWARE_INTRINSIC(Vector, GreaterThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, GreaterThanAll, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, GreaterThanAny, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, GreaterThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, GreaterThanOrEqualAll, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, GreaterThanOrEqualAny, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, GreaterThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, GreaterThanAll, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, GreaterThanAny, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, GreaterThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, GreaterThanOrEqualAll, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, GreaterThanOrEqualAny, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, IsEvenInteger, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, IsFinite, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, IsInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, IsInteger, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, IsNaN, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, IsNegative, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, IsNegativeInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, IsNormal, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, IsOddInteger, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, IsPositive, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, IsPositiveInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, IsSubnormal, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, IsZero, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, LessThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, LessThanAll, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, LessThanAny, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, LessThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, LessThanOrEqualAll, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, LessThanOrEqualAny, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, LoadAligned, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, LoadAlignedNonTemporal, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, LoadUnsafe, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, Max, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, MaxMagnitude, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, MaxMagnitudeNumber, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, MaxNative, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, MaxNumber, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, Min, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, MinMagnitude, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, MinMagnitudeNumber, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, MinNative, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, MinNumber, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, MultiplyAddEstimate, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, Narrow, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, IsEvenInteger, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, IsFinite, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, IsInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, IsInteger, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, IsNaN, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, IsNegative, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, IsNegativeInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, IsNormal, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, IsOddInteger, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, IsPositive, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, IsPositiveInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, IsSubnormal, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, IsZero, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, LessThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, LessThanAll, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, LessThanAny, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, LessThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, LessThanOrEqualAll, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, LessThanOrEqualAny, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, LoadAligned, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, LoadAlignedNonTemporal, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, LoadUnsafe, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, Max, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, MaxMagnitude, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, MaxMagnitudeNumber, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, MaxNative, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, MaxNumber, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, Min, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, MinMagnitude, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, MinMagnitudeNumber, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, MinNative, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, MinNumber, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, MultiplyAddEstimate, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, Narrow, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) #if defined(TARGET_ARM64) -HARDWARE_INTRINSIC(Vector, NarrowWithSaturation, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, NarrowWithSaturation, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) #else -HARDWARE_INTRINSIC(Vector, NarrowWithSaturation, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, NarrowWithSaturation, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) #endif -HARDWARE_INTRINSIC(Vector, Reverse, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, Round, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, ShiftLeft, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, Shuffle, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(Vector, ShuffleNative, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(Vector, ShuffleNativeFallback, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(Vector, Sqrt, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, StoreAligned, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, StoreAlignedNonTemporal, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, StoreUnsafe, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, SubtractSaturate, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, Sum, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, Reverse, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, Round, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, ShiftLeft, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, Shuffle, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(Vector, ShuffleNative, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(Vector, ShuffleNativeFallback, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(Vector, Sqrt, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, StoreAligned, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, StoreAlignedNonTemporal, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, StoreUnsafe, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, SubtractSaturate, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, Sum, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) #if defined(TARGET_XARCH) -HARDWARE_INTRINSIC(Vector, ToScalar, -1, 1, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd64, INS_movd64, INS_movss, INS_movsd_simd, -1, -1, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsScalarT|HW_Flag_SpecialImport|HW_Flag_NoRMWSemantics|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, ToVector256, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_movdqu32, INS_movdqu32, INS_movups, INS_movupd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_NoRMWSemantics|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, ToVector256Unsafe, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_movdqu32, INS_movdqu32, INS_movups, INS_movupd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_NoRMWSemantics|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, ToVector512, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_vmovdqu64, INS_vmovdqu64, INS_movups, INS_movupd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(Vector, ToVector512Unsafe, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_vmovdqu64, INS_vmovdqu64, INS_movups, INS_movupd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(Vector, ToScalar, -1, 1, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd64, INS_movd64, INS_movss, INS_movsd_simd, INS_invalid, -1, -1, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsScalarT|HW_Flag_SpecialImport|HW_Flag_NoRMWSemantics|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, ToVector256, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_movdqu32, INS_movdqu32, INS_movups, INS_movupd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_NoRMWSemantics|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, ToVector256Unsafe, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_movdqu32, INS_movdqu32, INS_movups, INS_movupd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_NoRMWSemantics|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, ToVector512, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_vmovdqu64, INS_vmovdqu64, INS_movups, INS_movupd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(Vector, ToVector512Unsafe, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_vmovdqu64, INS_vmovdqu64, INS_movups, INS_movupd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NormalizeSmallTypeToInt) #elif defined(TARGET_ARM64) -HARDWARE_INTRINSIC(Vector, ToScalar, -1, 1, INS_smov, INS_umov, INS_smov, INS_umov, INS_smov, INS_umov, INS_umov, INS_umov, INS_dup, INS_dup, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsScalarT|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(Vector, ToVector128, -1, 1, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, ToVector128Unsafe, -1, 1, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, ToScalar, -1, 1, INS_smov, INS_umov, INS_smov, INS_umov, INS_smov, INS_umov, INS_umov, INS_umov, INS_dup, INS_dup, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsScalarT|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Vector, ToVector128, -1, 1, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, ToVector128Unsafe, -1, 1, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg) #else -HARDWARE_INTRINSIC(Vector, ToScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsScalarT) +HARDWARE_INTRINSIC(Vector, ToScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsScalarT) #endif -HARDWARE_INTRINSIC(Vector, Truncate, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, UnzipEven, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, UnzipOdd, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, WidenLower, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, WidenUpper, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, WithElement, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoContainment|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, Truncate, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, UnzipEven, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, UnzipOdd, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, WidenLower, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, WidenUpper, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, WithElement, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoContainment|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) #if defined(TARGET_XARCH) || defined(TARGET_ARM64) -HARDWARE_INTRINSIC(Vector, WithLower, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, WithUpper, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, WithLower, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, WithUpper, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) #endif -HARDWARE_INTRINSIC(Vector, ZipLower, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, ZipUpper, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, get_AllBitsSet, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, get_E, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, get_Epsilon, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, get_Indices, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, get_NaN, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, get_NegativeInfinity, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, get_NegativeOne, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, get_NegativeZero, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, get_One, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, get_Pi, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, get_PositiveInfinity, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, get_SignSequence, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, get_Tau, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, get_Zero, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, op_Addition, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, op_BitwiseAnd, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, op_BitwiseOr, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, ZipLower, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, ZipUpper, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, get_AllBitsSet, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, get_E, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, get_Epsilon, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, get_Indices, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, get_NaN, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, get_NegativeInfinity, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, get_NegativeOne, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, get_NegativeZero, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, get_One, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, get_Pi, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, get_PositiveInfinity, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, get_SignSequence, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, get_Tau, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, get_Zero, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, op_Addition, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, op_BitwiseAnd, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, op_BitwiseOr, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) #if defined(TARGET_XARCH) -HARDWARE_INTRINSIC(Vector, op_Division, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialSideEffect_Other|HW_Flag_SpecialImport|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, op_Division, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialSideEffect_Other|HW_Flag_SpecialImport|HW_Flag_AvxOnlyCompatible) #else -HARDWARE_INTRINSIC(Vector, op_Division, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, op_Division, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) #endif -HARDWARE_INTRINSIC(Vector, op_Equality, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsBoolean|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, op_ExclusiveOr, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, op_Inequality, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsBoolean|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, op_LeftShift, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, op_Multiply, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, op_OnesComplement, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, op_RightShift, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, op_Subtraction, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, op_UnaryNegation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, op_UnaryPlus, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, op_UnsignedRightShift, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, op_Equality, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsBoolean|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, op_ExclusiveOr, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, op_Inequality, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsBoolean|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, op_LeftShift, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, op_Multiply, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, op_OnesComplement, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, op_RightShift, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, op_Subtraction, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, op_UnaryNegation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, op_UnaryPlus, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, op_UnsignedRightShift, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) #define LAST_NI_Vector NI_Vector_op_UnsignedRightShift #if defined(TARGET_XARCH) diff --git a/src/coreclr/jit/hwintrinsiclistarm64.h b/src/coreclr/jit/hwintrinsiclistarm64.h index c04f9bf9a8be30..35bc5966184561 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64.h +++ b/src/coreclr/jit/hwintrinsiclistarm64.h @@ -16,276 +16,276 @@ // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // AdvSimd Intrinsics #define FIRST_NI_AdvSimd NI_AdvSimd_Abs -HARDWARE_INTRINSIC(AdvSimd, Abs, -1, 1, INS_abs, INS_invalid, INS_abs, INS_invalid, INS_abs, INS_invalid, INS_invalid, INS_invalid, INS_fabs, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, AbsSaturate, -1, 1, INS_sqabs, INS_invalid, INS_sqabs, INS_invalid, INS_sqabs, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, AbsScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fabs, INS_fabs, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, AbsoluteCompareGreaterThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facgt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, AbsoluteCompareGreaterThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facge, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, AbsoluteCompareLessThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facgt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, AbsoluteCompareLessThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facge, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, AbsoluteDifference, -1, 2, INS_sabd, INS_uabd, INS_sabd, INS_uabd, INS_sabd, INS_uabd, INS_invalid, INS_invalid, INS_fabd, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, AbsoluteDifferenceAdd, -1, 3, INS_saba, INS_uaba, INS_saba, INS_uaba, INS_saba, INS_uaba, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, AbsoluteDifferenceWideningLower, 8, 2, INS_sabdl, INS_uabdl, INS_sabdl, INS_uabdl, INS_sabdl, INS_uabdl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, AbsoluteDifferenceWideningLowerAndAdd, 8, 3, INS_sabal, INS_uabal, INS_sabal, INS_uabal, INS_sabal, INS_uabal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, AbsoluteDifferenceWideningUpper, 16, 2, INS_sabdl2, INS_uabdl2, INS_sabdl2, INS_uabdl2, INS_sabdl2, INS_uabdl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, AbsoluteDifferenceWideningUpperAndAdd, 16, 3, INS_sabal2, INS_uabal2, INS_sabal2, INS_uabal2, INS_sabal2, INS_uabal2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, Add, -1, 2, INS_add, INS_add, INS_add, INS_add, INS_add, INS_add, INS_add, INS_add, INS_fadd, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, AddHighNarrowingLower, 8, 2, INS_addhn, INS_addhn, INS_addhn, INS_addhn, INS_addhn, INS_addhn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, AddHighNarrowingUpper, 16, 3, INS_addhn2, INS_addhn2, INS_addhn2, INS_addhn2, INS_addhn2, INS_addhn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, AddPairwise, 8, 2, INS_addp, INS_addp, INS_addp, INS_addp, INS_addp, INS_addp, INS_invalid, INS_invalid, INS_faddp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, AddPairwiseWidening, -1, 1, INS_saddlp, INS_uaddlp, INS_saddlp, INS_uaddlp, INS_saddlp, INS_uaddlp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, AddPairwiseWideningAndAdd, -1, 2, INS_sadalp, INS_uadalp, INS_sadalp, INS_uadalp, INS_sadalp, INS_uadalp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, AddPairwiseWideningAndAddScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sadalp, INS_uadalp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, AddPairwiseWideningScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_saddlp, INS_uaddlp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, AddRoundedHighNarrowingLower, 8, 2, INS_raddhn, INS_raddhn, INS_raddhn, INS_raddhn, INS_raddhn, INS_raddhn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, AddRoundedHighNarrowingUpper, 16, 3, INS_raddhn2, INS_raddhn2, INS_raddhn2, INS_raddhn2, INS_raddhn2, INS_raddhn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, AddSaturate, -1, 2, INS_sqadd, INS_uqadd, INS_sqadd, INS_uqadd, INS_sqadd, INS_uqadd, INS_sqadd, INS_uqadd, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, AddSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqadd, INS_uqadd, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, AddScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_add, INS_add, INS_fadd, INS_fadd, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, AddWideningLower, 8, 2, INS_saddl, INS_uaddl, INS_saddl, INS_uaddl, INS_saddl, INS_uaddl, INS_saddw, INS_uaddw, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, AddWideningUpper, 16, 2, INS_saddl2, INS_uaddl2, INS_saddl2, INS_uaddl2, INS_saddl2, INS_uaddl2, INS_saddw2, INS_uaddw2, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, And, -1, 2, INS_and, INS_and, INS_and, INS_and, INS_and, INS_and, INS_and, INS_and, INS_and, INS_and, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, BitwiseClear, -1, 2, INS_bic, INS_bic, INS_bic, INS_bic, INS_bic, INS_bic, INS_bic, INS_bic, INS_bic, INS_bic, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AdvSimd, BitwiseSelect, -1, 3, INS_bsl, INS_bsl, INS_bsl, INS_bsl, INS_bsl, INS_bsl, INS_bsl, INS_bsl, INS_bsl, INS_bsl, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, Ceiling, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, CeilingScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintp, INS_frintp, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, CompareEqual, -1, 2, INS_cmeq, INS_cmeq, INS_cmeq, INS_cmeq, INS_cmeq, INS_cmeq, INS_invalid, INS_invalid, INS_fcmeq, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AdvSimd, CompareGreaterThan, -1, 2, INS_cmgt, INS_cmhi, INS_cmgt, INS_cmhi, INS_cmgt, INS_cmhi, INS_invalid, INS_invalid, INS_fcmgt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AdvSimd, CompareGreaterThanOrEqual, -1, 2, INS_cmge, INS_cmhs, INS_cmge, INS_cmhs, INS_cmge, INS_cmhs, INS_invalid, INS_invalid, INS_fcmge, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AdvSimd, CompareLessThan, -1, 2, INS_cmgt, INS_cmhi, INS_cmgt, INS_cmhi, INS_cmgt, INS_cmhi, INS_invalid, INS_invalid, INS_fcmgt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AdvSimd, CompareLessThanOrEqual, -1, 2, INS_cmge, INS_cmhs, INS_cmge, INS_cmhs, INS_cmge, INS_cmhs, INS_invalid, INS_invalid, INS_fcmge, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AdvSimd, CompareTest, -1, 2, INS_cmtst, INS_cmtst, INS_cmtst, INS_cmtst, INS_cmtst, INS_cmtst, INS_invalid, INS_invalid, INS_cmtst, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundAwayFromZero, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtas, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundAwayFromZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtas, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToEven, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtns, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToEvenScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtns, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToNegativeInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtms, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToNegativeInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtms, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToPositiveInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtps, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToPositiveInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtps, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToZero, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzs, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzs, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ConvertToSingle, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_scvtf, INS_ucvtf, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, ConvertToSingleScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_scvtf, INS_ucvtf, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundAwayFromZero, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtau, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundAwayFromZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtau, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToEven, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtnu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToEvenScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtnu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToNegativeInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtmu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToNegativeInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtmu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToPositiveInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtpu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToPositiveInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtpu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToZero, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, DivideScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fdiv, INS_fdiv, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, DuplicateSelectedScalarToVector128, -1, 2, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_invalid, INS_invalid, INS_dup, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, DuplicateSelectedScalarToVector64, -1, 2, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_invalid, INS_invalid, INS_dup, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, DuplicateToVector128, 16, 1, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_invalid, INS_invalid, INS_dup, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment) -HARDWARE_INTRINSIC(AdvSimd, DuplicateToVector64, 8, 1, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_invalid, INS_invalid, INS_dup, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment) -HARDWARE_INTRINSIC(AdvSimd, Extract, -1, 2, INS_smov, INS_umov, INS_smov, INS_umov, INS_smov, INS_umov, INS_umov, INS_umov, INS_dup, INS_dup, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsScalarT) -HARDWARE_INTRINSIC(AdvSimd, ExtractNarrowingLower, 8, 1, INS_xtn, INS_xtn, INS_xtn, INS_xtn, INS_xtn, INS_xtn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, ExtractNarrowingSaturateLower, 8, 1, INS_sqxtn, INS_uqxtn, INS_sqxtn, INS_uqxtn, INS_sqxtn, INS_uqxtn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, ExtractNarrowingSaturateUnsignedLower, 8, 1, INS_invalid, INS_sqxtun, INS_invalid, INS_sqxtun, INS_invalid, INS_sqxtun, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, ExtractNarrowingSaturateUnsignedUpper, 16, 2, INS_invalid, INS_sqxtun2, INS_invalid, INS_sqxtun2, INS_invalid, INS_sqxtun2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ExtractNarrowingSaturateUpper, 16, 2, INS_sqxtn2, INS_uqxtn2, INS_sqxtn2, INS_uqxtn2, INS_sqxtn2, INS_uqxtn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ExtractNarrowingUpper, 16, 2, INS_xtn2, INS_xtn2, INS_xtn2, INS_xtn2, INS_xtn2, INS_xtn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ExtractVector128, 16, 3, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, -1, -1, HW_Category_SIMD, HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, ExtractVector64, 8, 3, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_invalid, INS_invalid, INS_ext, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, Floor, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintm, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, FloorScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintm, INS_frintm, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, FusedAddHalving, -1, 2, INS_shadd, INS_uhadd, INS_shadd, INS_uhadd, INS_shadd, INS_uhadd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, FusedAddRoundedHalving, -1, 2, INS_srhadd, INS_urhadd, INS_srhadd, INS_urhadd, INS_srhadd, INS_urhadd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, FusedMultiplyAdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmla, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, FusedMultiplyAddNegatedScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fnmadd, INS_fnmadd, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, FusedMultiplyAddScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmadd, INS_fmadd, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, FusedMultiplySubtract, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmls, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, FusedMultiplySubtractNegatedScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fnmsub, INS_fnmsub, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, FusedMultiplySubtractScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmsub, INS_fmsub, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, FusedSubtractHalving, -1, 2, INS_shsub, INS_uhsub, INS_shsub, INS_uhsub, INS_shsub, INS_uhsub, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, Insert, -1, 3, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, -1, -1, HW_Category_SIMD, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment) -HARDWARE_INTRINSIC(AdvSimd, InsertScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ins, INS_ins, INS_invalid, INS_ins, -1, -1, HW_Category_SIMD, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, LeadingSignCount, -1, 1, INS_cls, INS_invalid, INS_cls, INS_invalid, INS_cls, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, LeadingZeroCount, -1, 1, INS_clz, INS_clz, INS_clz, INS_clz, INS_clz, INS_clz, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, Load2xVector64, 8, 1, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_invalid, INS_invalid, INS_ld1_2regs, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, Load2xVector64AndUnzip, 8, 1, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_invalid, INS_invalid, INS_ld2, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, Load3xVector64, 8, 1, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_invalid, INS_invalid, INS_ld1_3regs, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, Load3xVector64AndUnzip, 8, 1, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_invalid, INS_invalid, INS_ld3, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, Load4xVector64, 8, 1, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_invalid, INS_invalid, INS_ld1_4regs, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, Load4xVector64AndUnzip, 8, 1, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_invalid, INS_invalid, INS_ld4, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, LoadAndInsertScalar, -1, 3, INS_ld1, INS_ld1, INS_ld1, INS_ld1, INS_ld1, INS_ld1, INS_ld1, INS_ld1, INS_ld1, INS_ld1, -1, -1, HW_Category_MemoryLoad, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, LoadAndInsertScalarVector64x2, 8, 3, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_invalid, INS_invalid, INS_ld2, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, LoadAndInsertScalarVector64x3, 8, 3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_invalid, INS_invalid, INS_ld3, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, LoadAndInsertScalarVector64x4, 8, 3, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_invalid, INS_invalid, INS_ld4, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, LoadAndReplicateToVector128, 16, 1, INS_ld1r, INS_ld1r, INS_ld1r, INS_ld1r, INS_ld1r, INS_ld1r, INS_invalid, INS_invalid, INS_ld1r, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, LoadAndReplicateToVector64, 8, 1, INS_ld1r, INS_ld1r, INS_ld1r, INS_ld1r, INS_ld1r, INS_ld1r, INS_invalid, INS_invalid, INS_ld1r, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, LoadAndReplicateToVector64x2, 8, 1, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_invalid, INS_invalid, INS_ld2r, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, LoadAndReplicateToVector64x3, 8, 1, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_invalid, INS_invalid, INS_ld3r, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, LoadAndReplicateToVector64x4, 8, 1, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_invalid, INS_invalid, INS_ld4r, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, LoadVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AdvSimd, LoadVector64, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AdvSimd, Max, -1, 2, INS_smax, INS_umax, INS_smax, INS_umax, INS_smax, INS_umax, INS_invalid, INS_invalid, INS_fmax, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, MaxNumber, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmaxnm, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, MaxNumberScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmaxnm, INS_fmaxnm, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, MaxPairwise, 8, 2, INS_smaxp, INS_umaxp, INS_smaxp, INS_umaxp, INS_smaxp, INS_umaxp, INS_invalid, INS_invalid, INS_fmaxp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, Min, -1, 2, INS_smin, INS_umin, INS_smin, INS_umin, INS_smin, INS_umin, INS_invalid, INS_invalid, INS_fmin, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, MinNumber, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fminnm, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, MinNumberScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fminnm, INS_fminnm, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, MinPairwise, 8, 2, INS_sminp, INS_uminp, INS_sminp, INS_uminp, INS_sminp, INS_uminp, INS_invalid, INS_invalid, INS_fminp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, Multiply, -1, 2, INS_mul, INS_mul, INS_mul, INS_mul, INS_mul, INS_mul, INS_invalid, INS_invalid, INS_fmul, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, MultiplyAdd, -1, 3, INS_mla, INS_mla, INS_mla, INS_mla, INS_mla, INS_mla, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyAddByScalar, -1, 3, INS_invalid, INS_invalid, INS_mla, INS_mla, INS_mla, INS_mla, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyAddBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_mla, INS_mla, INS_mla, INS_mla, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyByScalar, -1, 2, INS_invalid, INS_invalid, INS_mul, INS_mul, INS_mul, INS_mul, INS_invalid, INS_invalid, INS_fmul, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, MultiplyBySelectedScalar, -1, 3, INS_invalid, INS_invalid, INS_mul, INS_mul, INS_mul, INS_mul, INS_invalid, INS_invalid, INS_fmul, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, MultiplyBySelectedScalarWideningLower, 8, 3, INS_invalid, INS_invalid, INS_smull, INS_umull, INS_smull, INS_umull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, MultiplyBySelectedScalarWideningLowerAndAdd, 8, 4, INS_invalid, INS_invalid, INS_smlal, INS_umlal, INS_smlal, INS_umlal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyBySelectedScalarWideningLowerAndSubtract, 8, 4, INS_invalid, INS_invalid, INS_smlsl, INS_umlsl, INS_smlsl, INS_umlsl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyBySelectedScalarWideningUpper, 16, 3, INS_invalid, INS_invalid, INS_smull2, INS_umull2, INS_smull2, INS_umull2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, MultiplyBySelectedScalarWideningUpperAndAdd, 16, 4, INS_invalid, INS_invalid, INS_smlal2, INS_umlal2, INS_smlal2, INS_umlal2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyBySelectedScalarWideningUpperAndSubtract, 16, 4, INS_invalid, INS_invalid, INS_smlsl2, INS_umlsl2, INS_smlsl2, INS_umlsl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingByScalarSaturateHigh, -1, 2, INS_invalid, INS_invalid, INS_sqdmulh, INS_invalid, INS_sqdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingBySelectedScalarSaturateHigh, -1, 3, INS_invalid, INS_invalid, INS_sqdmulh, INS_invalid, INS_sqdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingSaturateHigh, -1, 2, INS_invalid, INS_invalid, INS_sqdmulh, INS_invalid, INS_sqdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningLowerAndAddSaturate, 8, 3, INS_invalid, INS_invalid, INS_sqdmlal, INS_invalid, INS_sqdmlal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningLowerAndSubtractSaturate, 8, 3, INS_invalid, INS_invalid, INS_sqdmlsl, INS_invalid, INS_sqdmlsl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningLowerByScalarAndAddSaturate, 8, 3, INS_invalid, INS_invalid, INS_sqdmlal, INS_invalid, INS_sqdmlal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningLowerByScalarAndSubtractSaturate, 8, 3, INS_invalid, INS_invalid, INS_sqdmlsl, INS_invalid, INS_sqdmlsl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningLowerBySelectedScalarAndAddSaturate, 8, 4, INS_invalid, INS_invalid, INS_sqdmlal, INS_invalid, INS_sqdmlal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningLowerBySelectedScalarAndSubtractSaturate, 8, 4, INS_invalid, INS_invalid, INS_sqdmlsl, INS_invalid, INS_sqdmlsl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningSaturateLower, 8, 2, INS_invalid, INS_invalid, INS_sqdmull, INS_invalid, INS_sqdmull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningSaturateLowerByScalar, 8, 2, INS_invalid, INS_invalid, INS_sqdmull, INS_invalid, INS_sqdmull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningSaturateLowerBySelectedScalar, 8, 3, INS_invalid, INS_invalid, INS_sqdmull, INS_invalid, INS_sqdmull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningSaturateUpper, 16, 2, INS_invalid, INS_invalid, INS_sqdmull2, INS_invalid, INS_sqdmull2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningSaturateUpperByScalar, 16, 2, INS_invalid, INS_invalid, INS_sqdmull2, INS_invalid, INS_sqdmull2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningSaturateUpperBySelectedScalar, 16, 3, INS_invalid, INS_invalid, INS_sqdmull2, INS_invalid, INS_sqdmull2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningUpperAndAddSaturate, 16, 3, INS_invalid, INS_invalid, INS_sqdmlal2, INS_invalid, INS_sqdmlal2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningUpperAndSubtractSaturate, 16, 3, INS_invalid, INS_invalid, INS_sqdmlsl2, INS_invalid, INS_sqdmlsl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningUpperByScalarAndAddSaturate, 16, 3, INS_invalid, INS_invalid, INS_sqdmlal2, INS_invalid, INS_sqdmlal2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningUpperByScalarAndSubtractSaturate, 16, 3, INS_invalid, INS_invalid, INS_sqdmlsl2, INS_invalid, INS_sqdmlsl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningUpperBySelectedScalarAndAddSaturate, 16, 4, INS_invalid, INS_invalid, INS_sqdmlal2, INS_invalid, INS_sqdmlal2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningUpperBySelectedScalarAndSubtractSaturate, 16, 4, INS_invalid, INS_invalid, INS_sqdmlsl2, INS_invalid, INS_sqdmlsl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyRoundedDoublingByScalarSaturateHigh, -1, 2, INS_invalid, INS_invalid, INS_sqrdmulh, INS_invalid, INS_sqrdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, MultiplyRoundedDoublingBySelectedScalarSaturateHigh, -1, 3, INS_invalid, INS_invalid, INS_sqrdmulh, INS_invalid, INS_sqrdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, MultiplyRoundedDoublingSaturateHigh, -1, 2, INS_invalid, INS_invalid, INS_sqrdmulh, INS_invalid, INS_sqrdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, MultiplyScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmul, INS_fmul, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, MultiplyScalarBySelectedScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmul, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, MultiplySubtract, -1, 3, INS_mls, INS_mls, INS_mls, INS_mls, INS_mls, INS_mls, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplySubtractByScalar, -1, 3, INS_invalid, INS_invalid, INS_mls, INS_mls, INS_mls, INS_mls, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplySubtractBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_mls, INS_mls, INS_mls, INS_mls, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyWideningLower, 8, 2, INS_smull, INS_umull, INS_smull, INS_umull, INS_smull, INS_umull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, MultiplyWideningLowerAndAdd, 8, 3, INS_smlal, INS_umlal, INS_smlal, INS_umlal, INS_smlal, INS_umlal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyWideningLowerAndSubtract, 8, 3, INS_smlsl, INS_umlsl, INS_smlsl, INS_umlsl, INS_smlsl, INS_umlsl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyWideningUpper, 16, 2, INS_smull2, INS_umull2, INS_smull2, INS_umull2, INS_smull2, INS_umull2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, MultiplyWideningUpperAndAdd, 16, 3, INS_smlal2, INS_umlal2, INS_smlal2, INS_umlal2, INS_smlal2, INS_umlal2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyWideningUpperAndSubtract, 16, 3, INS_smlsl2, INS_umlsl2, INS_smlsl2, INS_umlsl2, INS_smlsl2, INS_umlsl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, Negate, -1, 1, INS_neg, INS_invalid, INS_neg, INS_invalid, INS_neg, INS_invalid, INS_invalid, INS_invalid, INS_fneg, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, NegateSaturate, -1, 1, INS_sqneg, INS_invalid, INS_sqneg, INS_invalid, INS_sqneg, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, NegateScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fneg, INS_fneg, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, Not, -1, 1, INS_mvn, INS_mvn, INS_mvn, INS_mvn, INS_mvn, INS_mvn, INS_mvn, INS_mvn, INS_mvn, INS_mvn, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, Or, -1, 2, INS_orr, INS_orr, INS_orr, INS_orr, INS_orr, INS_orr, INS_orr, INS_orr, INS_orr, INS_orr, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, OrNot, -1, 2, INS_orn, INS_orn, INS_orn, INS_orn, INS_orn, INS_orn, INS_orn, INS_orn, INS_orn, INS_orn, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AdvSimd, PolynomialMultiply, -1, 2, INS_pmul, INS_pmul, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, PolynomialMultiplyWideningLower, 8, 2, INS_pmull, INS_pmull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, PolynomialMultiplyWideningUpper, 16, 2, INS_pmull2, INS_pmull2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, PopCount, -1, 1, INS_cnt, INS_cnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, ReciprocalEstimate, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_urecpe, INS_invalid, INS_invalid, INS_frecpe, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, ReciprocalSquareRootEstimate, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ursqrte, INS_invalid, INS_invalid, INS_frsqrte, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, ReciprocalSquareRootStep, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frsqrts, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, ReciprocalStep, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frecps, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, ReverseElement16, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rev32, INS_rev32, INS_rev64, INS_rev64, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, ReverseElement32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rev64, INS_rev64, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, ReverseElement8, -1, 1, INS_invalid, INS_invalid, INS_rev16, INS_rev16, INS_rev32, INS_rev32, INS_rev64, INS_rev64, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, RoundAwayFromZero, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frinta, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, RoundAwayFromZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frinta, INS_frinta, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, RoundToNearest, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintn, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, RoundToNearestScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintn, INS_frintn, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, RoundToNegativeInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintm, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, RoundToNegativeInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintm, INS_frintm, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, RoundToPositiveInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, RoundToPositiveInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintp, INS_frintp, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, RoundToZero, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintz, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, RoundToZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintz, INS_frintz, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftArithmetic, -1, 2, INS_sshl, INS_invalid, INS_sshl, INS_invalid, INS_sshl, INS_invalid, INS_sshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, ShiftArithmeticRounded, -1, 2, INS_srshl, INS_invalid, INS_srshl, INS_invalid, INS_srshl, INS_invalid, INS_srshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, ShiftArithmeticRoundedSaturate, -1, 2, INS_sqrshl, INS_invalid, INS_sqrshl, INS_invalid, INS_sqrshl, INS_invalid, INS_sqrshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, ShiftArithmeticRoundedSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqrshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftArithmeticRoundedScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_srshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftArithmeticSaturate, -1, 2, INS_sqshl, INS_invalid, INS_sqshl, INS_invalid, INS_sqshl, INS_invalid, INS_sqshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, ShiftArithmeticSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftArithmeticScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftLeftAndInsert, -1, 3, INS_sli, INS_sli, INS_sli, INS_sli, INS_sli, INS_sli, INS_sli, INS_sli, INS_sli, INS_sli, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ShiftLeftAndInsertScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sli, INS_sli, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogical, -1, 2, INS_shl, INS_shl, INS_shl, INS_shl, INS_shl, INS_shl, INS_shl, INS_shl, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_NoJmpTableIMM) -HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalSaturate, -1, 2, INS_sqshl, INS_uqshl, INS_sqshl, INS_uqshl, INS_sqshl, INS_uqshl, INS_sqshl, INS_uqshl, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqshl, INS_uqshl, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalSaturateUnsigned, -1, 2, INS_sqshlu, INS_invalid, INS_sqshlu, INS_invalid, INS_sqshlu, INS_invalid, INS_sqshlu, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalSaturateUnsignedScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqshlu, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_shl, INS_shl, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar|HW_Flag_NoJmpTableIMM) -HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalWideningLower, 8, 2, INS_sshll, INS_ushll, INS_sshll, INS_ushll, INS_sshll, INS_ushll, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalWideningUpper, 16, 2, INS_sshll2, INS_ushll2, INS_sshll2, INS_ushll2, INS_sshll2, INS_ushll2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, ShiftLogical, -1, 2, INS_ushl, INS_ushl, INS_ushl, INS_ushl, INS_ushl, INS_ushl, INS_ushl, INS_ushl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, ShiftLogicalRounded, -1, 2, INS_urshl, INS_urshl, INS_urshl, INS_urshl, INS_urshl, INS_urshl, INS_urshl, INS_urshl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, ShiftLogicalRoundedSaturate, -1, 2, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, ShiftLogicalRoundedSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_uqrshl, INS_uqrshl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftLogicalRoundedScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_urshl, INS_urshl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftLogicalSaturate, -1, 2, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, ShiftLogicalSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_uqshl, INS_uqshl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftLogicalScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ushl, INS_ushl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightAndInsert, -1, 3, INS_sri, INS_sri, INS_sri, INS_sri, INS_sri, INS_sri, INS_sri, INS_sri, INS_sri, INS_sri, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightAndInsertScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sri, INS_sri, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmetic, -1, 2, INS_sshr, INS_invalid, INS_sshr, INS_invalid, INS_sshr, INS_invalid, INS_sshr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_NoJmpTableIMM) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticAdd, -1, 3, INS_ssra, INS_invalid, INS_ssra, INS_invalid, INS_ssra, INS_invalid, INS_ssra, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticAddScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ssra, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticNarrowingSaturateLower, 8, 2, INS_sqshrn, INS_invalid, INS_sqshrn, INS_invalid, INS_sqshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticNarrowingSaturateUnsignedLower, 8, 2, INS_invalid, INS_sqshrun, INS_invalid, INS_sqshrun, INS_invalid, INS_sqshrun, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticNarrowingSaturateUnsignedUpper, 16, 3, INS_invalid, INS_sqshrun2, INS_invalid, INS_sqshrun2, INS_invalid, INS_sqshrun2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticNarrowingSaturateUpper, 16, 3, INS_sqshrn2, INS_invalid, INS_sqshrn2, INS_invalid, INS_sqshrn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRounded, -1, 2, INS_srshr, INS_invalid, INS_srshr, INS_invalid, INS_srshr, INS_invalid, INS_srshr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedAdd, -1, 3, INS_srsra, INS_invalid, INS_srsra, INS_invalid, INS_srsra, INS_invalid, INS_srsra, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedAddScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_srsra, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedNarrowingSaturateLower, 8, 2, INS_sqrshrn, INS_invalid, INS_sqrshrn, INS_invalid, INS_sqrshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedNarrowingSaturateUnsignedLower, 8, 2, INS_invalid, INS_sqrshrun, INS_invalid, INS_sqrshrun, INS_invalid, INS_sqrshrun, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedNarrowingSaturateUnsignedUpper, 16, 3, INS_invalid, INS_sqrshrun2, INS_invalid, INS_sqrshrun2, INS_invalid, INS_sqrshrun2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedNarrowingSaturateUpper, 16, 3, INS_sqrshrn2, INS_invalid, INS_sqrshrn2, INS_invalid, INS_sqrshrn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_srshr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sshr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar|HW_Flag_NoJmpTableIMM) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogical, -1, 2, INS_ushr, INS_ushr, INS_ushr, INS_ushr, INS_ushr, INS_ushr, INS_ushr, INS_ushr, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_NoJmpTableIMM) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalAdd, -1, 3, INS_usra, INS_usra, INS_usra, INS_usra, INS_usra, INS_usra, INS_usra, INS_usra, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalAddScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_usra, INS_usra, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalNarrowingLower, 8, 2, INS_shrn, INS_shrn, INS_shrn, INS_shrn, INS_shrn, INS_shrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalNarrowingSaturateLower, 8, 2, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalNarrowingSaturateUpper, 16, 3, INS_uqshrn2, INS_uqshrn2, INS_uqshrn2, INS_uqshrn2, INS_uqshrn2, INS_uqshrn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalNarrowingUpper, 16, 3, INS_shrn2, INS_shrn2, INS_shrn2, INS_shrn2, INS_shrn2, INS_shrn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRounded, -1, 2, INS_urshr, INS_urshr, INS_urshr, INS_urshr, INS_urshr, INS_urshr, INS_urshr, INS_urshr, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedAdd, -1, 3, INS_ursra, INS_ursra, INS_ursra, INS_ursra, INS_ursra, INS_ursra, INS_ursra, INS_ursra, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedAddScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ursra, INS_ursra, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedNarrowingLower, 8, 2, INS_rshrn, INS_rshrn, INS_rshrn, INS_rshrn, INS_rshrn, INS_rshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedNarrowingSaturateLower, 8, 2, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedNarrowingSaturateUpper, 16, 3, INS_uqrshrn2, INS_uqrshrn2, INS_uqrshrn2, INS_uqrshrn2, INS_uqrshrn2, INS_uqrshrn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedNarrowingUpper, 16, 3, INS_rshrn2, INS_rshrn2, INS_rshrn2, INS_rshrn2, INS_rshrn2, INS_rshrn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_urshr, INS_urshr, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ushr, INS_ushr, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar|HW_Flag_NoJmpTableIMM) -HARDWARE_INTRINSIC(AdvSimd, SignExtendWideningLower, 8, 1, INS_sxtl, INS_invalid, INS_sxtl, INS_invalid, INS_sxtl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, SignExtendWideningUpper, 16, 1, INS_sxtl2, INS_invalid, INS_sxtl2, INS_invalid, INS_sxtl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, SqrtScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fsqrt, INS_fsqrt, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, Store, -1, 2, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_invalid, INS_invalid, INS_st1_2regs, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_BaseTypeFromValueTupleArg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, StoreSelectedScalar, -1, 3, INS_st1, INS_st1, INS_st1, INS_st1, INS_st1, INS_st1, INS_st1, INS_st1, INS_st1, INS_st1, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_BaseTypeFromValueTupleArg|HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, StoreVectorAndZip, 8, 2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_BaseTypeFromValueTupleArg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, Subtract, -1, 2, INS_sub, INS_sub, INS_sub, INS_sub, INS_sub, INS_sub, INS_sub, INS_sub, INS_fsub, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, SubtractHighNarrowingLower, 8, 2, INS_subhn, INS_subhn, INS_subhn, INS_subhn, INS_subhn, INS_subhn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, SubtractHighNarrowingUpper, 16, 3, INS_subhn2, INS_subhn2, INS_subhn2, INS_subhn2, INS_subhn2, INS_subhn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, SubtractRoundedHighNarrowingLower, 8, 2, INS_rsubhn, INS_rsubhn, INS_rsubhn, INS_rsubhn, INS_rsubhn, INS_rsubhn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, SubtractRoundedHighNarrowingUpper, 16, 3, INS_rsubhn2, INS_rsubhn2, INS_rsubhn2, INS_rsubhn2, INS_rsubhn2, INS_rsubhn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, SubtractSaturate, -1, 2, INS_sqsub, INS_uqsub, INS_sqsub, INS_uqsub, INS_sqsub, INS_uqsub, INS_sqsub, INS_uqsub, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, SubtractSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqsub, INS_uqsub, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, SubtractScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sub, INS_sub, INS_fsub, INS_fsub, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, SubtractWideningLower, 8, 2, INS_ssubl, INS_usubl, INS_ssubl, INS_usubl, INS_ssubl, INS_usubl, INS_ssubw, INS_usubw, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, SubtractWideningUpper, 16, 2, INS_ssubl2, INS_usubl2, INS_ssubl2, INS_usubl2, INS_ssubl2, INS_usubl2, INS_ssubw2, INS_usubw2, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, VectorTableLookup, 8, 2, INS_tbl, INS_tbl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, VectorTableLookupExtension, 8, 3, INS_tbx, INS_tbx, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, Xor, -1, 2, INS_eor, INS_eor, INS_eor, INS_eor, INS_eor, INS_eor, INS_eor, INS_eor, INS_eor, INS_eor, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, ZeroExtendWideningLower, 8, 1, INS_uxtl, INS_uxtl, INS_uxtl, INS_uxtl, INS_uxtl, INS_uxtl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, ZeroExtendWideningUpper, 16, 1, INS_uxtl2, INS_uxtl2, INS_uxtl2, INS_uxtl2, INS_uxtl2, INS_uxtl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, Abs, -1, 1, INS_abs, INS_invalid, INS_abs, INS_invalid, INS_abs, INS_invalid, INS_invalid, INS_invalid, INS_fabs, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, AbsSaturate, -1, 1, INS_sqabs, INS_invalid, INS_sqabs, INS_invalid, INS_sqabs, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, AbsScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fabs, INS_fabs, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, AbsoluteCompareGreaterThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facgt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, AbsoluteCompareGreaterThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facge, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, AbsoluteCompareLessThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facgt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, AbsoluteCompareLessThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facge, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, AbsoluteDifference, -1, 2, INS_sabd, INS_uabd, INS_sabd, INS_uabd, INS_sabd, INS_uabd, INS_invalid, INS_invalid, INS_fabd, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, AbsoluteDifferenceAdd, -1, 3, INS_saba, INS_uaba, INS_saba, INS_uaba, INS_saba, INS_uaba, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, AbsoluteDifferenceWideningLower, 8, 2, INS_sabdl, INS_uabdl, INS_sabdl, INS_uabdl, INS_sabdl, INS_uabdl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, AbsoluteDifferenceWideningLowerAndAdd, 8, 3, INS_sabal, INS_uabal, INS_sabal, INS_uabal, INS_sabal, INS_uabal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, AbsoluteDifferenceWideningUpper, 16, 2, INS_sabdl2, INS_uabdl2, INS_sabdl2, INS_uabdl2, INS_sabdl2, INS_uabdl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, AbsoluteDifferenceWideningUpperAndAdd, 16, 3, INS_sabal2, INS_uabal2, INS_sabal2, INS_uabal2, INS_sabal2, INS_uabal2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, Add, -1, 2, INS_add, INS_add, INS_add, INS_add, INS_add, INS_add, INS_add, INS_add, INS_fadd, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, AddHighNarrowingLower, 8, 2, INS_addhn, INS_addhn, INS_addhn, INS_addhn, INS_addhn, INS_addhn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, AddHighNarrowingUpper, 16, 3, INS_addhn2, INS_addhn2, INS_addhn2, INS_addhn2, INS_addhn2, INS_addhn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, AddPairwise, 8, 2, INS_addp, INS_addp, INS_addp, INS_addp, INS_addp, INS_addp, INS_invalid, INS_invalid, INS_faddp, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, AddPairwiseWidening, -1, 1, INS_saddlp, INS_uaddlp, INS_saddlp, INS_uaddlp, INS_saddlp, INS_uaddlp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, AddPairwiseWideningAndAdd, -1, 2, INS_sadalp, INS_uadalp, INS_sadalp, INS_uadalp, INS_sadalp, INS_uadalp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, AddPairwiseWideningAndAddScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sadalp, INS_uadalp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, AddPairwiseWideningScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_saddlp, INS_uaddlp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, AddRoundedHighNarrowingLower, 8, 2, INS_raddhn, INS_raddhn, INS_raddhn, INS_raddhn, INS_raddhn, INS_raddhn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, AddRoundedHighNarrowingUpper, 16, 3, INS_raddhn2, INS_raddhn2, INS_raddhn2, INS_raddhn2, INS_raddhn2, INS_raddhn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, AddSaturate, -1, 2, INS_sqadd, INS_uqadd, INS_sqadd, INS_uqadd, INS_sqadd, INS_uqadd, INS_sqadd, INS_uqadd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, AddSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqadd, INS_uqadd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, AddScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_add, INS_add, INS_fadd, INS_fadd, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, AddWideningLower, 8, 2, INS_saddl, INS_uaddl, INS_saddl, INS_uaddl, INS_saddl, INS_uaddl, INS_saddw, INS_uaddw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, AddWideningUpper, 16, 2, INS_saddl2, INS_uaddl2, INS_saddl2, INS_uaddl2, INS_saddl2, INS_uaddl2, INS_saddw2, INS_uaddw2, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, And, -1, 2, INS_and, INS_and, INS_and, INS_and, INS_and, INS_and, INS_and, INS_and, INS_and, INS_and, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, BitwiseClear, -1, 2, INS_bic, INS_bic, INS_bic, INS_bic, INS_bic, INS_bic, INS_bic, INS_bic, INS_bic, INS_bic, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AdvSimd, BitwiseSelect, -1, 3, INS_bsl, INS_bsl, INS_bsl, INS_bsl, INS_bsl, INS_bsl, INS_bsl, INS_bsl, INS_bsl, INS_bsl, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, Ceiling, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintp, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, CeilingScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintp, INS_frintp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, CompareEqual, -1, 2, INS_cmeq, INS_cmeq, INS_cmeq, INS_cmeq, INS_cmeq, INS_cmeq, INS_invalid, INS_invalid, INS_fcmeq, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AdvSimd, CompareGreaterThan, -1, 2, INS_cmgt, INS_cmhi, INS_cmgt, INS_cmhi, INS_cmgt, INS_cmhi, INS_invalid, INS_invalid, INS_fcmgt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AdvSimd, CompareGreaterThanOrEqual, -1, 2, INS_cmge, INS_cmhs, INS_cmge, INS_cmhs, INS_cmge, INS_cmhs, INS_invalid, INS_invalid, INS_fcmge, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AdvSimd, CompareLessThan, -1, 2, INS_cmgt, INS_cmhi, INS_cmgt, INS_cmhi, INS_cmgt, INS_cmhi, INS_invalid, INS_invalid, INS_fcmgt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AdvSimd, CompareLessThanOrEqual, -1, 2, INS_cmge, INS_cmhs, INS_cmge, INS_cmhs, INS_cmge, INS_cmhs, INS_invalid, INS_invalid, INS_fcmge, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AdvSimd, CompareTest, -1, 2, INS_cmtst, INS_cmtst, INS_cmtst, INS_cmtst, INS_cmtst, INS_cmtst, INS_invalid, INS_invalid, INS_cmtst, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundAwayFromZero, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtas, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundAwayFromZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtas, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToEven, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtns, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToEvenScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtns, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToNegativeInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtms, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToNegativeInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtms, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToPositiveInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtps, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToPositiveInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtps, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToZero, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzs, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzs, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ConvertToSingle, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_scvtf, INS_ucvtf, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, ConvertToSingleScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_scvtf, INS_ucvtf, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundAwayFromZero, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtau, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundAwayFromZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtau, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToEven, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtnu, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToEvenScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtnu, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToNegativeInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtmu, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToNegativeInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtmu, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToPositiveInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtpu, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToPositiveInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtpu, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToZero, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzu, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzu, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, DivideScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fdiv, INS_fdiv, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, DuplicateSelectedScalarToVector128, -1, 2, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_invalid, INS_invalid, INS_dup, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, DuplicateSelectedScalarToVector64, -1, 2, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_invalid, INS_invalid, INS_dup, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, DuplicateToVector128, 16, 1, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_invalid, INS_invalid, INS_dup, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment) +HARDWARE_INTRINSIC(AdvSimd, DuplicateToVector64, 8, 1, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_invalid, INS_invalid, INS_dup, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment) +HARDWARE_INTRINSIC(AdvSimd, Extract, -1, 2, INS_smov, INS_umov, INS_smov, INS_umov, INS_smov, INS_umov, INS_umov, INS_umov, INS_dup, INS_dup, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsScalarT) +HARDWARE_INTRINSIC(AdvSimd, ExtractNarrowingLower, 8, 1, INS_xtn, INS_xtn, INS_xtn, INS_xtn, INS_xtn, INS_xtn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, ExtractNarrowingSaturateLower, 8, 1, INS_sqxtn, INS_uqxtn, INS_sqxtn, INS_uqxtn, INS_sqxtn, INS_uqxtn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, ExtractNarrowingSaturateUnsignedLower, 8, 1, INS_invalid, INS_sqxtun, INS_invalid, INS_sqxtun, INS_invalid, INS_sqxtun, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, ExtractNarrowingSaturateUnsignedUpper, 16, 2, INS_invalid, INS_sqxtun2, INS_invalid, INS_sqxtun2, INS_invalid, INS_sqxtun2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ExtractNarrowingSaturateUpper, 16, 2, INS_sqxtn2, INS_uqxtn2, INS_sqxtn2, INS_uqxtn2, INS_sqxtn2, INS_uqxtn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ExtractNarrowingUpper, 16, 2, INS_xtn2, INS_xtn2, INS_xtn2, INS_xtn2, INS_xtn2, INS_xtn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ExtractVector128, 16, 3, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, ExtractVector64, 8, 3, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_invalid, INS_invalid, INS_ext, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, Floor, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintm, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, FloorScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintm, INS_frintm, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, FusedAddHalving, -1, 2, INS_shadd, INS_uhadd, INS_shadd, INS_uhadd, INS_shadd, INS_uhadd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, FusedAddRoundedHalving, -1, 2, INS_srhadd, INS_urhadd, INS_srhadd, INS_urhadd, INS_srhadd, INS_urhadd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, FusedMultiplyAdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmla, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, FusedMultiplyAddNegatedScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fnmadd, INS_fnmadd, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, FusedMultiplyAddScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmadd, INS_fmadd, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, FusedMultiplySubtract, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmls, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, FusedMultiplySubtractNegatedScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fnmsub, INS_fnmsub, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, FusedMultiplySubtractScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmsub, INS_fmsub, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, FusedSubtractHalving, -1, 2, INS_shsub, INS_uhsub, INS_shsub, INS_uhsub, INS_shsub, INS_uhsub, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, Insert, -1, 3, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment) +HARDWARE_INTRINSIC(AdvSimd, InsertScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ins, INS_ins, INS_invalid, INS_ins, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, LeadingSignCount, -1, 1, INS_cls, INS_invalid, INS_cls, INS_invalid, INS_cls, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, LeadingZeroCount, -1, 1, INS_clz, INS_clz, INS_clz, INS_clz, INS_clz, INS_clz, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, Load2xVector64, 8, 1, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_invalid, INS_invalid, INS_ld1_2regs, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, Load2xVector64AndUnzip, 8, 1, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_invalid, INS_invalid, INS_ld2, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, Load3xVector64, 8, 1, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_invalid, INS_invalid, INS_ld1_3regs, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, Load3xVector64AndUnzip, 8, 1, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_invalid, INS_invalid, INS_ld3, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, Load4xVector64, 8, 1, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_invalid, INS_invalid, INS_ld1_4regs, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, Load4xVector64AndUnzip, 8, 1, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_invalid, INS_invalid, INS_ld4, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, LoadAndInsertScalar, -1, 3, INS_ld1, INS_ld1, INS_ld1, INS_ld1, INS_ld1, INS_ld1, INS_ld1, INS_ld1, INS_ld1, INS_ld1, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, LoadAndInsertScalarVector64x2, 8, 3, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_invalid, INS_invalid, INS_ld2, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, LoadAndInsertScalarVector64x3, 8, 3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_invalid, INS_invalid, INS_ld3, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, LoadAndInsertScalarVector64x4, 8, 3, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_invalid, INS_invalid, INS_ld4, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, LoadAndReplicateToVector128, 16, 1, INS_ld1r, INS_ld1r, INS_ld1r, INS_ld1r, INS_ld1r, INS_ld1r, INS_invalid, INS_invalid, INS_ld1r, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, LoadAndReplicateToVector64, 8, 1, INS_ld1r, INS_ld1r, INS_ld1r, INS_ld1r, INS_ld1r, INS_ld1r, INS_invalid, INS_invalid, INS_ld1r, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, LoadAndReplicateToVector64x2, 8, 1, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_invalid, INS_invalid, INS_ld2r, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, LoadAndReplicateToVector64x3, 8, 1, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_invalid, INS_invalid, INS_ld3r, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, LoadAndReplicateToVector64x4, 8, 1, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_invalid, INS_invalid, INS_ld4r, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, LoadVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AdvSimd, LoadVector64, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AdvSimd, Max, -1, 2, INS_smax, INS_umax, INS_smax, INS_umax, INS_smax, INS_umax, INS_invalid, INS_invalid, INS_fmax, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, MaxNumber, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmaxnm, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, MaxNumberScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmaxnm, INS_fmaxnm, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, MaxPairwise, 8, 2, INS_smaxp, INS_umaxp, INS_smaxp, INS_umaxp, INS_smaxp, INS_umaxp, INS_invalid, INS_invalid, INS_fmaxp, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, Min, -1, 2, INS_smin, INS_umin, INS_smin, INS_umin, INS_smin, INS_umin, INS_invalid, INS_invalid, INS_fmin, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, MinNumber, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fminnm, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, MinNumberScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fminnm, INS_fminnm, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, MinPairwise, 8, 2, INS_sminp, INS_uminp, INS_sminp, INS_uminp, INS_sminp, INS_uminp, INS_invalid, INS_invalid, INS_fminp, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, Multiply, -1, 2, INS_mul, INS_mul, INS_mul, INS_mul, INS_mul, INS_mul, INS_invalid, INS_invalid, INS_fmul, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, MultiplyAdd, -1, 3, INS_mla, INS_mla, INS_mla, INS_mla, INS_mla, INS_mla, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyAddByScalar, -1, 3, INS_invalid, INS_invalid, INS_mla, INS_mla, INS_mla, INS_mla, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyAddBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_mla, INS_mla, INS_mla, INS_mla, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyByScalar, -1, 2, INS_invalid, INS_invalid, INS_mul, INS_mul, INS_mul, INS_mul, INS_invalid, INS_invalid, INS_fmul, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, MultiplyBySelectedScalar, -1, 3, INS_invalid, INS_invalid, INS_mul, INS_mul, INS_mul, INS_mul, INS_invalid, INS_invalid, INS_fmul, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, MultiplyBySelectedScalarWideningLower, 8, 3, INS_invalid, INS_invalid, INS_smull, INS_umull, INS_smull, INS_umull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, MultiplyBySelectedScalarWideningLowerAndAdd, 8, 4, INS_invalid, INS_invalid, INS_smlal, INS_umlal, INS_smlal, INS_umlal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyBySelectedScalarWideningLowerAndSubtract, 8, 4, INS_invalid, INS_invalid, INS_smlsl, INS_umlsl, INS_smlsl, INS_umlsl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyBySelectedScalarWideningUpper, 16, 3, INS_invalid, INS_invalid, INS_smull2, INS_umull2, INS_smull2, INS_umull2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, MultiplyBySelectedScalarWideningUpperAndAdd, 16, 4, INS_invalid, INS_invalid, INS_smlal2, INS_umlal2, INS_smlal2, INS_umlal2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyBySelectedScalarWideningUpperAndSubtract, 16, 4, INS_invalid, INS_invalid, INS_smlsl2, INS_umlsl2, INS_smlsl2, INS_umlsl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingByScalarSaturateHigh, -1, 2, INS_invalid, INS_invalid, INS_sqdmulh, INS_invalid, INS_sqdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingBySelectedScalarSaturateHigh, -1, 3, INS_invalid, INS_invalid, INS_sqdmulh, INS_invalid, INS_sqdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingSaturateHigh, -1, 2, INS_invalid, INS_invalid, INS_sqdmulh, INS_invalid, INS_sqdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningLowerAndAddSaturate, 8, 3, INS_invalid, INS_invalid, INS_sqdmlal, INS_invalid, INS_sqdmlal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningLowerAndSubtractSaturate, 8, 3, INS_invalid, INS_invalid, INS_sqdmlsl, INS_invalid, INS_sqdmlsl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningLowerByScalarAndAddSaturate, 8, 3, INS_invalid, INS_invalid, INS_sqdmlal, INS_invalid, INS_sqdmlal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningLowerByScalarAndSubtractSaturate, 8, 3, INS_invalid, INS_invalid, INS_sqdmlsl, INS_invalid, INS_sqdmlsl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningLowerBySelectedScalarAndAddSaturate, 8, 4, INS_invalid, INS_invalid, INS_sqdmlal, INS_invalid, INS_sqdmlal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningLowerBySelectedScalarAndSubtractSaturate, 8, 4, INS_invalid, INS_invalid, INS_sqdmlsl, INS_invalid, INS_sqdmlsl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningSaturateLower, 8, 2, INS_invalid, INS_invalid, INS_sqdmull, INS_invalid, INS_sqdmull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningSaturateLowerByScalar, 8, 2, INS_invalid, INS_invalid, INS_sqdmull, INS_invalid, INS_sqdmull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningSaturateLowerBySelectedScalar, 8, 3, INS_invalid, INS_invalid, INS_sqdmull, INS_invalid, INS_sqdmull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningSaturateUpper, 16, 2, INS_invalid, INS_invalid, INS_sqdmull2, INS_invalid, INS_sqdmull2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningSaturateUpperByScalar, 16, 2, INS_invalid, INS_invalid, INS_sqdmull2, INS_invalid, INS_sqdmull2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningSaturateUpperBySelectedScalar, 16, 3, INS_invalid, INS_invalid, INS_sqdmull2, INS_invalid, INS_sqdmull2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningUpperAndAddSaturate, 16, 3, INS_invalid, INS_invalid, INS_sqdmlal2, INS_invalid, INS_sqdmlal2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningUpperAndSubtractSaturate, 16, 3, INS_invalid, INS_invalid, INS_sqdmlsl2, INS_invalid, INS_sqdmlsl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningUpperByScalarAndAddSaturate, 16, 3, INS_invalid, INS_invalid, INS_sqdmlal2, INS_invalid, INS_sqdmlal2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningUpperByScalarAndSubtractSaturate, 16, 3, INS_invalid, INS_invalid, INS_sqdmlsl2, INS_invalid, INS_sqdmlsl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningUpperBySelectedScalarAndAddSaturate, 16, 4, INS_invalid, INS_invalid, INS_sqdmlal2, INS_invalid, INS_sqdmlal2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningUpperBySelectedScalarAndSubtractSaturate, 16, 4, INS_invalid, INS_invalid, INS_sqdmlsl2, INS_invalid, INS_sqdmlsl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyRoundedDoublingByScalarSaturateHigh, -1, 2, INS_invalid, INS_invalid, INS_sqrdmulh, INS_invalid, INS_sqrdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, MultiplyRoundedDoublingBySelectedScalarSaturateHigh, -1, 3, INS_invalid, INS_invalid, INS_sqrdmulh, INS_invalid, INS_sqrdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, MultiplyRoundedDoublingSaturateHigh, -1, 2, INS_invalid, INS_invalid, INS_sqrdmulh, INS_invalid, INS_sqrdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, MultiplyScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmul, INS_fmul, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, MultiplyScalarBySelectedScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmul, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, MultiplySubtract, -1, 3, INS_mls, INS_mls, INS_mls, INS_mls, INS_mls, INS_mls, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplySubtractByScalar, -1, 3, INS_invalid, INS_invalid, INS_mls, INS_mls, INS_mls, INS_mls, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplySubtractBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_mls, INS_mls, INS_mls, INS_mls, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyWideningLower, 8, 2, INS_smull, INS_umull, INS_smull, INS_umull, INS_smull, INS_umull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, MultiplyWideningLowerAndAdd, 8, 3, INS_smlal, INS_umlal, INS_smlal, INS_umlal, INS_smlal, INS_umlal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyWideningLowerAndSubtract, 8, 3, INS_smlsl, INS_umlsl, INS_smlsl, INS_umlsl, INS_smlsl, INS_umlsl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyWideningUpper, 16, 2, INS_smull2, INS_umull2, INS_smull2, INS_umull2, INS_smull2, INS_umull2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, MultiplyWideningUpperAndAdd, 16, 3, INS_smlal2, INS_umlal2, INS_smlal2, INS_umlal2, INS_smlal2, INS_umlal2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyWideningUpperAndSubtract, 16, 3, INS_smlsl2, INS_umlsl2, INS_smlsl2, INS_umlsl2, INS_smlsl2, INS_umlsl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, Negate, -1, 1, INS_neg, INS_invalid, INS_neg, INS_invalid, INS_neg, INS_invalid, INS_invalid, INS_invalid, INS_fneg, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, NegateSaturate, -1, 1, INS_sqneg, INS_invalid, INS_sqneg, INS_invalid, INS_sqneg, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, NegateScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fneg, INS_fneg, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, Not, -1, 1, INS_mvn, INS_mvn, INS_mvn, INS_mvn, INS_mvn, INS_mvn, INS_mvn, INS_mvn, INS_mvn, INS_mvn, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, Or, -1, 2, INS_orr, INS_orr, INS_orr, INS_orr, INS_orr, INS_orr, INS_orr, INS_orr, INS_orr, INS_orr, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, OrNot, -1, 2, INS_orn, INS_orn, INS_orn, INS_orn, INS_orn, INS_orn, INS_orn, INS_orn, INS_orn, INS_orn, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AdvSimd, PolynomialMultiply, -1, 2, INS_pmul, INS_pmul, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, PolynomialMultiplyWideningLower, 8, 2, INS_pmull, INS_pmull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, PolynomialMultiplyWideningUpper, 16, 2, INS_pmull2, INS_pmull2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, PopCount, -1, 1, INS_cnt, INS_cnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, ReciprocalEstimate, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_urecpe, INS_invalid, INS_invalid, INS_frecpe, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, ReciprocalSquareRootEstimate, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ursqrte, INS_invalid, INS_invalid, INS_frsqrte, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, ReciprocalSquareRootStep, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frsqrts, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, ReciprocalStep, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frecps, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, ReverseElement16, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rev32, INS_rev32, INS_rev64, INS_rev64, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, ReverseElement32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rev64, INS_rev64, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, ReverseElement8, -1, 1, INS_invalid, INS_invalid, INS_rev16, INS_rev16, INS_rev32, INS_rev32, INS_rev64, INS_rev64, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, RoundAwayFromZero, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frinta, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, RoundAwayFromZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frinta, INS_frinta, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, RoundToNearest, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintn, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, RoundToNearestScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintn, INS_frintn, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, RoundToNegativeInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintm, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, RoundToNegativeInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintm, INS_frintm, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, RoundToPositiveInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintp, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, RoundToPositiveInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintp, INS_frintp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, RoundToZero, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintz, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, RoundToZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintz, INS_frintz, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftArithmetic, -1, 2, INS_sshl, INS_invalid, INS_sshl, INS_invalid, INS_sshl, INS_invalid, INS_sshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, ShiftArithmeticRounded, -1, 2, INS_srshl, INS_invalid, INS_srshl, INS_invalid, INS_srshl, INS_invalid, INS_srshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, ShiftArithmeticRoundedSaturate, -1, 2, INS_sqrshl, INS_invalid, INS_sqrshl, INS_invalid, INS_sqrshl, INS_invalid, INS_sqrshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, ShiftArithmeticRoundedSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqrshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftArithmeticRoundedScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_srshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftArithmeticSaturate, -1, 2, INS_sqshl, INS_invalid, INS_sqshl, INS_invalid, INS_sqshl, INS_invalid, INS_sqshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, ShiftArithmeticSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftArithmeticScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftLeftAndInsert, -1, 3, INS_sli, INS_sli, INS_sli, INS_sli, INS_sli, INS_sli, INS_sli, INS_sli, INS_sli, INS_sli, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ShiftLeftAndInsertScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sli, INS_sli, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogical, -1, 2, INS_shl, INS_shl, INS_shl, INS_shl, INS_shl, INS_shl, INS_shl, INS_shl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_NoJmpTableIMM) +HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalSaturate, -1, 2, INS_sqshl, INS_uqshl, INS_sqshl, INS_uqshl, INS_sqshl, INS_uqshl, INS_sqshl, INS_uqshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqshl, INS_uqshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalSaturateUnsigned, -1, 2, INS_sqshlu, INS_invalid, INS_sqshlu, INS_invalid, INS_sqshlu, INS_invalid, INS_sqshlu, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalSaturateUnsignedScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqshlu, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_shl, INS_shl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar|HW_Flag_NoJmpTableIMM) +HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalWideningLower, 8, 2, INS_sshll, INS_ushll, INS_sshll, INS_ushll, INS_sshll, INS_ushll, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalWideningUpper, 16, 2, INS_sshll2, INS_ushll2, INS_sshll2, INS_ushll2, INS_sshll2, INS_ushll2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, ShiftLogical, -1, 2, INS_ushl, INS_ushl, INS_ushl, INS_ushl, INS_ushl, INS_ushl, INS_ushl, INS_ushl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, ShiftLogicalRounded, -1, 2, INS_urshl, INS_urshl, INS_urshl, INS_urshl, INS_urshl, INS_urshl, INS_urshl, INS_urshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, ShiftLogicalRoundedSaturate, -1, 2, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, ShiftLogicalRoundedSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_uqrshl, INS_uqrshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftLogicalRoundedScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_urshl, INS_urshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftLogicalSaturate, -1, 2, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, ShiftLogicalSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_uqshl, INS_uqshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftLogicalScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ushl, INS_ushl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightAndInsert, -1, 3, INS_sri, INS_sri, INS_sri, INS_sri, INS_sri, INS_sri, INS_sri, INS_sri, INS_sri, INS_sri, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightAndInsertScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sri, INS_sri, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmetic, -1, 2, INS_sshr, INS_invalid, INS_sshr, INS_invalid, INS_sshr, INS_invalid, INS_sshr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_NoJmpTableIMM) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticAdd, -1, 3, INS_ssra, INS_invalid, INS_ssra, INS_invalid, INS_ssra, INS_invalid, INS_ssra, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticAddScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ssra, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticNarrowingSaturateLower, 8, 2, INS_sqshrn, INS_invalid, INS_sqshrn, INS_invalid, INS_sqshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticNarrowingSaturateUnsignedLower, 8, 2, INS_invalid, INS_sqshrun, INS_invalid, INS_sqshrun, INS_invalid, INS_sqshrun, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticNarrowingSaturateUnsignedUpper, 16, 3, INS_invalid, INS_sqshrun2, INS_invalid, INS_sqshrun2, INS_invalid, INS_sqshrun2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticNarrowingSaturateUpper, 16, 3, INS_sqshrn2, INS_invalid, INS_sqshrn2, INS_invalid, INS_sqshrn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRounded, -1, 2, INS_srshr, INS_invalid, INS_srshr, INS_invalid, INS_srshr, INS_invalid, INS_srshr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedAdd, -1, 3, INS_srsra, INS_invalid, INS_srsra, INS_invalid, INS_srsra, INS_invalid, INS_srsra, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedAddScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_srsra, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedNarrowingSaturateLower, 8, 2, INS_sqrshrn, INS_invalid, INS_sqrshrn, INS_invalid, INS_sqrshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedNarrowingSaturateUnsignedLower, 8, 2, INS_invalid, INS_sqrshrun, INS_invalid, INS_sqrshrun, INS_invalid, INS_sqrshrun, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedNarrowingSaturateUnsignedUpper, 16, 3, INS_invalid, INS_sqrshrun2, INS_invalid, INS_sqrshrun2, INS_invalid, INS_sqrshrun2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedNarrowingSaturateUpper, 16, 3, INS_sqrshrn2, INS_invalid, INS_sqrshrn2, INS_invalid, INS_sqrshrn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_srshr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sshr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar|HW_Flag_NoJmpTableIMM) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogical, -1, 2, INS_ushr, INS_ushr, INS_ushr, INS_ushr, INS_ushr, INS_ushr, INS_ushr, INS_ushr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_NoJmpTableIMM) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalAdd, -1, 3, INS_usra, INS_usra, INS_usra, INS_usra, INS_usra, INS_usra, INS_usra, INS_usra, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalAddScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_usra, INS_usra, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalNarrowingLower, 8, 2, INS_shrn, INS_shrn, INS_shrn, INS_shrn, INS_shrn, INS_shrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalNarrowingSaturateLower, 8, 2, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalNarrowingSaturateUpper, 16, 3, INS_uqshrn2, INS_uqshrn2, INS_uqshrn2, INS_uqshrn2, INS_uqshrn2, INS_uqshrn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalNarrowingUpper, 16, 3, INS_shrn2, INS_shrn2, INS_shrn2, INS_shrn2, INS_shrn2, INS_shrn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRounded, -1, 2, INS_urshr, INS_urshr, INS_urshr, INS_urshr, INS_urshr, INS_urshr, INS_urshr, INS_urshr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedAdd, -1, 3, INS_ursra, INS_ursra, INS_ursra, INS_ursra, INS_ursra, INS_ursra, INS_ursra, INS_ursra, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedAddScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ursra, INS_ursra, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedNarrowingLower, 8, 2, INS_rshrn, INS_rshrn, INS_rshrn, INS_rshrn, INS_rshrn, INS_rshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedNarrowingSaturateLower, 8, 2, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedNarrowingSaturateUpper, 16, 3, INS_uqrshrn2, INS_uqrshrn2, INS_uqrshrn2, INS_uqrshrn2, INS_uqrshrn2, INS_uqrshrn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedNarrowingUpper, 16, 3, INS_rshrn2, INS_rshrn2, INS_rshrn2, INS_rshrn2, INS_rshrn2, INS_rshrn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_urshr, INS_urshr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ushr, INS_ushr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar|HW_Flag_NoJmpTableIMM) +HARDWARE_INTRINSIC(AdvSimd, SignExtendWideningLower, 8, 1, INS_sxtl, INS_invalid, INS_sxtl, INS_invalid, INS_sxtl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, SignExtendWideningUpper, 16, 1, INS_sxtl2, INS_invalid, INS_sxtl2, INS_invalid, INS_sxtl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, SqrtScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fsqrt, INS_fsqrt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, Store, -1, 2, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_invalid, INS_invalid, INS_st1_2regs, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_BaseTypeFromValueTupleArg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, StoreSelectedScalar, -1, 3, INS_st1, INS_st1, INS_st1, INS_st1, INS_st1, INS_st1, INS_st1, INS_st1, INS_st1, INS_st1, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_BaseTypeFromValueTupleArg|HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, StoreVectorAndZip, 8, 2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_BaseTypeFromValueTupleArg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, Subtract, -1, 2, INS_sub, INS_sub, INS_sub, INS_sub, INS_sub, INS_sub, INS_sub, INS_sub, INS_fsub, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, SubtractHighNarrowingLower, 8, 2, INS_subhn, INS_subhn, INS_subhn, INS_subhn, INS_subhn, INS_subhn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, SubtractHighNarrowingUpper, 16, 3, INS_subhn2, INS_subhn2, INS_subhn2, INS_subhn2, INS_subhn2, INS_subhn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, SubtractRoundedHighNarrowingLower, 8, 2, INS_rsubhn, INS_rsubhn, INS_rsubhn, INS_rsubhn, INS_rsubhn, INS_rsubhn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, SubtractRoundedHighNarrowingUpper, 16, 3, INS_rsubhn2, INS_rsubhn2, INS_rsubhn2, INS_rsubhn2, INS_rsubhn2, INS_rsubhn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, SubtractSaturate, -1, 2, INS_sqsub, INS_uqsub, INS_sqsub, INS_uqsub, INS_sqsub, INS_uqsub, INS_sqsub, INS_uqsub, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, SubtractSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqsub, INS_uqsub, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, SubtractScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sub, INS_sub, INS_fsub, INS_fsub, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, SubtractWideningLower, 8, 2, INS_ssubl, INS_usubl, INS_ssubl, INS_usubl, INS_ssubl, INS_usubl, INS_ssubw, INS_usubw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, SubtractWideningUpper, 16, 2, INS_ssubl2, INS_usubl2, INS_ssubl2, INS_usubl2, INS_ssubl2, INS_usubl2, INS_ssubw2, INS_usubw2, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, VectorTableLookup, 8, 2, INS_tbl, INS_tbl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, VectorTableLookupExtension, 8, 3, INS_tbx, INS_tbx, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, Xor, -1, 2, INS_eor, INS_eor, INS_eor, INS_eor, INS_eor, INS_eor, INS_eor, INS_eor, INS_eor, INS_eor, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, ZeroExtendWideningLower, 8, 1, INS_uxtl, INS_uxtl, INS_uxtl, INS_uxtl, INS_uxtl, INS_uxtl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, ZeroExtendWideningUpper, 16, 1, INS_uxtl2, INS_uxtl2, INS_uxtl2, INS_uxtl2, INS_uxtl2, INS_uxtl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) #define LAST_NI_AdvSimd NI_AdvSimd_ZeroExtendWideningUpper // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -294,189 +294,189 @@ HARDWARE_INTRINSIC(AdvSimd, ZeroExtendWideningUpper, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // AdvSimd 64-bit only Intrinsics #define FIRST_NI_AdvSimd_Arm64 NI_AdvSimd_Arm64_Abs -HARDWARE_INTRINSIC(AdvSimd_Arm64, Abs, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_abs, INS_invalid, INS_invalid, INS_fabs, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsSaturate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqabs, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsSaturateScalar, 8, 1, INS_sqabs, INS_invalid, INS_sqabs, INS_invalid, INS_sqabs, INS_invalid, INS_sqabs, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_abs, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facgt, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facge, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareGreaterThanOrEqualScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facge, INS_facge, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareGreaterThanScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facgt, INS_facgt, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facgt, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facge, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareLessThanOrEqualScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facge, INS_facge, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareLessThanScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facgt, INS_facgt, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteDifference, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fabd, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteDifferenceScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fabd, INS_fabd, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Add, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fadd, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AddAcross, -1, 1, INS_addv, INS_addv, INS_addv, INS_addv, INS_addv, INS_addv, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AddAcrossWidening, -1, 1, INS_saddlv, INS_uaddlv, INS_saddlv, INS_uaddlv, INS_saddlv, INS_uaddlv, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AddPairwise, 16, 2, INS_addp, INS_addp, INS_addp, INS_addp, INS_addp, INS_addp, INS_addp, INS_addp, INS_faddp, INS_faddp, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AddPairwiseScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_addp, INS_addp, INS_faddp, INS_faddp, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AddSaturate, -1, 2, INS_suqadd, INS_usqadd, INS_suqadd, INS_usqadd, INS_suqadd, INS_usqadd, INS_suqadd, INS_usqadd, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AddSaturateScalar, 8, 2, INS_sqadd, INS_uqadd, INS_sqadd, INS_uqadd, INS_sqadd, INS_uqadd, INS_suqadd, INS_usqadd, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Ceiling, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintp, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmeq, INS_cmeq, INS_invalid, INS_fcmeq, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareEqualScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmeq, INS_cmeq, INS_fcmeq, INS_fcmeq, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmgt, INS_cmhi, INS_invalid, INS_fcmgt, -1, -1, HW_Category_SIMD, HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmge, INS_cmhs, INS_invalid, INS_fcmge, -1, -1, HW_Category_SIMD, HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareGreaterThanOrEqualScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmge, INS_cmhs, INS_fcmge, INS_fcmge, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareGreaterThanScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmgt, INS_cmhi, INS_fcmgt, INS_fcmgt, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmgt, INS_cmhi, INS_invalid, INS_fcmgt, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmge, INS_cmhs, INS_invalid, INS_fcmge, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareLessThanOrEqualScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmge, INS_cmhs, INS_fcmge, INS_fcmge, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareLessThanScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmgt, INS_cmhi, INS_fcmgt, INS_fcmgt, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareTest, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmtst, INS_cmtst, INS_invalid, INS_cmtst, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareTestScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmtst, INS_cmtst, INS_invalid, INS_cmtst, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToDouble, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_scvtf, INS_ucvtf, INS_fcvtl, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToDoubleScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_scvtf, INS_ucvtf, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToDoubleUpper, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtl2, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundAwayFromZero, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtas, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundAwayFromZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtas, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToEven, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtns, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToEvenScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtns, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToNegativeInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtms, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToNegativeInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtms, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToPositiveInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtps, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToPositiveInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtps, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToZero, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzs, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzs, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToSingleLower, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtn, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToSingleRoundToOddLower, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtxn, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToSingleRoundToOddUpper, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtxn2, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToSingleUpper, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtn2, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundAwayFromZero, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtau, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundAwayFromZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtau, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToEven, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtnu, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToEvenScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtnu, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToNegativeInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtmu, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToNegativeInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtmu, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToPositiveInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtpu, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToPositiveInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtpu, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToZero, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzu, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzu, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Divide, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fdiv, INS_fdiv, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, DuplicateSelectedScalarToVector128, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_dup, INS_dup, INS_invalid, INS_dup, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd_Arm64, DuplicateToVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_dup, INS_dup, INS_invalid, INS_dup, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment) -HARDWARE_INTRINSIC(AdvSimd_Arm64, DuplicateToVector64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mov, INS_mov, INS_invalid, INS_fmov, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ExtractNarrowingSaturateScalar, 8, 1, INS_sqxtn, INS_uqxtn, INS_sqxtn, INS_uqxtn, INS_sqxtn, INS_uqxtn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ExtractNarrowingSaturateUnsignedScalar, 8, 1, INS_invalid, INS_sqxtun, INS_invalid, INS_sqxtun, INS_invalid, INS_sqxtun, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Floor, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintm, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplyAdd, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmla, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplyAddByScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmla, INS_fmla, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplyAddBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmla, INS_fmla, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplyAddScalarBySelectedScalar, 8, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmla, INS_fmla, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplySubtract, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmls, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplySubtractByScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmls, INS_fmls, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplySubtractBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmls, INS_fmls, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplySubtractScalarBySelectedScalar, 8, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmls, INS_fmls, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, InsertSelectedScalar, -1, 4, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, -1, -1, HW_Category_SIMD, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_NoJmpTableIMM|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Load2xVector128, 16, 1, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Load2xVector128AndUnzip, 16, 1, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Load3xVector128, 16, 1, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Load3xVector128AndUnzip, 16, 1, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Load4xVector128, 16, 1, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Load4xVector128AndUnzip, 16, 1, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndInsertScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoCodeGen) -HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndInsertScalarVector128x2, 16, 3, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, -1, -1, HW_Category_MemoryLoad, HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndInsertScalarVector128x3, 16, 3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, -1, -1, HW_Category_MemoryLoad, HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndInsertScalarVector128x4, 16, 3, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, -1, -1, HW_Category_MemoryLoad, HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndReplicateToVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ld1r, INS_ld1r, INS_invalid, INS_ld1r, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndReplicateToVector128x2, 16, 1, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndReplicateToVector128x3, 16, 1, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndReplicateToVector128x4, 16, 1, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadPairScalarVector64, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ldp, INS_ldp, INS_invalid, INS_invalid, INS_ldp, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_MultiReg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadPairScalarVector64NonTemporal, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ldnp, INS_ldnp, INS_invalid, INS_invalid, INS_ldnp, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_MultiReg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadPairVector128, 16, 1, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_MultiReg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadPairVector128NonTemporal, 16, 1, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_MultiReg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadPairVector64, 8, 1, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_MultiReg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadPairVector64NonTemporal, 8, 1, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_MultiReg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Max, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmax, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxAcross, -1, 1, INS_smaxv, INS_umaxv, INS_smaxv, INS_umaxv, INS_smaxv, INS_umaxv, INS_invalid, INS_invalid, INS_fmaxv, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxNumber, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmaxnm, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxNumberAcross, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmaxnmv, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxNumberPairwise, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmaxnmp, INS_fmaxnmp, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxNumberPairwiseScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmaxnmp, INS_fmaxnmp, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxPairwise, 16, 2, INS_smaxp, INS_umaxp, INS_smaxp, INS_umaxp, INS_smaxp, INS_umaxp, INS_invalid, INS_invalid, INS_fmaxp, INS_fmaxp, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxPairwiseScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmaxp, INS_fmaxp, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmax, INS_fmax, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Min, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmin, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MinAcross, -1, 1, INS_sminv, INS_uminv, INS_sminv, INS_uminv, INS_sminv, INS_uminv, INS_invalid, INS_invalid, INS_fminv, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MinNumber, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fminnm, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MinNumberAcross, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fminnmv, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MinNumberPairwise, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fminnmp, INS_fminnmp, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MinNumberPairwiseScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fminnmp, INS_fminnmp, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MinPairwise, 16, 2, INS_sminp, INS_uminp, INS_sminp, INS_uminp, INS_sminp, INS_uminp, INS_invalid, INS_invalid, INS_fminp, INS_fminp, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MinPairwiseScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fminp, INS_fminp, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MinScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmin, INS_fmin, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Multiply, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmul, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyByScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmul, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyBySelectedScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmul, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingSaturateHighScalar, 8, 2, INS_invalid, INS_invalid, INS_sqdmulh, INS_invalid, INS_sqdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingScalarBySelectedScalarSaturateHigh, 8, 3, INS_invalid, INS_invalid, INS_sqdmulh, INS_invalid, INS_sqdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingWideningAndAddSaturateScalar, 8, 3, INS_invalid, INS_invalid, INS_sqdmlal, INS_invalid, INS_sqdmlal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingWideningAndSubtractSaturateScalar, 8, 3, INS_invalid, INS_invalid, INS_sqdmlsl, INS_invalid, INS_sqdmlsl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingWideningSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_sqdmull, INS_invalid, INS_sqdmull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingWideningSaturateScalarBySelectedScalar, 8, 3, INS_invalid, INS_invalid, INS_sqdmull, INS_invalid, INS_sqdmull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingWideningScalarBySelectedScalarAndAddSaturate, 8, 4, INS_invalid, INS_invalid, INS_sqdmlal, INS_invalid, INS_sqdmlal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingWideningScalarBySelectedScalarAndSubtractSaturate, 8, 4, INS_invalid, INS_invalid, INS_sqdmlsl, INS_invalid, INS_sqdmlsl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyExtended, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmulx, INS_fmulx, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyExtendedByScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmulx, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyExtendedBySelectedScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmulx, INS_fmulx, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyExtendedScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmulx, INS_fmulx, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyExtendedScalarBySelectedScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmulx, INS_fmulx, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyRoundedDoublingSaturateHighScalar, 8, 2, INS_invalid, INS_invalid, INS_sqrdmulh, INS_invalid, INS_sqrdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyRoundedDoublingScalarBySelectedScalarSaturateHigh, 8, 3, INS_invalid, INS_invalid, INS_sqrdmulh, INS_invalid, INS_sqrdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyScalarBySelectedScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmul, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Negate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_neg, INS_invalid, INS_invalid, INS_fneg, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, NegateSaturate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqneg, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, NegateSaturateScalar, 8, 1, INS_sqneg, INS_invalid, INS_sqneg, INS_invalid, INS_sqneg, INS_invalid, INS_sqneg, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, NegateScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_neg, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalEstimate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frecpe, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalEstimateScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frecpe, INS_frecpe, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalExponentScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frecpx, INS_frecpx, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalSquareRootEstimate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frsqrte, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalSquareRootEstimateScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frsqrte, INS_frsqrte, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalSquareRootStep, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frsqrts, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalSquareRootStepScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frsqrts, INS_frsqrts, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalStep, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frecps, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalStepScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frecps, INS_frecps, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ReverseElementBits, -1, 1, INS_rbit, INS_rbit, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, RoundAwayFromZero, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frinta, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, RoundToNearest, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintn, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, RoundToNegativeInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintm, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, RoundToPositiveInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintp, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, RoundToZero, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintz, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftArithmeticRoundedSaturateScalar, 8, 2, INS_sqrshl, INS_invalid, INS_sqrshl, INS_invalid, INS_sqrshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftArithmeticSaturateScalar, 8, 2, INS_sqshl, INS_invalid, INS_sqshl, INS_invalid, INS_sqshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftLeftLogicalSaturateScalar, 8, 2, INS_sqshl, INS_uqshl, INS_sqshl, INS_uqshl, INS_sqshl, INS_uqshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftLeftLogicalSaturateUnsignedScalar, 8, 2, INS_sqshlu, INS_invalid, INS_sqshlu, INS_invalid, INS_sqshlu, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftLogicalRoundedSaturateScalar, 8, 2, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftLogicalSaturateScalar, 8, 2, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftRightArithmeticNarrowingSaturateScalar, 8, 2, INS_sqshrn, INS_invalid, INS_sqshrn, INS_invalid, INS_sqshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftRightArithmeticNarrowingSaturateUnsignedScalar, 8, 2, INS_invalid, INS_sqshrun, INS_invalid, INS_sqshrun, INS_invalid, INS_sqshrun, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftRightArithmeticRoundedNarrowingSaturateScalar, 8, 2, INS_sqrshrn, INS_invalid, INS_sqrshrn, INS_invalid, INS_sqrshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftRightArithmeticRoundedNarrowingSaturateUnsignedScalar, 8, 2, INS_invalid, INS_sqrshrun, INS_invalid, INS_sqrshrun, INS_invalid, INS_sqrshrun, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftRightLogicalNarrowingSaturateScalar, 8, 2, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftRightLogicalRoundedNarrowingSaturateScalar, 8, 2, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Sqrt, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fsqrt, INS_fsqrt, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Store, 16, 2, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_BaseTypeFromValueTupleArg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, StorePair, -1, 3, INS_stp, INS_stp, INS_stp, INS_stp, INS_stp, INS_stp, INS_stp, INS_stp, INS_stp, INS_stp, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd_Arm64, StorePairNonTemporal, -1, 3, INS_stnp, INS_stnp, INS_stnp, INS_stnp, INS_stnp, INS_stnp, INS_stnp, INS_stnp, INS_stnp, INS_stp, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd_Arm64, StorePairScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_stp, INS_stp, INS_invalid, INS_invalid, INS_stp, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd_Arm64, StorePairScalarNonTemporal, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_stnp, INS_stnp, INS_invalid, INS_invalid, INS_stnp, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd_Arm64, StoreSelectedScalar, 16, 3, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_BaseTypeFromValueTupleArg|HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, StoreVectorAndZip, 16, 2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_BaseTypeFromValueTupleArg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Subtract, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fsub, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, SubtractSaturateScalar, 8, 2, INS_sqsub, INS_uqsub, INS_sqsub, INS_uqsub, INS_sqsub, INS_uqsub, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, TransposeEven, -1, 2, INS_trn1, INS_trn1, INS_trn1, INS_trn1, INS_trn1, INS_trn1, INS_trn1, INS_trn1, INS_trn1, INS_trn1, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, TransposeOdd, -1, 2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, UnzipEven, -1, 2, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, UnzipOdd, -1, 2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, VectorTableLookup, 16, 2, INS_tbl, INS_tbl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, VectorTableLookupExtension, 16, 3, INS_tbx, INS_tbx, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ZipHigh, -1, 2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ZipLow, -1, 2, INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_zip1, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Abs, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_abs, INS_invalid, INS_invalid, INS_fabs, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsSaturate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqabs, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsSaturateScalar, 8, 1, INS_sqabs, INS_invalid, INS_sqabs, INS_invalid, INS_sqabs, INS_invalid, INS_sqabs, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_abs, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facgt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facge, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareGreaterThanOrEqualScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facge, INS_facge, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareGreaterThanScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facgt, INS_facgt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facgt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facge, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareLessThanOrEqualScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facge, INS_facge, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareLessThanScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facgt, INS_facgt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteDifference, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fabd, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteDifferenceScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fabd, INS_fabd, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Add, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fadd, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AddAcross, -1, 1, INS_addv, INS_addv, INS_addv, INS_addv, INS_addv, INS_addv, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AddAcrossWidening, -1, 1, INS_saddlv, INS_uaddlv, INS_saddlv, INS_uaddlv, INS_saddlv, INS_uaddlv, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AddPairwise, 16, 2, INS_addp, INS_addp, INS_addp, INS_addp, INS_addp, INS_addp, INS_addp, INS_addp, INS_faddp, INS_faddp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AddPairwiseScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_addp, INS_addp, INS_faddp, INS_faddp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AddSaturate, -1, 2, INS_suqadd, INS_usqadd, INS_suqadd, INS_usqadd, INS_suqadd, INS_usqadd, INS_suqadd, INS_usqadd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AddSaturateScalar, 8, 2, INS_sqadd, INS_uqadd, INS_sqadd, INS_uqadd, INS_sqadd, INS_uqadd, INS_suqadd, INS_usqadd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Ceiling, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmeq, INS_cmeq, INS_invalid, INS_fcmeq, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareEqualScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmeq, INS_cmeq, INS_fcmeq, INS_fcmeq, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmgt, INS_cmhi, INS_invalid, INS_fcmgt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmge, INS_cmhs, INS_invalid, INS_fcmge, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareGreaterThanOrEqualScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmge, INS_cmhs, INS_fcmge, INS_fcmge, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareGreaterThanScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmgt, INS_cmhi, INS_fcmgt, INS_fcmgt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmgt, INS_cmhi, INS_invalid, INS_fcmgt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmge, INS_cmhs, INS_invalid, INS_fcmge, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareLessThanOrEqualScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmge, INS_cmhs, INS_fcmge, INS_fcmge, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareLessThanScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmgt, INS_cmhi, INS_fcmgt, INS_fcmgt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareTest, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmtst, INS_cmtst, INS_invalid, INS_cmtst, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareTestScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmtst, INS_cmtst, INS_invalid, INS_cmtst, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToDouble, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_scvtf, INS_ucvtf, INS_fcvtl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToDoubleScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_scvtf, INS_ucvtf, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToDoubleUpper, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtl2, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundAwayFromZero, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtas, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundAwayFromZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtas, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToEven, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtns, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToEvenScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtns, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToNegativeInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtms, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToNegativeInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtms, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToPositiveInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtps, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToPositiveInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtps, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToZero, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzs, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzs, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToSingleLower, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtn, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToSingleRoundToOddLower, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtxn, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToSingleRoundToOddUpper, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtxn2, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToSingleUpper, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtn2, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundAwayFromZero, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtau, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundAwayFromZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtau, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToEven, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtnu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToEvenScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtnu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToNegativeInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtmu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToNegativeInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtmu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToPositiveInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtpu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToPositiveInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtpu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToZero, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Divide, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fdiv, INS_fdiv, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, DuplicateSelectedScalarToVector128, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_dup, INS_dup, INS_invalid, INS_dup, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd_Arm64, DuplicateToVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_dup, INS_dup, INS_invalid, INS_dup, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment) +HARDWARE_INTRINSIC(AdvSimd_Arm64, DuplicateToVector64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mov, INS_mov, INS_invalid, INS_fmov, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ExtractNarrowingSaturateScalar, 8, 1, INS_sqxtn, INS_uqxtn, INS_sqxtn, INS_uqxtn, INS_sqxtn, INS_uqxtn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ExtractNarrowingSaturateUnsignedScalar, 8, 1, INS_invalid, INS_sqxtun, INS_invalid, INS_sqxtun, INS_invalid, INS_sqxtun, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Floor, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintm, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplyAdd, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmla, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplyAddByScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmla, INS_fmla, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplyAddBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmla, INS_fmla, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplyAddScalarBySelectedScalar, 8, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmla, INS_fmla, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplySubtract, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmls, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplySubtractByScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmls, INS_fmls, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplySubtractBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmls, INS_fmls, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplySubtractScalarBySelectedScalar, 8, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmls, INS_fmls, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, InsertSelectedScalar, -1, 4, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_NoJmpTableIMM|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Load2xVector128, 16, 1, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Load2xVector128AndUnzip, 16, 1, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Load3xVector128, 16, 1, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Load3xVector128AndUnzip, 16, 1, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Load4xVector128, 16, 1, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Load4xVector128AndUnzip, 16, 1, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndInsertScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoCodeGen) +HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndInsertScalarVector128x2, 16, 3, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndInsertScalarVector128x3, 16, 3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndInsertScalarVector128x4, 16, 3, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndReplicateToVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ld1r, INS_ld1r, INS_invalid, INS_ld1r, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndReplicateToVector128x2, 16, 1, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndReplicateToVector128x3, 16, 1, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndReplicateToVector128x4, 16, 1, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadPairScalarVector64, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ldp, INS_ldp, INS_invalid, INS_invalid, INS_ldp, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_MultiReg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadPairScalarVector64NonTemporal, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ldnp, INS_ldnp, INS_invalid, INS_invalid, INS_ldnp, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_MultiReg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadPairVector128, 16, 1, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_MultiReg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadPairVector128NonTemporal, 16, 1, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_MultiReg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadPairVector64, 8, 1, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_MultiReg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadPairVector64NonTemporal, 8, 1, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_MultiReg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Max, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmax, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxAcross, -1, 1, INS_smaxv, INS_umaxv, INS_smaxv, INS_umaxv, INS_smaxv, INS_umaxv, INS_invalid, INS_invalid, INS_fmaxv, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxNumber, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmaxnm, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxNumberAcross, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmaxnmv, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxNumberPairwise, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmaxnmp, INS_fmaxnmp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxNumberPairwiseScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmaxnmp, INS_fmaxnmp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxPairwise, 16, 2, INS_smaxp, INS_umaxp, INS_smaxp, INS_umaxp, INS_smaxp, INS_umaxp, INS_invalid, INS_invalid, INS_fmaxp, INS_fmaxp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxPairwiseScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmaxp, INS_fmaxp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmax, INS_fmax, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Min, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmin, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MinAcross, -1, 1, INS_sminv, INS_uminv, INS_sminv, INS_uminv, INS_sminv, INS_uminv, INS_invalid, INS_invalid, INS_fminv, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MinNumber, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fminnm, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MinNumberAcross, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fminnmv, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MinNumberPairwise, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fminnmp, INS_fminnmp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MinNumberPairwiseScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fminnmp, INS_fminnmp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MinPairwise, 16, 2, INS_sminp, INS_uminp, INS_sminp, INS_uminp, INS_sminp, INS_uminp, INS_invalid, INS_invalid, INS_fminp, INS_fminp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MinPairwiseScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fminp, INS_fminp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MinScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmin, INS_fmin, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Multiply, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmul, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyByScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmul, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyBySelectedScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmul, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingSaturateHighScalar, 8, 2, INS_invalid, INS_invalid, INS_sqdmulh, INS_invalid, INS_sqdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingScalarBySelectedScalarSaturateHigh, 8, 3, INS_invalid, INS_invalid, INS_sqdmulh, INS_invalid, INS_sqdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingWideningAndAddSaturateScalar, 8, 3, INS_invalid, INS_invalid, INS_sqdmlal, INS_invalid, INS_sqdmlal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingWideningAndSubtractSaturateScalar, 8, 3, INS_invalid, INS_invalid, INS_sqdmlsl, INS_invalid, INS_sqdmlsl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingWideningSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_sqdmull, INS_invalid, INS_sqdmull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingWideningSaturateScalarBySelectedScalar, 8, 3, INS_invalid, INS_invalid, INS_sqdmull, INS_invalid, INS_sqdmull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingWideningScalarBySelectedScalarAndAddSaturate, 8, 4, INS_invalid, INS_invalid, INS_sqdmlal, INS_invalid, INS_sqdmlal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingWideningScalarBySelectedScalarAndSubtractSaturate, 8, 4, INS_invalid, INS_invalid, INS_sqdmlsl, INS_invalid, INS_sqdmlsl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyExtended, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmulx, INS_fmulx, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyExtendedByScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmulx, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyExtendedBySelectedScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmulx, INS_fmulx, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyExtendedScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmulx, INS_fmulx, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyExtendedScalarBySelectedScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmulx, INS_fmulx, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyRoundedDoublingSaturateHighScalar, 8, 2, INS_invalid, INS_invalid, INS_sqrdmulh, INS_invalid, INS_sqrdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyRoundedDoublingScalarBySelectedScalarSaturateHigh, 8, 3, INS_invalid, INS_invalid, INS_sqrdmulh, INS_invalid, INS_sqrdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyScalarBySelectedScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmul, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Negate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_neg, INS_invalid, INS_invalid, INS_fneg, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, NegateSaturate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqneg, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, NegateSaturateScalar, 8, 1, INS_sqneg, INS_invalid, INS_sqneg, INS_invalid, INS_sqneg, INS_invalid, INS_sqneg, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, NegateScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_neg, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalEstimate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frecpe, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalEstimateScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frecpe, INS_frecpe, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalExponentScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frecpx, INS_frecpx, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalSquareRootEstimate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frsqrte, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalSquareRootEstimateScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frsqrte, INS_frsqrte, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalSquareRootStep, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frsqrts, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalSquareRootStepScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frsqrts, INS_frsqrts, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalStep, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frecps, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalStepScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frecps, INS_frecps, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ReverseElementBits, -1, 1, INS_rbit, INS_rbit, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, RoundAwayFromZero, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frinta, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, RoundToNearest, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintn, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, RoundToNegativeInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintm, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, RoundToPositiveInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, RoundToZero, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintz, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftArithmeticRoundedSaturateScalar, 8, 2, INS_sqrshl, INS_invalid, INS_sqrshl, INS_invalid, INS_sqrshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftArithmeticSaturateScalar, 8, 2, INS_sqshl, INS_invalid, INS_sqshl, INS_invalid, INS_sqshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftLeftLogicalSaturateScalar, 8, 2, INS_sqshl, INS_uqshl, INS_sqshl, INS_uqshl, INS_sqshl, INS_uqshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftLeftLogicalSaturateUnsignedScalar, 8, 2, INS_sqshlu, INS_invalid, INS_sqshlu, INS_invalid, INS_sqshlu, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftLogicalRoundedSaturateScalar, 8, 2, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftLogicalSaturateScalar, 8, 2, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftRightArithmeticNarrowingSaturateScalar, 8, 2, INS_sqshrn, INS_invalid, INS_sqshrn, INS_invalid, INS_sqshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftRightArithmeticNarrowingSaturateUnsignedScalar, 8, 2, INS_invalid, INS_sqshrun, INS_invalid, INS_sqshrun, INS_invalid, INS_sqshrun, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftRightArithmeticRoundedNarrowingSaturateScalar, 8, 2, INS_sqrshrn, INS_invalid, INS_sqrshrn, INS_invalid, INS_sqrshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftRightArithmeticRoundedNarrowingSaturateUnsignedScalar, 8, 2, INS_invalid, INS_sqrshrun, INS_invalid, INS_sqrshrun, INS_invalid, INS_sqrshrun, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftRightLogicalNarrowingSaturateScalar, 8, 2, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftRightLogicalRoundedNarrowingSaturateScalar, 8, 2, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Sqrt, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fsqrt, INS_fsqrt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Store, 16, 2, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_BaseTypeFromValueTupleArg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, StorePair, -1, 3, INS_stp, INS_stp, INS_stp, INS_stp, INS_stp, INS_stp, INS_stp, INS_stp, INS_stp, INS_stp, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd_Arm64, StorePairNonTemporal, -1, 3, INS_stnp, INS_stnp, INS_stnp, INS_stnp, INS_stnp, INS_stnp, INS_stnp, INS_stnp, INS_stnp, INS_stp, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd_Arm64, StorePairScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_stp, INS_stp, INS_invalid, INS_invalid, INS_stp, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd_Arm64, StorePairScalarNonTemporal, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_stnp, INS_stnp, INS_invalid, INS_invalid, INS_stnp, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd_Arm64, StoreSelectedScalar, 16, 3, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_BaseTypeFromValueTupleArg|HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, StoreVectorAndZip, 16, 2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_BaseTypeFromValueTupleArg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Subtract, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fsub, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, SubtractSaturateScalar, 8, 2, INS_sqsub, INS_uqsub, INS_sqsub, INS_uqsub, INS_sqsub, INS_uqsub, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, TransposeEven, -1, 2, INS_trn1, INS_trn1, INS_trn1, INS_trn1, INS_trn1, INS_trn1, INS_trn1, INS_trn1, INS_trn1, INS_trn1, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, TransposeOdd, -1, 2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, UnzipEven, -1, 2, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, UnzipOdd, -1, 2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, VectorTableLookup, 16, 2, INS_tbl, INS_tbl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, VectorTableLookupExtension, 16, 3, INS_tbx, INS_tbx, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ZipHigh, -1, 2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ZipLow, -1, 2, INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) #define LAST_NI_AdvSimd_Arm64 NI_AdvSimd_Arm64_ZipLow // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -485,12 +485,12 @@ HARDWARE_INTRINSIC(AdvSimd_Arm64, ZipLow, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // AES Intrinsics #define FIRST_NI_Aes NI_Aes_Decrypt -HARDWARE_INTRINSIC(Aes, Decrypt, 16, 2, INS_invalid, INS_aesd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Aes, Encrypt, 16, 2, INS_invalid, INS_aese, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Aes, InverseMixColumns, 16, 1, INS_invalid, INS_aesimc, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(Aes, MixColumns, 16, 1, INS_invalid, INS_aesmc, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(Aes, PolynomialMultiplyWideningLower, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pmull, INS_pmull, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(Aes, PolynomialMultiplyWideningUpper, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pmull2, INS_pmull2, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(Aes, Decrypt, 16, 2, INS_invalid, INS_aesd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Aes, Encrypt, 16, 2, INS_invalid, INS_aese, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Aes, InverseMixColumns, 16, 1, INS_invalid, INS_aesimc, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(Aes, MixColumns, 16, 1, INS_invalid, INS_aesmc, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(Aes, PolynomialMultiplyWideningLower, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pmull, INS_pmull, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(Aes, PolynomialMultiplyWideningUpper, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pmull2, INS_pmull2, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) #define LAST_NI_Aes NI_Aes_PolynomialMultiplyWideningUpper // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -499,9 +499,9 @@ HARDWARE_INTRINSIC(Aes, PolynomialMultiplyWideningUpper, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Base Intrinsics #define FIRST_NI_ArmBase NI_ArmBase_LeadingZeroCount -HARDWARE_INTRINSIC(ArmBase, LeadingZeroCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_clz, INS_clz, INS_clz, INS_clz, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(ArmBase, ReverseElementBits, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rbit, INS_rbit, INS_rbit, INS_rbit, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(ArmBase, Yield, 0, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(ArmBase, LeadingZeroCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_clz, INS_clz, INS_clz, INS_clz, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(ArmBase, ReverseElementBits, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rbit, INS_rbit, INS_rbit, INS_rbit, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(ArmBase, Yield, 0, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_SpecialSideEffect_Other) #define LAST_NI_ArmBase NI_ArmBase_Yield // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -510,13 +510,13 @@ HARDWARE_INTRINSIC(ArmBase, Yield, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Base 64-bit only Intrinsics #define FIRST_NI_ArmBase_Arm64 NI_ArmBase_Arm64_LeadingSignCount -HARDWARE_INTRINSIC(ArmBase_Arm64, LeadingSignCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cls, INS_invalid, INS_cls, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(ArmBase_Arm64, LeadingZeroCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_clz, INS_clz, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(ArmBase_Arm64, MultiplyHigh, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_smulh, INS_umulh, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(ArmBase_Arm64, MultiplyLongAdd, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_smaddl, INS_umaddl, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(ArmBase_Arm64, MultiplyLongNeg, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_smnegl, INS_umnegl, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(ArmBase_Arm64, MultiplyLongSub, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_smsubl, INS_umsubl, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(ArmBase_Arm64, ReverseElementBits, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rbit, INS_rbit, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(ArmBase_Arm64, LeadingSignCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cls, INS_invalid, INS_cls, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(ArmBase_Arm64, LeadingZeroCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_clz, INS_clz, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(ArmBase_Arm64, MultiplyHigh, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_smulh, INS_umulh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(ArmBase_Arm64, MultiplyLongAdd, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_smaddl, INS_umaddl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(ArmBase_Arm64, MultiplyLongNeg, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_smnegl, INS_umnegl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(ArmBase_Arm64, MultiplyLongSub, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_smsubl, INS_umsubl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(ArmBase_Arm64, ReverseElementBits, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rbit, INS_rbit, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) #define LAST_NI_ArmBase_Arm64 NI_ArmBase_Arm64_ReverseElementBits // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -525,8 +525,8 @@ HARDWARE_INTRINSIC(ArmBase_Arm64, ReverseElementBits, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // CRC32 Intrinsics #define FIRST_NI_Crc32 NI_Crc32_ComputeCrc32 -HARDWARE_INTRINSIC(Crc32, ComputeCrc32, 0, 2, INS_invalid, INS_crc32b, INS_invalid, INS_crc32h, INS_invalid, INS_crc32w, INS_invalid, INS_crc32x, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Crc32, ComputeCrc32C, 0, 2, INS_invalid, INS_crc32cb, INS_invalid, INS_crc32ch, INS_invalid, INS_crc32cw, INS_invalid, INS_crc32cx, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Crc32, ComputeCrc32, 0, 2, INS_invalid, INS_crc32b, INS_invalid, INS_crc32h, INS_invalid, INS_crc32w, INS_invalid, INS_crc32x, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Crc32, ComputeCrc32C, 0, 2, INS_invalid, INS_crc32cb, INS_invalid, INS_crc32ch, INS_invalid, INS_crc32cw, INS_invalid, INS_crc32cx, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen) #define LAST_NI_Crc32 NI_Crc32_ComputeCrc32C // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -535,8 +535,8 @@ HARDWARE_INTRINSIC(Crc32, ComputeCrc32C, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // CRC32 64-bit only Intrinsics #define FIRST_NI_Crc32_Arm64 NI_Crc32_Arm64_ComputeCrc32 -HARDWARE_INTRINSIC(Crc32_Arm64, ComputeCrc32, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_crc32x, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Crc32_Arm64, ComputeCrc32C, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_crc32cx, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Crc32_Arm64, ComputeCrc32, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_crc32x, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Crc32_Arm64, ComputeCrc32C, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_crc32cx, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen) #define LAST_NI_Crc32_Arm64 NI_Crc32_Arm64_ComputeCrc32C // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -545,8 +545,8 @@ HARDWARE_INTRINSIC(Crc32_Arm64, ComputeCrc32C, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // DP Intrinsics #define FIRST_NI_Dp NI_Dp_DotProduct -HARDWARE_INTRINSIC(Dp, DotProduct, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sdot, INS_udot, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Dp, DotProductBySelectedQuadruplet, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sdot, INS_udot, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Dp, DotProduct, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sdot, INS_udot, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Dp, DotProductBySelectedQuadruplet, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sdot, INS_udot, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) #define LAST_NI_Dp NI_Dp_DotProductBySelectedQuadruplet // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -555,10 +555,10 @@ HARDWARE_INTRINSIC(Dp, DotProductBySelectedQuadruplet, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // RDM Intrinsics #define FIRST_NI_Rdm NI_Rdm_MultiplyRoundedDoublingAndAddSaturateHigh -HARDWARE_INTRINSIC(Rdm, MultiplyRoundedDoublingAndAddSaturateHigh, -1, 3, INS_invalid, INS_invalid, INS_sqrdmlah, INS_invalid, INS_sqrdmlah, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Rdm, MultiplyRoundedDoublingAndSubtractSaturateHigh, -1, 3, INS_invalid, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Rdm, MultiplyRoundedDoublingBySelectedScalarAndAddSaturateHigh, -1, 4, INS_invalid, INS_invalid, INS_sqrdmlah, INS_invalid, INS_sqrdmlah, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Rdm, MultiplyRoundedDoublingBySelectedScalarAndSubtractSaturateHigh, -1, 4, INS_invalid, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Rdm, MultiplyRoundedDoublingAndAddSaturateHigh, -1, 3, INS_invalid, INS_invalid, INS_sqrdmlah, INS_invalid, INS_sqrdmlah, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Rdm, MultiplyRoundedDoublingAndSubtractSaturateHigh, -1, 3, INS_invalid, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Rdm, MultiplyRoundedDoublingBySelectedScalarAndAddSaturateHigh, -1, 4, INS_invalid, INS_invalid, INS_sqrdmlah, INS_invalid, INS_sqrdmlah, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Rdm, MultiplyRoundedDoublingBySelectedScalarAndSubtractSaturateHigh, -1, 4, INS_invalid, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) #define LAST_NI_Rdm NI_Rdm_MultiplyRoundedDoublingBySelectedScalarAndSubtractSaturateHigh // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -567,10 +567,10 @@ HARDWARE_INTRINSIC(Rdm, MultiplyRoundedDoublingBySelectedScalarAndSu // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // RDM 64-bit only Intrinsics #define FIRST_NI_Rdm_Arm64 NI_Rdm_Arm64_MultiplyRoundedDoublingAndAddSaturateHighScalar -HARDWARE_INTRINSIC(Rdm_Arm64, MultiplyRoundedDoublingAndAddSaturateHighScalar, 8, 3, INS_invalid, INS_invalid, INS_sqrdmlah, INS_invalid, INS_sqrdmlah, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(Rdm_Arm64, MultiplyRoundedDoublingAndSubtractSaturateHighScalar, 8, 3, INS_invalid, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(Rdm_Arm64, MultiplyRoundedDoublingScalarBySelectedScalarAndAddSaturateHigh, 8, 4, INS_invalid, INS_invalid, INS_sqrdmlah, INS_invalid, INS_sqrdmlah, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(Rdm_Arm64, MultiplyRoundedDoublingScalarBySelectedScalarAndSubtractSaturateHigh, 8, 4, INS_invalid, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Rdm_Arm64, MultiplyRoundedDoublingAndAddSaturateHighScalar, 8, 3, INS_invalid, INS_invalid, INS_sqrdmlah, INS_invalid, INS_sqrdmlah, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Rdm_Arm64, MultiplyRoundedDoublingAndSubtractSaturateHighScalar, 8, 3, INS_invalid, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Rdm_Arm64, MultiplyRoundedDoublingScalarBySelectedScalarAndAddSaturateHigh, 8, 4, INS_invalid, INS_invalid, INS_sqrdmlah, INS_invalid, INS_sqrdmlah, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Rdm_Arm64, MultiplyRoundedDoublingScalarBySelectedScalarAndSubtractSaturateHigh, 8, 4, INS_invalid, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) #define LAST_NI_Rdm_Arm64 NI_Rdm_Arm64_MultiplyRoundedDoublingScalarBySelectedScalarAndSubtractSaturateHigh // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -579,12 +579,12 @@ HARDWARE_INTRINSIC(Rdm_Arm64, MultiplyRoundedDoublingScalarBySelectedScala // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // SHA1 Intrinsics #define FIRST_NI_Sha1 NI_Sha1_FixedRotate -HARDWARE_INTRINSIC(Sha1, FixedRotate, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(Sha1, HashUpdateChoose, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha1c, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sha1, HashUpdateMajority, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha1m, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sha1, HashUpdateParity, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha1p, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sha1, ScheduleUpdate0, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha1su0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sha1, ScheduleUpdate1, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha1su1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sha1, FixedRotate, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Sha1, HashUpdateChoose, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha1c, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sha1, HashUpdateMajority, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha1m, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sha1, HashUpdateParity, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha1p, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sha1, ScheduleUpdate0, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha1su0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sha1, ScheduleUpdate1, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha1su1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) #define LAST_NI_Sha1 NI_Sha1_ScheduleUpdate1 // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -593,10 +593,10 @@ HARDWARE_INTRINSIC(Sha1, ScheduleUpdate1, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // SHA256 Intrinsics #define FIRST_NI_Sha256 NI_Sha256_HashUpdate1 -HARDWARE_INTRINSIC(Sha256, HashUpdate1, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha256h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sha256, HashUpdate2, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha256h2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sha256, ScheduleUpdate0, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha256su0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sha256, ScheduleUpdate1, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha256su1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sha256, HashUpdate1, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha256h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sha256, HashUpdate2, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha256h2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sha256, ScheduleUpdate0, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha256su0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sha256, ScheduleUpdate1, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha256su1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) #define LAST_NI_Sha256 NI_Sha256_ScheduleUpdate1 // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -605,10 +605,10 @@ HARDWARE_INTRINSIC(Sha256, ScheduleUpdate1, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // SHA3 Intrinsics #define FIRST_NI_Sha3 NI_Sha3_BitwiseClearXor -HARDWARE_INTRINSIC(Sha3, BitwiseClearXor, 16, 3, INS_bcax, INS_bcax, INS_bcax, INS_bcax, INS_bcax, INS_bcax, INS_bcax, INS_bcax, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sha3, BitwiseRotateLeftBy1AndXor, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rax1, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(Sha3, Xor, 16, 3, INS_eor3, INS_eor3, INS_eor3, INS_eor3, INS_eor3, INS_eor3, INS_eor3, INS_eor3, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sha3, XorRotateRight, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_xar, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sha3, BitwiseClearXor, 16, 3, INS_bcax, INS_bcax, INS_bcax, INS_bcax, INS_bcax, INS_bcax, INS_bcax, INS_bcax, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sha3, BitwiseRotateLeftBy1AndXor, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rax1, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(Sha3, Xor, 16, 3, INS_eor3, INS_eor3, INS_eor3, INS_eor3, INS_eor3, INS_eor3, INS_eor3, INS_eor3, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sha3, XorRotateRight, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_xar, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasImmediateOperand) #define LAST_NI_Sha3 NI_Sha3_XorRotateRight diff --git a/src/coreclr/jit/hwintrinsiclistwasm.h b/src/coreclr/jit/hwintrinsiclistwasm.h index ae0eb686c75edd..53cdb3824a40d1 100644 --- a/src/coreclr/jit/hwintrinsiclistwasm.h +++ b/src/coreclr/jit/hwintrinsiclistwasm.h @@ -16,70 +16,70 @@ // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // PackedSimd Intrinsics #define FIRST_NI_PackedSimd NI_PackedSimd_Abs -HARDWARE_INTRINSIC(PackedSimd, Abs, 16, 1, INS_i8x16_abs, INS_invalid, INS_i16x8_abs, INS_invalid, INS_i32x4_abs, INS_invalid, INS_i64x2_abs, INS_invalid, INS_f32x4_abs, INS_f64x2_abs, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, Add, 16, 2, INS_i8x16_add, INS_i8x16_add, INS_i16x8_add, INS_i16x8_add, INS_i32x4_add, INS_i32x4_add, INS_i64x2_add, INS_i64x2_add, INS_f32x4_add, INS_f64x2_add, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(PackedSimd, AddPairwiseWidening, 16, 1, INS_i16x8_extadd_pairwise_s_i8x16, INS_i16x8_extadd_pairwise_u_i8x16, INS_i32x4_extadd_pairwise_s_i16x8, INS_i32x4_extadd_pairwise_u_i16x8, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, AddSaturate, 16, 2, INS_i8x16_add_sat_s, INS_i8x16_add_sat_u, INS_i16x8_add_sat_s, INS_i16x8_add_sat_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(PackedSimd, AllTrue, 16, 1, INS_i8x16_all_true, INS_i8x16_all_true, INS_i16x8_all_true, INS_i16x8_all_true, INS_i32x4_all_true, INS_i32x4_all_true, INS_i64x2_all_true, INS_i64x2_all_true, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, And, 16, 2, INS_v128_and, INS_v128_and, INS_v128_and, INS_v128_and, INS_v128_and, INS_v128_and, INS_v128_and, INS_v128_and, INS_v128_and, INS_v128_and, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(PackedSimd, AndNot, 16, 2, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, AnyTrue, 16, 1, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(PackedSimd, AverageRounded, 16, 2, INS_invalid, INS_i8x16_avgr_u, INS_invalid, INS_i16x8_avgr_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(PackedSimd, Bitmask, 16, 1, INS_i8x16_bitmask, INS_i8x16_bitmask, INS_i16x8_bitmask, INS_i16x8_bitmask, INS_i32x4_bitmask, INS_i32x4_bitmask, INS_i64x2_bitmask, INS_i64x2_bitmask, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, BitwiseSelect, 16, 3, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, Ceiling, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_ceil, INS_f64x2_ceil, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, CompareEqual, 16, 2, INS_i8x16_eq, INS_i8x16_eq, INS_i16x8_eq, INS_i16x8_eq, INS_i32x4_eq, INS_i32x4_eq, INS_i64x2_eq, INS_i64x2_eq, INS_f32x4_eq, INS_f64x2_eq, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(PackedSimd, CompareGreaterThan, 16, 2, INS_i8x16_gt_s, INS_i8x16_gt_u, INS_i16x8_gt_s, INS_i16x8_gt_u, INS_i32x4_gt_s, INS_i32x4_gt_u, INS_i64x2_gt_s, INS_invalid, INS_f32x4_gt, INS_f64x2_gt, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(PackedSimd, CompareGreaterThanOrEqual, 16, 2, INS_i8x16_ge_s, INS_i8x16_ge_u, INS_i16x8_ge_s, INS_i16x8_ge_u, INS_i32x4_ge_s, INS_i32x4_ge_u, INS_i64x2_ge_s, INS_invalid, INS_f32x4_ge, INS_f64x2_ge, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(PackedSimd, CompareLessThan, 16, 2, INS_i8x16_lt_s, INS_i8x16_lt_u, INS_i16x8_lt_s, INS_i16x8_lt_u, INS_i32x4_lt_s, INS_i32x4_lt_u, INS_i64x2_lt_s, INS_invalid, INS_f32x4_lt, INS_f64x2_lt, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(PackedSimd, CompareLessThanOrEqual, 16, 2, INS_i8x16_le_s, INS_i8x16_le_u, INS_i16x8_le_s, INS_i16x8_le_u, INS_i32x4_le_s, INS_i32x4_le_u, INS_i64x2_le_s, INS_invalid, INS_f32x4_le, INS_f64x2_le, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(PackedSimd, CompareNotEqual, 16, 2, INS_i8x16_ne, INS_i8x16_ne, INS_i16x8_ne, INS_i16x8_ne, INS_i32x4_ne, INS_i32x4_ne, INS_i64x2_ne, INS_i64x2_ne, INS_f32x4_ne, INS_f64x2_ne, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(PackedSimd, ConvertNarrowingSaturateSigned, 16, 2, INS_invalid, INS_invalid, INS_i8x16_narrow_i16x8_s, INS_invalid, INS_i16x8_narrow_i32x4_s, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, ConvertNarrowingSaturateUnsigned, 16, 2, INS_invalid, INS_invalid, INS_i8x16_narrow_i16x8_u, INS_invalid, INS_i16x8_narrow_i32x4_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, ConvertToDoubleLower, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f64x2_convert_low_s_i32x4, INS_f64x2_convert_low_u_i32x4, INS_invalid, INS_invalid, INS_f64x2_promote_low_f32x4, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, ConvertToInt32Saturate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_i32x4_trunc_sat_s_f32x4, INS_i32x4_trunc_sat_s_f64x2_zero, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, ConvertToSingle, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_convert_s_i32x4, INS_f32x4_convert_u_i32x4, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_demote_f64x2_zero, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, ConvertToUInt32Saturate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_i32x4_trunc_sat_u_f32x4, INS_i32x4_trunc_sat_u_f64x2_zero, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, Divide, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_div, INS_f64x2_div, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, Dot, 16, 2, INS_invalid, INS_invalid, INS_i32x4_dot_i16x8_s, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, ExtractScalar, 16, 2, INS_i8x16_extract_lane_s, INS_i8x16_extract_lane_u, INS_i16x8_extract_lane_s, INS_i16x8_extract_lane_u, INS_i32x4_extract_lane, INS_i32x4_extract_lane, INS_i64x2_extract_lane, INS_i64x2_extract_lane, INS_f32x4_extract_lane, INS_f64x2_extract_lane, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(PackedSimd, Floor, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_floor, INS_f64x2_floor, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndInsert, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndSplatVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(PackedSimd, LoadScalarVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(PackedSimd, LoadVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(PackedSimd, LoadWideningVector128, 16, 1, INS_v128_load8x8_s, INS_v128_load8x8_u, INS_v128_load16x4_s, INS_v128_load16x4_u, INS_v128_load32x2_s, INS_v128_load32x2_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(PackedSimd, Max, 16, 2, INS_i8x16_max_s, INS_i8x16_max_u, INS_i16x8_max_s, INS_i16x8_max_u, INS_i32x4_max_s, INS_i32x4_max_u, INS_invalid, INS_invalid, INS_f32x4_max, INS_f64x2_max, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(PackedSimd, Min, 16, 2, INS_i8x16_min_s, INS_i8x16_min_u, INS_i16x8_min_s, INS_i16x8_min_u, INS_i32x4_min_s, INS_i32x4_min_u, INS_invalid, INS_invalid, INS_f32x4_min, INS_f64x2_min, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(PackedSimd, Multiply, 16, 2, INS_invalid, INS_invalid, INS_i16x8_mul, INS_i16x8_mul, INS_i32x4_mul, INS_i32x4_mul, INS_i64x2_mul, INS_i64x2_mul, INS_f32x4_mul, INS_f64x2_mul, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(PackedSimd, MultiplyRoundedSaturateQ15, 16, 2, INS_invalid, INS_invalid, INS_i16x8_q15mulr_sat_s, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(PackedSimd, MultiplyWideningLower, 16, 2, INS_i16x8_extmul_low_s_i8x16, INS_i16x8_extmul_low_u_i8x16, INS_i32x4_extmul_low_s_i16x8, INS_i32x4_extmul_low_u_i16x8, INS_i64x2_extmul_low_s_i32x4, INS_i64x2_extmul_low_u_i32x4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, MultiplyWideningUpper, 16, 2, INS_i16x8_extmul_high_s_i8x16, INS_i16x8_extmul_high_u_i8x16, INS_i32x4_extmul_high_s_i16x8, INS_i32x4_extmul_high_u_i16x8, INS_i64x2_extmul_high_s_i32x4, INS_i64x2_extmul_high_u_i32x4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, Negate, 16, 1, INS_i8x16_neg, INS_i8x16_neg, INS_i16x8_neg, INS_i16x8_neg, INS_i32x4_neg, INS_i32x4_neg, INS_i64x2_neg, INS_i64x2_neg, INS_f32x4_neg, INS_f64x2_neg, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, Not, 16, 1, INS_v128_not, INS_v128_not, INS_v128_not, INS_v128_not, INS_v128_not, INS_v128_not, INS_v128_not, INS_v128_not, INS_v128_not, INS_v128_not, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, Or, 16, 2, INS_v128_or, INS_v128_or, INS_v128_or, INS_v128_or, INS_v128_or, INS_v128_or, INS_v128_or, INS_v128_or, INS_v128_or, INS_v128_or, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(PackedSimd, PopCount, 16, 1, INS_invalid, INS_i8x16_popcnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, PseudoMax, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_pmax, INS_f64x2_pmax, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, PseudoMin, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_pmin, INS_f64x2_pmin, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, ReplaceScalar, 16, 3, INS_i8x16_replace_lane, INS_i8x16_replace_lane, INS_i16x8_replace_lane, INS_i16x8_replace_lane, INS_i32x4_replace_lane, INS_i32x4_replace_lane, INS_i64x2_replace_lane, INS_i64x2_replace_lane, INS_f32x4_replace_lane, INS_f64x2_replace_lane, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(PackedSimd, RoundToNearest, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_nearest, INS_f64x2_nearest, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, ShiftLeft, 16, 2, INS_i8x16_shl, INS_i8x16_shl, INS_i16x8_shl, INS_i16x8_shl, INS_i32x4_shl, INS_i32x4_shl, INS_i64x2_shl, INS_i64x2_shl, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(PackedSimd, ShiftRightArithmetic, 16, 2, INS_i8x16_shr_s, INS_i8x16_shr_s, INS_i16x8_shr_s, INS_i16x8_shr_s, INS_i32x4_shr_s, INS_i32x4_shr_s, INS_i64x2_shr_s, INS_i64x2_shr_s, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(PackedSimd, ShiftRightLogical, 16, 2, INS_i8x16_shr_u, INS_i8x16_shr_u, INS_i16x8_shr_u, INS_i16x8_shr_u, INS_i32x4_shr_u, INS_i32x4_shr_u, INS_i64x2_shr_u, INS_i64x2_shr_u, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(PackedSimd, Shuffle, 16, 3, INS_i8x16_shuffle, INS_i8x16_shuffle, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, SignExtendWideningLower, 16, 1, INS_i16x8_extend_low_s_i8x16, INS_i16x8_extend_low_s_i8x16, INS_i32x4_extend_low_s_i16x8, INS_i32x4_extend_low_s_i16x8, INS_i64x2_extend_low_s_i32x4, INS_i64x2_extend_low_s_i32x4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, SignExtendWideningUpper, 16, 1, INS_i16x8_extend_high_s_i8x16, INS_i16x8_extend_high_s_i8x16, INS_i32x4_extend_high_s_i16x8, INS_i32x4_extend_high_s_i16x8, INS_i64x2_extend_high_s_i32x4, INS_i64x2_extend_high_s_i32x4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, Splat, 16, 1, INS_i8x16_splat, INS_i8x16_splat, INS_i16x8_splat, INS_i16x8_splat, INS_i32x4_splat, INS_i32x4_splat, INS_i64x2_splat, INS_i64x2_splat, INS_f32x4_splat, INS_f64x2_splat, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, Sqrt, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_sqrt, INS_f64x2_sqrt, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, Store, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(PackedSimd, StoreSelectedScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(PackedSimd, Subtract, 16, 2, INS_i8x16_sub, INS_i8x16_sub, INS_i16x8_sub, INS_i16x8_sub, INS_i32x4_sub, INS_i32x4_sub, INS_i64x2_sub, INS_i64x2_sub, INS_f32x4_sub, INS_f64x2_sub, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, SubtractSaturate, 16, 2, INS_i8x16_sub_sat_s, INS_i8x16_sub_sat_u, INS_i16x8_sub_sat_s, INS_i16x8_sub_sat_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, Swizzle, 16, 2, INS_i8x16_swizzle, INS_i8x16_swizzle, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, Truncate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_trunc, INS_f64x2_trunc, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, Xor, 16, 2, INS_v128_xor, INS_v128_xor, INS_v128_xor, INS_v128_xor, INS_v128_xor, INS_v128_xor, INS_v128_xor, INS_v128_xor, INS_v128_xor, INS_v128_xor, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(PackedSimd, ZeroExtendWideningLower, 16, 1, INS_i16x8_extend_low_u_i8x16, INS_i16x8_extend_low_u_i8x16, INS_i32x4_extend_low_u_i16x8, INS_i32x4_extend_low_u_i16x8, INS_i64x2_extend_low_u_i32x4, INS_i64x2_extend_low_u_i32x4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, ZeroExtendWideningUpper, 16, 1, INS_i16x8_extend_high_u_i8x16, INS_i16x8_extend_high_u_i8x16, INS_i32x4_extend_high_u_i16x8, INS_i32x4_extend_high_u_i16x8, INS_i64x2_extend_high_u_i32x4, INS_i64x2_extend_high_u_i32x4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, Abs, 16, 1, INS_i8x16_abs, INS_invalid, INS_i16x8_abs, INS_invalid, INS_i32x4_abs, INS_invalid, INS_i64x2_abs, INS_invalid, INS_f32x4_abs, INS_f64x2_abs, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, Add, 16, 2, INS_i8x16_add, INS_i8x16_add, INS_i16x8_add, INS_i16x8_add, INS_i32x4_add, INS_i32x4_add, INS_i64x2_add, INS_i64x2_add, INS_f32x4_add, INS_f64x2_add, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(PackedSimd, AddPairwiseWidening, 16, 1, INS_i16x8_extadd_pairwise_s_i8x16, INS_i16x8_extadd_pairwise_u_i8x16, INS_i32x4_extadd_pairwise_s_i16x8, INS_i32x4_extadd_pairwise_u_i16x8, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, AddSaturate, 16, 2, INS_i8x16_add_sat_s, INS_i8x16_add_sat_u, INS_i16x8_add_sat_s, INS_i16x8_add_sat_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(PackedSimd, AllTrue, 16, 1, INS_i8x16_all_true, INS_i8x16_all_true, INS_i16x8_all_true, INS_i16x8_all_true, INS_i32x4_all_true, INS_i32x4_all_true, INS_i64x2_all_true, INS_i64x2_all_true, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, And, 16, 2, INS_v128_and, INS_v128_and, INS_v128_and, INS_v128_and, INS_v128_and, INS_v128_and, INS_v128_and, INS_v128_and, INS_v128_and, INS_v128_and, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(PackedSimd, AndNot, 16, 2, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, AnyTrue, 16, 1, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(PackedSimd, AverageRounded, 16, 2, INS_invalid, INS_i8x16_avgr_u, INS_invalid, INS_i16x8_avgr_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(PackedSimd, Bitmask, 16, 1, INS_i8x16_bitmask, INS_i8x16_bitmask, INS_i16x8_bitmask, INS_i16x8_bitmask, INS_i32x4_bitmask, INS_i32x4_bitmask, INS_i64x2_bitmask, INS_i64x2_bitmask, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, BitwiseSelect, 16, 3, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, Ceiling, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_ceil, INS_f64x2_ceil, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, CompareEqual, 16, 2, INS_i8x16_eq, INS_i8x16_eq, INS_i16x8_eq, INS_i16x8_eq, INS_i32x4_eq, INS_i32x4_eq, INS_i64x2_eq, INS_i64x2_eq, INS_f32x4_eq, INS_f64x2_eq, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(PackedSimd, CompareGreaterThan, 16, 2, INS_i8x16_gt_s, INS_i8x16_gt_u, INS_i16x8_gt_s, INS_i16x8_gt_u, INS_i32x4_gt_s, INS_i32x4_gt_u, INS_i64x2_gt_s, INS_invalid, INS_f32x4_gt, INS_f64x2_gt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(PackedSimd, CompareGreaterThanOrEqual, 16, 2, INS_i8x16_ge_s, INS_i8x16_ge_u, INS_i16x8_ge_s, INS_i16x8_ge_u, INS_i32x4_ge_s, INS_i32x4_ge_u, INS_i64x2_ge_s, INS_invalid, INS_f32x4_ge, INS_f64x2_ge, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(PackedSimd, CompareLessThan, 16, 2, INS_i8x16_lt_s, INS_i8x16_lt_u, INS_i16x8_lt_s, INS_i16x8_lt_u, INS_i32x4_lt_s, INS_i32x4_lt_u, INS_i64x2_lt_s, INS_invalid, INS_f32x4_lt, INS_f64x2_lt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(PackedSimd, CompareLessThanOrEqual, 16, 2, INS_i8x16_le_s, INS_i8x16_le_u, INS_i16x8_le_s, INS_i16x8_le_u, INS_i32x4_le_s, INS_i32x4_le_u, INS_i64x2_le_s, INS_invalid, INS_f32x4_le, INS_f64x2_le, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(PackedSimd, CompareNotEqual, 16, 2, INS_i8x16_ne, INS_i8x16_ne, INS_i16x8_ne, INS_i16x8_ne, INS_i32x4_ne, INS_i32x4_ne, INS_i64x2_ne, INS_i64x2_ne, INS_f32x4_ne, INS_f64x2_ne, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(PackedSimd, ConvertNarrowingSaturateSigned, 16, 2, INS_invalid, INS_invalid, INS_i8x16_narrow_i16x8_s, INS_invalid, INS_i16x8_narrow_i32x4_s, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, ConvertNarrowingSaturateUnsigned, 16, 2, INS_invalid, INS_invalid, INS_i8x16_narrow_i16x8_u, INS_invalid, INS_i16x8_narrow_i32x4_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, ConvertToDoubleLower, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f64x2_convert_low_s_i32x4, INS_f64x2_convert_low_u_i32x4, INS_invalid, INS_invalid, INS_f64x2_promote_low_f32x4, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, ConvertToInt32Saturate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_i32x4_trunc_sat_s_f32x4, INS_i32x4_trunc_sat_s_f64x2_zero, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, ConvertToSingle, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_convert_s_i32x4, INS_f32x4_convert_u_i32x4, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_demote_f64x2_zero, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, ConvertToUInt32Saturate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_i32x4_trunc_sat_u_f32x4, INS_i32x4_trunc_sat_u_f64x2_zero, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, Divide, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_div, INS_f64x2_div, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, Dot, 16, 2, INS_invalid, INS_invalid, INS_i32x4_dot_i16x8_s, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, ExtractScalar, 16, 2, INS_i8x16_extract_lane_s, INS_i8x16_extract_lane_u, INS_i16x8_extract_lane_s, INS_i16x8_extract_lane_u, INS_i32x4_extract_lane, INS_i32x4_extract_lane, INS_i64x2_extract_lane, INS_i64x2_extract_lane, INS_f32x4_extract_lane, INS_f64x2_extract_lane, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(PackedSimd, Floor, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_floor, INS_f64x2_floor, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndInsert, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndSplatVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(PackedSimd, LoadScalarVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(PackedSimd, LoadVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(PackedSimd, LoadWideningVector128, 16, 1, INS_v128_load8x8_s, INS_v128_load8x8_u, INS_v128_load16x4_s, INS_v128_load16x4_u, INS_v128_load32x2_s, INS_v128_load32x2_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(PackedSimd, Max, 16, 2, INS_i8x16_max_s, INS_i8x16_max_u, INS_i16x8_max_s, INS_i16x8_max_u, INS_i32x4_max_s, INS_i32x4_max_u, INS_invalid, INS_invalid, INS_f32x4_max, INS_f64x2_max, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(PackedSimd, Min, 16, 2, INS_i8x16_min_s, INS_i8x16_min_u, INS_i16x8_min_s, INS_i16x8_min_u, INS_i32x4_min_s, INS_i32x4_min_u, INS_invalid, INS_invalid, INS_f32x4_min, INS_f64x2_min, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(PackedSimd, Multiply, 16, 2, INS_invalid, INS_invalid, INS_i16x8_mul, INS_i16x8_mul, INS_i32x4_mul, INS_i32x4_mul, INS_i64x2_mul, INS_i64x2_mul, INS_f32x4_mul, INS_f64x2_mul, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(PackedSimd, MultiplyRoundedSaturateQ15, 16, 2, INS_invalid, INS_invalid, INS_i16x8_q15mulr_sat_s, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(PackedSimd, MultiplyWideningLower, 16, 2, INS_i16x8_extmul_low_s_i8x16, INS_i16x8_extmul_low_u_i8x16, INS_i32x4_extmul_low_s_i16x8, INS_i32x4_extmul_low_u_i16x8, INS_i64x2_extmul_low_s_i32x4, INS_i64x2_extmul_low_u_i32x4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, MultiplyWideningUpper, 16, 2, INS_i16x8_extmul_high_s_i8x16, INS_i16x8_extmul_high_u_i8x16, INS_i32x4_extmul_high_s_i16x8, INS_i32x4_extmul_high_u_i16x8, INS_i64x2_extmul_high_s_i32x4, INS_i64x2_extmul_high_u_i32x4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, Negate, 16, 1, INS_i8x16_neg, INS_i8x16_neg, INS_i16x8_neg, INS_i16x8_neg, INS_i32x4_neg, INS_i32x4_neg, INS_i64x2_neg, INS_i64x2_neg, INS_f32x4_neg, INS_f64x2_neg, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, Not, 16, 1, INS_v128_not, INS_v128_not, INS_v128_not, INS_v128_not, INS_v128_not, INS_v128_not, INS_v128_not, INS_v128_not, INS_v128_not, INS_v128_not, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, Or, 16, 2, INS_v128_or, INS_v128_or, INS_v128_or, INS_v128_or, INS_v128_or, INS_v128_or, INS_v128_or, INS_v128_or, INS_v128_or, INS_v128_or, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(PackedSimd, PopCount, 16, 1, INS_invalid, INS_i8x16_popcnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, PseudoMax, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_pmax, INS_f64x2_pmax, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, PseudoMin, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_pmin, INS_f64x2_pmin, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, ReplaceScalar, 16, 3, INS_i8x16_replace_lane, INS_i8x16_replace_lane, INS_i16x8_replace_lane, INS_i16x8_replace_lane, INS_i32x4_replace_lane, INS_i32x4_replace_lane, INS_i64x2_replace_lane, INS_i64x2_replace_lane, INS_f32x4_replace_lane, INS_f64x2_replace_lane, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(PackedSimd, RoundToNearest, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_nearest, INS_f64x2_nearest, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, ShiftLeft, 16, 2, INS_i8x16_shl, INS_i8x16_shl, INS_i16x8_shl, INS_i16x8_shl, INS_i32x4_shl, INS_i32x4_shl, INS_i64x2_shl, INS_i64x2_shl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(PackedSimd, ShiftRightArithmetic, 16, 2, INS_i8x16_shr_s, INS_i8x16_shr_s, INS_i16x8_shr_s, INS_i16x8_shr_s, INS_i32x4_shr_s, INS_i32x4_shr_s, INS_i64x2_shr_s, INS_i64x2_shr_s, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(PackedSimd, ShiftRightLogical, 16, 2, INS_i8x16_shr_u, INS_i8x16_shr_u, INS_i16x8_shr_u, INS_i16x8_shr_u, INS_i32x4_shr_u, INS_i32x4_shr_u, INS_i64x2_shr_u, INS_i64x2_shr_u, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(PackedSimd, Shuffle, 16, 3, INS_i8x16_shuffle, INS_i8x16_shuffle, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, SignExtendWideningLower, 16, 1, INS_i16x8_extend_low_s_i8x16, INS_i16x8_extend_low_s_i8x16, INS_i32x4_extend_low_s_i16x8, INS_i32x4_extend_low_s_i16x8, INS_i64x2_extend_low_s_i32x4, INS_i64x2_extend_low_s_i32x4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, SignExtendWideningUpper, 16, 1, INS_i16x8_extend_high_s_i8x16, INS_i16x8_extend_high_s_i8x16, INS_i32x4_extend_high_s_i16x8, INS_i32x4_extend_high_s_i16x8, INS_i64x2_extend_high_s_i32x4, INS_i64x2_extend_high_s_i32x4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, Splat, 16, 1, INS_i8x16_splat, INS_i8x16_splat, INS_i16x8_splat, INS_i16x8_splat, INS_i32x4_splat, INS_i32x4_splat, INS_i64x2_splat, INS_i64x2_splat, INS_f32x4_splat, INS_f64x2_splat, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, Sqrt, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_sqrt, INS_f64x2_sqrt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, Store, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(PackedSimd, StoreSelectedScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(PackedSimd, Subtract, 16, 2, INS_i8x16_sub, INS_i8x16_sub, INS_i16x8_sub, INS_i16x8_sub, INS_i32x4_sub, INS_i32x4_sub, INS_i64x2_sub, INS_i64x2_sub, INS_f32x4_sub, INS_f64x2_sub, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, SubtractSaturate, 16, 2, INS_i8x16_sub_sat_s, INS_i8x16_sub_sat_u, INS_i16x8_sub_sat_s, INS_i16x8_sub_sat_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, Swizzle, 16, 2, INS_i8x16_swizzle, INS_i8x16_swizzle, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, Truncate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_trunc, INS_f64x2_trunc, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, Xor, 16, 2, INS_v128_xor, INS_v128_xor, INS_v128_xor, INS_v128_xor, INS_v128_xor, INS_v128_xor, INS_v128_xor, INS_v128_xor, INS_v128_xor, INS_v128_xor, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(PackedSimd, ZeroExtendWideningLower, 16, 1, INS_i16x8_extend_low_u_i8x16, INS_i16x8_extend_low_u_i8x16, INS_i32x4_extend_low_u_i16x8, INS_i32x4_extend_low_u_i16x8, INS_i64x2_extend_low_u_i32x4, INS_i64x2_extend_low_u_i32x4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, ZeroExtendWideningUpper, 16, 1, INS_i16x8_extend_high_u_i8x16, INS_i16x8_extend_high_u_i8x16, INS_i32x4_extend_high_u_i16x8, INS_i32x4_extend_high_u_i16x8, INS_i64x2_extend_high_u_i32x4, INS_i64x2_extend_high_u_i32x4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) #define LAST_NI_PackedSimd NI_PackedSimd_ZeroExtendWideningUpper #endif // FEATURE_HW_INTRINSICS diff --git a/src/coreclr/jit/hwintrinsiclistxarch.h b/src/coreclr/jit/hwintrinsiclistxarch.h index 12bea05dbae65f..b851830b4566f2 100644 --- a/src/coreclr/jit/hwintrinsiclistxarch.h +++ b/src/coreclr/jit/hwintrinsiclistxarch.h @@ -28,165 +28,165 @@ // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Intrinsics for X86Base, SSE, SSE2, SSE3, SSSE3, SSE41, SSE42, POPCNT #define FIRST_NI_X86Base NI_X86Base_Abs -HARDWARE_INTRINSIC(X86Base, Abs, 16, 1, INS_pabsb, INS_invalid, INS_pabsw, INS_invalid, INS_pabsd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, Add, 16, 2, INS_paddb, INS_paddb, INS_paddw, INS_paddw, INS_paddd, INS_paddd, INS_paddq, INS_paddq, INS_addps, INS_addpd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(X86Base, AddSaturate, 16, 2, INS_paddsb, INS_paddusb, INS_paddsw, INS_paddusw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(X86Base, AddScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_addss, INS_addsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(X86Base, AddSubtract, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_addsubps, INS_addsubpd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, AlignRight, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_palignr, INS_palignr, INS_palignr, INS_palignr, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(X86Base, And, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pandd, INS_pandd, INS_pandd, INS_pandd, INS_andps, INS_andpd, 1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(X86Base, AndNot, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pandnd, INS_pandnd, INS_pandnd, INS_pandnd, INS_andnps, INS_andnpd, 1, 1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(X86Base, Average, 16, 2, INS_invalid, INS_pavgb, INS_invalid, INS_pavgw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(X86Base, BitScanForward, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bsf, INS_bsf, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, BitScanReverse, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bsr, INS_bsr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, Blend, 16, 3, INS_invalid, INS_invalid, INS_pblendw, INS_pblendw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blendps, INS_blendpd, 1, 1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, BlendVariable, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pblendvb, INS_pblendvb, INS_pblendvb, INS_pblendvb, INS_blendvps, INS_blendvpd, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(X86Base, Ceiling, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, CeilingScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundss, INS_roundsd, -1, 8, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(X86Base, CompareEqual, 16, 2, INS_pcmpeqb, INS_pcmpeqb, INS_pcmpeqw, INS_pcmpeqw, INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqq, INS_pcmpeqq, INS_cmpps, INS_cmppd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(X86Base, CompareGreaterThan, 16, 2, INS_pcmpgtb, INS_invalid, INS_pcmpgtw, INS_invalid, INS_pcmpgtd, INS_invalid, INS_pcmpgtq, INS_invalid, INS_cmpps, INS_cmppd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(X86Base, CompareGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(X86Base, CompareLessThan, 16, 2, INS_pcmpgtb, INS_invalid, INS_pcmpgtw, INS_invalid, INS_pcmpgtd, INS_invalid, INS_pcmpgtq, INS_invalid, INS_cmpps, INS_cmppd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(X86Base, CompareLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(X86Base, CompareNotEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(X86Base, CompareNotGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(X86Base, CompareNotGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(X86Base, CompareNotLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(X86Base, CompareNotLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(X86Base, CompareOrdered, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(X86Base, CompareScalarEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, CompareScalarGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, CompareScalarGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, CompareScalarLessThan, 16, 2, INS_pcmpgtb, INS_invalid, INS_pcmpgtw, INS_invalid, INS_pcmpgtd, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, 1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, CompareScalarLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, CompareScalarNotEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, CompareScalarNotGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, CompareScalarNotGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, CompareScalarNotLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, CompareScalarNotLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, CompareScalarOrdered, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, CompareScalarOrderedEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_comiss, INS_comisd, -1, 3, HW_Category_SIMDScalar, HW_Flag_Commutative|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(X86Base, CompareScalarOrderedGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_comiss, INS_comisd, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(X86Base, CompareScalarOrderedGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_comiss, INS_comisd, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(X86Base, CompareScalarOrderedLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_comiss, INS_comisd, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(X86Base, CompareScalarOrderedLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_comiss, INS_comisd, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(X86Base, CompareScalarOrderedNotEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_comiss, INS_comisd, -1, 3, HW_Category_SIMDScalar, HW_Flag_Commutative|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(X86Base, CompareScalarUnordered, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, CompareScalarUnorderedEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ucomiss, INS_ucomisd, -1, 3, HW_Category_SIMDScalar, HW_Flag_Commutative|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(X86Base, CompareScalarUnorderedGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ucomiss, INS_ucomisd, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(X86Base, CompareScalarUnorderedGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ucomiss, INS_ucomisd, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(X86Base, CompareScalarUnorderedLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ucomiss, INS_ucomisd, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(X86Base, CompareScalarUnorderedLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ucomiss, INS_ucomisd, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(X86Base, CompareScalarUnorderedNotEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ucomiss, INS_ucomisd, -1, 3, HW_Category_SIMDScalar, HW_Flag_Commutative|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(X86Base, CompareUnordered, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(X86Base, ConvertScalarToVector128Double, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2sd32, INS_invalid, INS_invalid, INS_invalid, INS_cvtss2sd, INS_invalid, 5, 4, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(X86Base, ConvertScalarToVector128Int32, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, ConvertScalarToVector128Single, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2ss32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsd2ss, 5, 4, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(X86Base, ConvertScalarToVector128UInt32, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, ConvertToInt32, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_invalid, INS_invalid, INS_invalid, INS_cvtss2si32, INS_cvtsd2si32, 3, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, ConvertToInt32WithTruncation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvttss2si32, INS_cvttsd2si32, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, ConvertToUInt32, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, ConvertToVector128Double, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtdq2pd, INS_invalid, INS_invalid, INS_invalid, INS_cvtps2pd, INS_invalid, 5, 5, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, ConvertToVector128Int16, 16, 1, INS_pmovsxbw, INS_pmovzxbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_MaybeMemoryLoad) -HARDWARE_INTRINSIC(X86Base, ConvertToVector128Int32, 16, 1, INS_pmovsxbd, INS_pmovzxbd, INS_pmovsxwd, INS_pmovzxwd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtps2dq, INS_cvtpd2dq, 1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_MaybeMemoryLoad) -HARDWARE_INTRINSIC(X86Base, ConvertToVector128Int32WithTruncation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvttps2dq, INS_cvttpd2dq, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, ConvertToVector128Int64, 16, 1, INS_pmovsxbq, INS_pmovzxbq, INS_pmovsxwq, INS_pmovzxwq, INS_pmovsxdq, INS_pmovzxdq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_MaybeMemoryLoad) -HARDWARE_INTRINSIC(X86Base, ConvertToVector128Single, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtdq2ps, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtpd2ps, 4, 5, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, Crc32, 0, 2, INS_invalid, INS_crc32, INS_invalid, INS_crc32, INS_invalid, INS_crc32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(X86Base, DivRem, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_idiv, INS_div, INS_idiv, INS_div, INS_invalid, INS_invalid, 25, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_BaseTypeFromSecondArg|HW_Flag_MultiReg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(X86Base, Divide, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_divps, INS_divpd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(X86Base, DivideScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_divss, INS_divsd, -1, -1, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(X86Base, DotProduct, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_dpps, INS_dppd, -1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, Extract, 16, 2, INS_pextrb, INS_pextrb, INS_pextrw, INS_pextrw, INS_pextrd, INS_pextrd, INS_invalid, INS_invalid, INS_extractps, INS_invalid, 4, 4, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsScalarT) -HARDWARE_INTRINSIC(X86Base, Floor, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, FloorScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundss, INS_roundsd, -1, 8, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(X86Base, HorizontalAdd, 16, 2, INS_invalid, INS_invalid, INS_phaddw, INS_phaddw, INS_phaddd, INS_phaddd, INS_invalid, INS_invalid, INS_haddps, INS_haddpd, 3, 6, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, HorizontalAddSaturate, 16, 2, INS_invalid, INS_invalid, INS_phaddsw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, HorizontalSubtract, 16, 2, INS_invalid, INS_invalid, INS_phsubw, INS_invalid, INS_phsubd, INS_invalid, INS_invalid, INS_invalid, INS_hsubps, INS_hsubpd, 3, 6, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, HorizontalSubtractSaturate, 16, 2, INS_invalid, INS_invalid, INS_phsubsw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, Insert, 16, 3, INS_pinsrb, INS_pinsrb, INS_pinsrw, INS_pinsrw, INS_pinsrd, INS_pinsrd, INS_invalid, INS_invalid, INS_insertps, INS_invalid, 4, 1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(X86Base, LoadAlignedVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movaps, INS_movapd, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoRMWSemantics|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(X86Base, LoadAlignedVector128NonTemporal, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movntdqa, INS_movntdqa, INS_movntdqa, INS_movntdqa, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoRMWSemantics|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(X86Base, LoadAndDuplicateToVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movddup, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, LoadDquVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_lddqu, INS_lddqu, INS_lddqu, INS_lddqu, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoRMWSemantics|HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(X86Base, LoadFence, 0, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Barrier) -HARDWARE_INTRINSIC(X86Base, LoadHigh, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movhps, INS_movhpd, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, LoadLow, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movlps, INS_movlpd, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, LoadScalarVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_movd32, INS_movq, INS_movq, INS_movss, INS_movsd_simd, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, LoadVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(X86Base, MaskMove, 16, 3, INS_maskmovdqu, INS_maskmovdqu, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(X86Base, Max, 16, 2, INS_pmaxsb, INS_pmaxub, INS_pmaxsw, INS_pmaxuw, INS_pmaxsd, INS_pmaxud, INS_invalid, INS_invalid, INS_maxps, INS_maxpd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative) -HARDWARE_INTRINSIC(X86Base, MaxScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_maxss, INS_maxsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(X86Base, MemoryFence, 0, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Barrier) -HARDWARE_INTRINSIC(X86Base, Min, 16, 2, INS_pminsb, INS_pminub, INS_pminsw, INS_pminuw, INS_pminsd, INS_pminud, INS_invalid, INS_invalid, INS_minps, INS_minpd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative) -HARDWARE_INTRINSIC(X86Base, MinHorizontal, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_phminposuw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, MinScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_minss, INS_minsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(X86Base, MoveAndDuplicate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movddup, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, MoveHighAndDuplicate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movshdup, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, MoveHighToLow, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movhlps, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment) -HARDWARE_INTRINSIC(X86Base, MoveLowAndDuplicate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movsldup, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, MoveLowToHigh, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movlhps, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment) -HARDWARE_INTRINSIC(X86Base, MoveMask, 16, 1, INS_pmovmskb, INS_pmovmskb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movmskps, INS_movmskpd, 3, 3, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, MoveScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movq, INS_movq, INS_movss, INS_movsd_simd, 1, 1, HW_Category_SIMDScalar, HW_Flag_NoContainment) -HARDWARE_INTRINSIC(X86Base, MultipleSumAbsoluteDifferences, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_mpsadbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, Multiply, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pmuldq, INS_pmuludq, INS_mulps, INS_mulpd, 5, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(X86Base, MultiplyAddAdjacent, 16, 2, INS_invalid, INS_invalid, INS_pmaddubsw, INS_invalid, INS_pmaddwd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative) -HARDWARE_INTRINSIC(X86Base, MultiplyHigh, 16, 2, INS_invalid, INS_invalid, INS_pmulhw, INS_pmulhuw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(X86Base, MultiplyHighRoundScale, 16, 2, INS_invalid, INS_invalid, INS_pmulhrsw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(X86Base, MultiplyLow, 16, 2, INS_invalid, INS_invalid, INS_pmullw, INS_pmullw, INS_pmulld, INS_pmulld, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(X86Base, MultiplyScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mulss, INS_mulsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(X86Base, Or, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pord, INS_pord, INS_pord, INS_pord, INS_orps, INS_orpd, 1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(X86Base, PackSignedSaturate, 16, 2, INS_packsswb, INS_invalid, INS_packssdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(X86Base, PackUnsignedSaturate, 16, 2, INS_invalid, INS_packuswb, INS_invalid, INS_packusdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(X86Base, Pause, 0, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(X86Base, PopCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_popcnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(X86Base, Prefetch0, 0, 1, INS_invalid, INS_prefetcht0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(X86Base, Prefetch1, 0, 1, INS_invalid, INS_prefetcht1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(X86Base, Prefetch2, 0, 1, INS_invalid, INS_prefetcht2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(X86Base, PrefetchNonTemporal, 0, 1, INS_invalid, INS_prefetchnta, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(X86Base, Reciprocal, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rcpps, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, ReciprocalScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rcpss, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, ReciprocalSqrt, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rsqrtps, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, ReciprocalSqrtScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rsqrtss, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, RoundCurrentDirection, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, RoundCurrentDirectionScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundss, INS_roundsd, -1, 8, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(X86Base, RoundToNearestInteger, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, RoundToNearestIntegerScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundss, INS_roundsd, -1, 8, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(X86Base, RoundToNegativeInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, RoundToNegativeInfinityScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundss, INS_roundsd, -1, 8, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(X86Base, RoundToPositiveInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, RoundToPositiveInfinityScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundss, INS_roundsd, -1, 8, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(X86Base, RoundToZero, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, RoundToZeroScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundss, INS_roundsd, -1, 8, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(X86Base, ShiftLeftLogical, 16, 2, INS_invalid, INS_invalid, INS_psllw, INS_psllw, INS_pslld, INS_pslld, INS_psllq, INS_psllq, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(X86Base, ShiftLeftLogical128BitLane, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pslldq, INS_pslldq, INS_pslldq, INS_pslldq, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(X86Base, ShiftRightArithmetic, 16, 2, INS_invalid, INS_invalid, INS_psraw, INS_invalid, INS_psrad, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(X86Base, ShiftRightLogical, 16, 2, INS_invalid, INS_invalid, INS_psrlw, INS_psrlw, INS_psrld, INS_psrld, INS_psrlq, INS_psrlq, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(X86Base, ShiftRightLogical128BitLane, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_psrldq, INS_psrldq, INS_psrldq, INS_psrldq, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(X86Base, Shuffle, 16, -1, INS_pshufb, INS_pshufb, INS_invalid, INS_invalid, INS_pshufd, INS_pshufd, INS_invalid, INS_invalid, INS_shufps, INS_shufpd, 1, 1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(X86Base, ShuffleHigh, 16, 2, INS_invalid, INS_invalid, INS_pshufhw, INS_pshufhw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(X86Base, ShuffleLow, 16, 2, INS_invalid, INS_invalid, INS_pshuflw, INS_pshuflw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(X86Base, Sign, 16, 2, INS_psignb, INS_invalid, INS_psignw, INS_invalid, INS_psignd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, Sqrt, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqrtps, INS_sqrtpd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, SqrtScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqrtss, INS_sqrtsd, -1, -1, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(X86Base, Store, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(X86Base, StoreAligned, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movaps, INS_movapd, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromSecondArg|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(X86Base, StoreAlignedNonTemporal, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movntdq, INS_movntdq, INS_movntdq, INS_movntdq, INS_movntps, INS_movntpd, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromSecondArg|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(X86Base, StoreFence, 0, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Barrier) -HARDWARE_INTRINSIC(X86Base, StoreHigh, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movhps, INS_movhpd, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(X86Base, StoreLow, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movlps, INS_movlpd, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(X86Base, StoreNonTemporal, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movnti32, INS_movnti32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(X86Base, StoreScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_movd32, INS_movq, INS_movq, INS_movss, INS_movsd_simd, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(X86Base, Subtract, 16, 2, INS_psubb, INS_psubb, INS_psubw, INS_psubw, INS_psubd, INS_psubd, INS_psubq, INS_psubq, INS_subps, INS_subpd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(X86Base, SubtractSaturate, 16, 2, INS_psubsb, INS_psubusb, INS_psubsw, INS_psubusw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(X86Base, SubtractScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_subss, INS_subsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(X86Base, SumAbsoluteDifferences, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_psadbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(X86Base, TestC, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(X86Base, TestNotZAndNotC, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(X86Base, TestZ, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(X86Base, UnpackHigh, 16, 2, INS_punpckhbw, INS_punpckhbw, INS_punpckhwd, INS_punpckhwd, INS_punpckhdq, INS_punpckhdq, INS_punpckhqdq, INS_punpckhqdq, INS_unpckhps, INS_unpckhpd, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(X86Base, UnpackLow, 16, 2, INS_punpcklbw, INS_punpcklbw, INS_punpcklwd, INS_punpcklwd, INS_punpckldq, INS_punpckldq, INS_punpcklqdq, INS_punpcklqdq, INS_unpcklps, INS_unpcklpd, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(X86Base, Xor, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pxord, INS_pxord, INS_pxord, INS_pxord, INS_xorps, INS_xorpd, 1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(X86Base, Abs, 16, 1, INS_pabsb, INS_invalid, INS_pabsw, INS_invalid, INS_pabsd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, Add, 16, 2, INS_paddb, INS_paddb, INS_paddw, INS_paddw, INS_paddd, INS_paddd, INS_paddq, INS_paddq, INS_addps, INS_addpd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(X86Base, AddSaturate, 16, 2, INS_paddsb, INS_paddusb, INS_paddsw, INS_paddusw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(X86Base, AddScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_addss, INS_addsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(X86Base, AddSubtract, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_addsubps, INS_addsubpd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, AlignRight, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_palignr, INS_palignr, INS_palignr, INS_palignr, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(X86Base, And, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pandd, INS_pandd, INS_pandd, INS_pandd, INS_andps, INS_andpd, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(X86Base, AndNot, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pandnd, INS_pandnd, INS_pandnd, INS_pandnd, INS_andnps, INS_andnpd, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(X86Base, Average, 16, 2, INS_invalid, INS_pavgb, INS_invalid, INS_pavgw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(X86Base, BitScanForward, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bsf, INS_bsf, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, BitScanReverse, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bsr, INS_bsr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, Blend, 16, 3, INS_invalid, INS_invalid, INS_pblendw, INS_pblendw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blendps, INS_blendpd, INS_invalid, 1, 1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, BlendVariable, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pblendvb, INS_pblendvb, INS_pblendvb, INS_pblendvb, INS_blendvps, INS_blendvpd, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(X86Base, Ceiling, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, INS_invalid, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, CeilingScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundss, INS_roundsd, INS_invalid, -1, 8, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(X86Base, CompareEqual, 16, 2, INS_pcmpeqb, INS_pcmpeqb, INS_pcmpeqw, INS_pcmpeqw, INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqq, INS_pcmpeqq, INS_cmpps, INS_cmppd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(X86Base, CompareGreaterThan, 16, 2, INS_pcmpgtb, INS_invalid, INS_pcmpgtw, INS_invalid, INS_pcmpgtd, INS_invalid, INS_pcmpgtq, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(X86Base, CompareGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(X86Base, CompareLessThan, 16, 2, INS_pcmpgtb, INS_invalid, INS_pcmpgtw, INS_invalid, INS_pcmpgtd, INS_invalid, INS_pcmpgtq, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(X86Base, CompareLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(X86Base, CompareNotEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(X86Base, CompareNotGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(X86Base, CompareNotGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(X86Base, CompareNotLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(X86Base, CompareNotLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(X86Base, CompareOrdered, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(X86Base, CompareScalarEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, CompareScalarGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, CompareScalarGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, CompareScalarLessThan, 16, 2, INS_pcmpgtb, INS_invalid, INS_pcmpgtw, INS_invalid, INS_pcmpgtd, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, INS_invalid, 1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, CompareScalarLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, CompareScalarNotEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, CompareScalarNotGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, CompareScalarNotGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, CompareScalarNotLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, CompareScalarNotLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, CompareScalarOrdered, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, CompareScalarOrderedEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_comiss, INS_comisd, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_Commutative|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(X86Base, CompareScalarOrderedGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_comiss, INS_comisd, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(X86Base, CompareScalarOrderedGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_comiss, INS_comisd, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(X86Base, CompareScalarOrderedLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_comiss, INS_comisd, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(X86Base, CompareScalarOrderedLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_comiss, INS_comisd, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(X86Base, CompareScalarOrderedNotEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_comiss, INS_comisd, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_Commutative|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(X86Base, CompareScalarUnordered, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, CompareScalarUnorderedEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ucomiss, INS_ucomisd, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_Commutative|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(X86Base, CompareScalarUnorderedGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ucomiss, INS_ucomisd, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(X86Base, CompareScalarUnorderedGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ucomiss, INS_ucomisd, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(X86Base, CompareScalarUnorderedLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ucomiss, INS_ucomisd, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(X86Base, CompareScalarUnorderedLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ucomiss, INS_ucomisd, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(X86Base, CompareScalarUnorderedNotEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ucomiss, INS_ucomisd, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_Commutative|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(X86Base, CompareUnordered, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(X86Base, ConvertScalarToVector128Double, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2sd32, INS_invalid, INS_invalid, INS_invalid, INS_cvtss2sd, INS_invalid, INS_invalid, 5, 4, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(X86Base, ConvertScalarToVector128Int32, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, ConvertScalarToVector128Single, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2ss32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsd2ss, INS_invalid, 5, 4, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(X86Base, ConvertScalarToVector128UInt32, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, ConvertToInt32, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_invalid, INS_invalid, INS_invalid, INS_cvtss2si32, INS_cvtsd2si32, INS_invalid, 3, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, ConvertToInt32WithTruncation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvttss2si32, INS_cvttsd2si32, INS_invalid, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, ConvertToUInt32, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, ConvertToVector128Double, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtdq2pd, INS_invalid, INS_invalid, INS_invalid, INS_cvtps2pd, INS_invalid, INS_invalid, 5, 5, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, ConvertToVector128Int16, 16, 1, INS_pmovsxbw, INS_pmovzxbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_MaybeMemoryLoad) +HARDWARE_INTRINSIC(X86Base, ConvertToVector128Int32, 16, 1, INS_pmovsxbd, INS_pmovzxbd, INS_pmovsxwd, INS_pmovzxwd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtps2dq, INS_cvtpd2dq, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_MaybeMemoryLoad) +HARDWARE_INTRINSIC(X86Base, ConvertToVector128Int32WithTruncation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvttps2dq, INS_cvttpd2dq, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, ConvertToVector128Int64, 16, 1, INS_pmovsxbq, INS_pmovzxbq, INS_pmovsxwq, INS_pmovzxwq, INS_pmovsxdq, INS_pmovzxdq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_MaybeMemoryLoad) +HARDWARE_INTRINSIC(X86Base, ConvertToVector128Single, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtdq2ps, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtpd2ps, INS_invalid, 4, 5, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, Crc32, 0, 2, INS_invalid, INS_crc32, INS_invalid, INS_crc32, INS_invalid, INS_crc32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(X86Base, DivRem, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_idiv, INS_div, INS_idiv, INS_div, INS_invalid, INS_invalid, INS_invalid, 25, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_BaseTypeFromSecondArg|HW_Flag_MultiReg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(X86Base, Divide, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_divps, INS_divpd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(X86Base, DivideScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_divss, INS_divsd, INS_invalid, -1, -1, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(X86Base, DotProduct, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_dpps, INS_dppd, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, Extract, 16, 2, INS_pextrb, INS_pextrb, INS_pextrw, INS_pextrw, INS_pextrd, INS_pextrd, INS_invalid, INS_invalid, INS_extractps, INS_invalid, INS_invalid, 4, 4, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsScalarT) +HARDWARE_INTRINSIC(X86Base, Floor, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, INS_invalid, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, FloorScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundss, INS_roundsd, INS_invalid, -1, 8, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(X86Base, HorizontalAdd, 16, 2, INS_invalid, INS_invalid, INS_phaddw, INS_phaddw, INS_phaddd, INS_phaddd, INS_invalid, INS_invalid, INS_haddps, INS_haddpd, INS_invalid, 3, 6, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, HorizontalAddSaturate, 16, 2, INS_invalid, INS_invalid, INS_phaddsw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, HorizontalSubtract, 16, 2, INS_invalid, INS_invalid, INS_phsubw, INS_invalid, INS_phsubd, INS_invalid, INS_invalid, INS_invalid, INS_hsubps, INS_hsubpd, INS_invalid, 3, 6, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, HorizontalSubtractSaturate, 16, 2, INS_invalid, INS_invalid, INS_phsubsw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, Insert, 16, 3, INS_pinsrb, INS_pinsrb, INS_pinsrw, INS_pinsrw, INS_pinsrd, INS_pinsrd, INS_invalid, INS_invalid, INS_insertps, INS_invalid, INS_invalid, 4, 1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(X86Base, LoadAlignedVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movaps, INS_movapd, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoRMWSemantics|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(X86Base, LoadAlignedVector128NonTemporal, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movntdqa, INS_movntdqa, INS_movntdqa, INS_movntdqa, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoRMWSemantics|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(X86Base, LoadAndDuplicateToVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movddup, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, LoadDquVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_lddqu, INS_lddqu, INS_lddqu, INS_lddqu, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoRMWSemantics|HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(X86Base, LoadFence, 0, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Barrier) +HARDWARE_INTRINSIC(X86Base, LoadHigh, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movhps, INS_movhpd, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, LoadLow, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movlps, INS_movlpd, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, LoadScalarVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_movd32, INS_movq, INS_movq, INS_movss, INS_movsd_simd, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, LoadVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(X86Base, MaskMove, 16, 3, INS_maskmovdqu, INS_maskmovdqu, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(X86Base, Max, 16, 2, INS_pmaxsb, INS_pmaxub, INS_pmaxsw, INS_pmaxuw, INS_pmaxsd, INS_pmaxud, INS_invalid, INS_invalid, INS_maxps, INS_maxpd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative) +HARDWARE_INTRINSIC(X86Base, MaxScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_maxss, INS_maxsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(X86Base, MemoryFence, 0, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Barrier) +HARDWARE_INTRINSIC(X86Base, Min, 16, 2, INS_pminsb, INS_pminub, INS_pminsw, INS_pminuw, INS_pminsd, INS_pminud, INS_invalid, INS_invalid, INS_minps, INS_minpd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative) +HARDWARE_INTRINSIC(X86Base, MinHorizontal, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_phminposuw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, MinScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_minss, INS_minsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(X86Base, MoveAndDuplicate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movddup, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, MoveHighAndDuplicate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movshdup, INS_invalid, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, MoveHighToLow, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movhlps, INS_invalid, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment) +HARDWARE_INTRINSIC(X86Base, MoveLowAndDuplicate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movsldup, INS_invalid, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, MoveLowToHigh, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movlhps, INS_invalid, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment) +HARDWARE_INTRINSIC(X86Base, MoveMask, 16, 1, INS_pmovmskb, INS_pmovmskb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movmskps, INS_movmskpd, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, MoveScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movq, INS_movq, INS_movss, INS_movsd_simd, INS_invalid, 1, 1, HW_Category_SIMDScalar, HW_Flag_NoContainment) +HARDWARE_INTRINSIC(X86Base, MultipleSumAbsoluteDifferences, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_mpsadbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, Multiply, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pmuldq, INS_pmuludq, INS_mulps, INS_mulpd, INS_invalid, 5, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(X86Base, MultiplyAddAdjacent, 16, 2, INS_invalid, INS_invalid, INS_pmaddubsw, INS_invalid, INS_pmaddwd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative) +HARDWARE_INTRINSIC(X86Base, MultiplyHigh, 16, 2, INS_invalid, INS_invalid, INS_pmulhw, INS_pmulhuw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(X86Base, MultiplyHighRoundScale, 16, 2, INS_invalid, INS_invalid, INS_pmulhrsw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(X86Base, MultiplyLow, 16, 2, INS_invalid, INS_invalid, INS_pmullw, INS_pmullw, INS_pmulld, INS_pmulld, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(X86Base, MultiplyScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mulss, INS_mulsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(X86Base, Or, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pord, INS_pord, INS_pord, INS_pord, INS_orps, INS_orpd, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(X86Base, PackSignedSaturate, 16, 2, INS_packsswb, INS_invalid, INS_packssdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(X86Base, PackUnsignedSaturate, 16, 2, INS_invalid, INS_packuswb, INS_invalid, INS_packusdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(X86Base, Pause, 0, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(X86Base, PopCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_popcnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(X86Base, Prefetch0, 0, 1, INS_invalid, INS_prefetcht0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(X86Base, Prefetch1, 0, 1, INS_invalid, INS_prefetcht1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(X86Base, Prefetch2, 0, 1, INS_invalid, INS_prefetcht2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(X86Base, PrefetchNonTemporal, 0, 1, INS_invalid, INS_prefetchnta, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(X86Base, Reciprocal, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rcpps, INS_invalid, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, ReciprocalScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rcpss, INS_invalid, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, ReciprocalSqrt, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rsqrtps, INS_invalid, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, ReciprocalSqrtScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rsqrtss, INS_invalid, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, RoundCurrentDirection, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, INS_invalid, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, RoundCurrentDirectionScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundss, INS_roundsd, INS_invalid, -1, 8, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(X86Base, RoundToNearestInteger, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, INS_invalid, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, RoundToNearestIntegerScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundss, INS_roundsd, INS_invalid, -1, 8, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(X86Base, RoundToNegativeInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, INS_invalid, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, RoundToNegativeInfinityScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundss, INS_roundsd, INS_invalid, -1, 8, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(X86Base, RoundToPositiveInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, INS_invalid, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, RoundToPositiveInfinityScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundss, INS_roundsd, INS_invalid, -1, 8, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(X86Base, RoundToZero, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, INS_invalid, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, RoundToZeroScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundss, INS_roundsd, INS_invalid, -1, 8, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(X86Base, ShiftLeftLogical, 16, 2, INS_invalid, INS_invalid, INS_psllw, INS_psllw, INS_pslld, INS_pslld, INS_psllq, INS_psllq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(X86Base, ShiftLeftLogical128BitLane, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pslldq, INS_pslldq, INS_pslldq, INS_pslldq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(X86Base, ShiftRightArithmetic, 16, 2, INS_invalid, INS_invalid, INS_psraw, INS_invalid, INS_psrad, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(X86Base, ShiftRightLogical, 16, 2, INS_invalid, INS_invalid, INS_psrlw, INS_psrlw, INS_psrld, INS_psrld, INS_psrlq, INS_psrlq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(X86Base, ShiftRightLogical128BitLane, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_psrldq, INS_psrldq, INS_psrldq, INS_psrldq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(X86Base, Shuffle, 16, -1, INS_pshufb, INS_pshufb, INS_invalid, INS_invalid, INS_pshufd, INS_pshufd, INS_invalid, INS_invalid, INS_shufps, INS_shufpd, INS_invalid, 1, 1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(X86Base, ShuffleHigh, 16, 2, INS_invalid, INS_invalid, INS_pshufhw, INS_pshufhw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(X86Base, ShuffleLow, 16, 2, INS_invalid, INS_invalid, INS_pshuflw, INS_pshuflw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(X86Base, Sign, 16, 2, INS_psignb, INS_invalid, INS_psignw, INS_invalid, INS_psignd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, Sqrt, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqrtps, INS_sqrtpd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, SqrtScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqrtss, INS_sqrtsd, INS_invalid, -1, -1, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(X86Base, Store, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(X86Base, StoreAligned, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movaps, INS_movapd, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromSecondArg|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(X86Base, StoreAlignedNonTemporal, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movntdq, INS_movntdq, INS_movntdq, INS_movntdq, INS_movntps, INS_movntpd, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromSecondArg|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(X86Base, StoreFence, 0, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Barrier) +HARDWARE_INTRINSIC(X86Base, StoreHigh, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movhps, INS_movhpd, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(X86Base, StoreLow, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movlps, INS_movlpd, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(X86Base, StoreNonTemporal, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movnti32, INS_movnti32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(X86Base, StoreScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_movd32, INS_movq, INS_movq, INS_movss, INS_movsd_simd, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(X86Base, Subtract, 16, 2, INS_psubb, INS_psubb, INS_psubw, INS_psubw, INS_psubd, INS_psubd, INS_psubq, INS_psubq, INS_subps, INS_subpd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(X86Base, SubtractSaturate, 16, 2, INS_psubsb, INS_psubusb, INS_psubsw, INS_psubusw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(X86Base, SubtractScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_subss, INS_subsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(X86Base, SumAbsoluteDifferences, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_psadbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(X86Base, TestC, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(X86Base, TestNotZAndNotC, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(X86Base, TestZ, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(X86Base, UnpackHigh, 16, 2, INS_punpckhbw, INS_punpckhbw, INS_punpckhwd, INS_punpckhwd, INS_punpckhdq, INS_punpckhdq, INS_punpckhqdq, INS_punpckhqdq, INS_unpckhps, INS_unpckhpd, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(X86Base, UnpackLow, 16, 2, INS_punpcklbw, INS_punpcklbw, INS_punpcklwd, INS_punpcklwd, INS_punpckldq, INS_punpckldq, INS_punpcklqdq, INS_punpcklqdq, INS_unpcklps, INS_unpcklpd, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(X86Base, Xor, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pxord, INS_pxord, INS_pxord, INS_pxord, INS_xorps, INS_xorpd, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp|HW_Flag_NormalizeSmallTypeToInt) #define LAST_NI_X86Base NI_X86Base_Xor // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -195,98 +195,98 @@ HARDWARE_INTRINSIC(X86Base, Xor, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // 64-bit only Intrinsics for X86Base, SSE, SSE2, SSE3, SSSE3, SSE41, SSE42, POPCNT #define FIRST_NI_X86Base_X64 NI_X86Base_X64_BigMul -HARDWARE_INTRINSIC(X86Base_X64, BigMul, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_imulEAX, INS_mulEAX, INS_invalid, INS_invalid, 4, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_BaseTypeFromSecondArg|HW_Flag_MultiReg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_RmwIntrinsic|HW_Flag_Commutative) -HARDWARE_INTRINSIC(X86Base_X64, BitScanForward, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bsf, INS_bsf, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base_X64, BitScanReverse, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bsr, INS_bsr, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base_X64, ConvertScalarToVector128Double, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2sd64, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(X86Base_X64, ConvertScalarToVector128Int64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd64, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(X86Base_X64, ConvertScalarToVector128Single, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2ss64, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_CopyUpperBits|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(X86Base_X64, ConvertScalarToVector128UInt64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd64, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(X86Base_X64, ConvertToInt64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd64, INS_invalid, INS_cvtss2si64, INS_cvtsd2si64, 3, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(X86Base_X64, ConvertToInt64WithTruncation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvttss2si64, INS_cvttsd2si64, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(X86Base_X64, ConvertToUInt64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd64, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(X86Base_X64, Crc32, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_crc32, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(X86Base_X64, DivRem, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_idiv, INS_div, INS_invalid, INS_invalid, 57, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_BaseTypeFromSecondArg|HW_Flag_MultiReg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(X86Base_X64, Extract, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pextrq, INS_pextrq, INS_invalid, INS_invalid, 4, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsScalarT) -HARDWARE_INTRINSIC(X86Base_X64, Insert, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pinsrq, INS_pinsrq, INS_invalid, INS_invalid, 4, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(X86Base_X64, PopCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_popcnt, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(X86Base_X64, StoreNonTemporal, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movnti64, INS_movnti64, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(X86Base_X64, BigMul, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_imulEAX, INS_mulEAX, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_BaseTypeFromSecondArg|HW_Flag_MultiReg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_RmwIntrinsic|HW_Flag_Commutative) +HARDWARE_INTRINSIC(X86Base_X64, BitScanForward, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bsf, INS_bsf, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base_X64, BitScanReverse, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bsr, INS_bsr, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base_X64, ConvertScalarToVector128Double, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2sd64, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(X86Base_X64, ConvertScalarToVector128Int64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd64, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(X86Base_X64, ConvertScalarToVector128Single, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2ss64, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_CopyUpperBits|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(X86Base_X64, ConvertScalarToVector128UInt64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd64, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(X86Base_X64, ConvertToInt64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd64, INS_invalid, INS_cvtss2si64, INS_cvtsd2si64, INS_invalid, 3, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(X86Base_X64, ConvertToInt64WithTruncation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvttss2si64, INS_cvttsd2si64, INS_invalid, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(X86Base_X64, ConvertToUInt64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd64, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(X86Base_X64, Crc32, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_crc32, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(X86Base_X64, DivRem, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_idiv, INS_div, INS_invalid, INS_invalid, INS_invalid, 57, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_BaseTypeFromSecondArg|HW_Flag_MultiReg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(X86Base_X64, Extract, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pextrq, INS_pextrq, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsScalarT) +HARDWARE_INTRINSIC(X86Base_X64, Insert, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pinsrq, INS_pinsrq, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(X86Base_X64, PopCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_popcnt, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(X86Base_X64, StoreNonTemporal, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movnti64, INS_movnti64, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromSecondArg) #define LAST_NI_X86Base_X64 NI_X86Base_X64_StoreNonTemporal // Intrinsics for AVX #define FIRST_NI_AVX NI_AVX_Add -HARDWARE_INTRINSIC(AVX, Add, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_addps, INS_addpd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX, AddSubtract, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_addsubps, INS_addsubpd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX, And, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_andps, INS_andpd, -1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX, AndNot, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_andnps, INS_andnpd, -1, 1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, Blend, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blendps, INS_blendpd, -1, 1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX, BlendVariable, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vblendvps, INS_vblendvpd, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, BroadcastScalarToVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcastss, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, BroadcastScalarToVector256, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcastss, INS_vbroadcastsd, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, BroadcastVector128ToVector256, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcastf32x4, INS_vbroadcastf32x4, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, Ceiling, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, Compare, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_IMM, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, CompareEqual, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, CompareGreaterThan, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, CompareGreaterThanOrEqual, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, CompareLessThan, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, CompareLessThanOrEqual, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, CompareNotEqual, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, CompareNotGreaterThan, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, CompareNotGreaterThanOrEqual, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, CompareNotLessThan, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, CompareNotLessThanOrEqual, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, CompareOrdered, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, CompareScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, -1, 4, HW_Category_IMM, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, CompareUnordered, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, ConvertToVector128Int32, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtpd2dq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtpd2dq, 7, 7, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, ConvertToVector128Int32WithTruncation, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvttpd2dq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 7, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, ConvertToVector128Single, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtpd2ps, -1, 7, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX, ConvertToVector256Double, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtdq2pd, INS_invalid, INS_invalid, INS_invalid, INS_cvtps2pd, INS_invalid, 7, 7, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX, ConvertToVector256Int32, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtps2dq, INS_invalid, INS_invalid, INS_invalid, INS_cvtps2dq, INS_invalid, 4, 4, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, ConvertToVector256Int32WithTruncation, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvttps2dq, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX, ConvertToVector256Single, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtdq2ps, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX, Divide, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_divps, INS_divpd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, DotProduct, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_dpps, INS_invalid, -1, 13, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX, DuplicateEvenIndexed, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movsldup, INS_movddup, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, DuplicateOddIndexed, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movshdup, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, ExtractVector128, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vextractf32x4, INS_vextractf32x4, INS_vextractf32x4, INS_vextractf32x4, INS_vextractf32x4, INS_vextractf32x4, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX, Floor, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, HorizontalAdd, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_haddps, INS_haddpd, -1, 6, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX, HorizontalSubtract, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_hsubps, INS_hsubpd, -1, 6, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX, InsertVector128, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vinsertf32x4, INS_vinsertf32x4, INS_vinsertf32x4, INS_vinsertf32x4, INS_vinsertf32x4, INS_vinsertf32x4, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX, LoadAlignedVector256, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movaps, INS_movapd, -1, -1, HW_Category_MemoryLoad, HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX, LoadDquVector256, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_lddqu, INS_lddqu, INS_lddqu, INS_lddqu, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX, LoadVector256, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX, MaskLoad, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vmaskmovps, INS_vmaskmovpd, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX, MaskStore, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vmaskmovps, INS_vmaskmovpd, -1, -1, HW_Category_MemoryStore, HW_Flag_NoContainment|HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX, Max, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_maxps, INS_maxpd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative) -HARDWARE_INTRINSIC(AVX, Min, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_minps, INS_minpd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative) -HARDWARE_INTRINSIC(AVX, MoveMask, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movmskps, INS_movmskpd, -1, 5, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX, Multiply, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mulps, INS_mulpd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX, Or, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_orps, INS_orpd, -1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX, Permute, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermilps, INS_vpermilpd, -1, 1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX, Permute2x128, 32, 3, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX, PermuteVar, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermilpsvar, INS_vpermilpdvar, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, Reciprocal, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rcpps, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX, ReciprocalSqrt, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rsqrtps, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX, RoundCurrentDirection, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, RoundToNearestInteger, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, RoundToNegativeInfinity, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, RoundToPositiveInfinity, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, RoundToZero, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, Shuffle, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_shufps, INS_shufpd, -1, 1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX, Sqrt, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqrtps, INS_sqrtpd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, Store, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(AVX, StoreAligned, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movaps, INS_movapd, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX, StoreAlignedNonTemporal, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movntdq, INS_movntdq, INS_movntdq, INS_movntdq, INS_movntps, INS_movntpd, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX, Subtract, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_subps, INS_subpd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, TestC, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_vtestps, INS_vtestpd, 6, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(AVX, TestNotZAndNotC, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_vtestps, INS_vtestpd, 6, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(AVX, TestZ, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_vtestps, INS_vtestpd, 6, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(AVX, UnpackHigh, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_unpckhps, INS_unpckhpd, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, UnpackLow, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_unpcklps, INS_unpcklpd, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, Xor, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_xorps, INS_xorpd, -1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AVX, Add, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_addps, INS_addpd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX, AddSubtract, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_addsubps, INS_addsubpd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX, And, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_andps, INS_andpd, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX, AndNot, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_andnps, INS_andnpd, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, Blend, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blendps, INS_blendpd, INS_invalid, -1, 1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX, BlendVariable, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vblendvps, INS_vblendvpd, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, BroadcastScalarToVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcastss, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, BroadcastScalarToVector256, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcastss, INS_vbroadcastsd, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, BroadcastVector128ToVector256, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcastf32x4, INS_vbroadcastf32x4, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, Ceiling, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, INS_invalid, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, Compare, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_IMM, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, CompareEqual, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, CompareGreaterThan, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, CompareGreaterThanOrEqual, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, CompareLessThan, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, CompareLessThanOrEqual, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, CompareNotEqual, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, CompareNotGreaterThan, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, CompareNotGreaterThanOrEqual, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, CompareNotLessThan, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, CompareNotLessThanOrEqual, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, CompareOrdered, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, CompareScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, INS_invalid, -1, 4, HW_Category_IMM, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, CompareUnordered, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, ConvertToVector128Int32, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtpd2dq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtpd2dq, INS_invalid, 7, 7, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, ConvertToVector128Int32WithTruncation, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvttpd2dq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 7, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, ConvertToVector128Single, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtpd2ps, INS_invalid, -1, 7, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX, ConvertToVector256Double, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtdq2pd, INS_invalid, INS_invalid, INS_invalid, INS_cvtps2pd, INS_invalid, INS_invalid, 7, 7, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX, ConvertToVector256Int32, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtps2dq, INS_invalid, INS_invalid, INS_invalid, INS_cvtps2dq, INS_invalid, INS_invalid, 4, 4, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, ConvertToVector256Int32WithTruncation, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvttps2dq, INS_invalid, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX, ConvertToVector256Single, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtdq2ps, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX, Divide, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_divps, INS_divpd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, DotProduct, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_dpps, INS_invalid, INS_invalid, -1, 13, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX, DuplicateEvenIndexed, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movsldup, INS_movddup, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, DuplicateOddIndexed, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movshdup, INS_invalid, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, ExtractVector128, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vextractf32x4, INS_vextractf32x4, INS_vextractf32x4, INS_vextractf32x4, INS_vextractf32x4, INS_vextractf32x4, INS_invalid, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX, Floor, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, INS_invalid, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, HorizontalAdd, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_haddps, INS_haddpd, INS_invalid, -1, 6, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX, HorizontalSubtract, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_hsubps, INS_hsubpd, INS_invalid, -1, 6, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX, InsertVector128, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vinsertf32x4, INS_vinsertf32x4, INS_vinsertf32x4, INS_vinsertf32x4, INS_vinsertf32x4, INS_vinsertf32x4, INS_invalid, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX, LoadAlignedVector256, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movaps, INS_movapd, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX, LoadDquVector256, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_lddqu, INS_lddqu, INS_lddqu, INS_lddqu, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX, LoadVector256, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX, MaskLoad, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vmaskmovps, INS_vmaskmovpd, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX, MaskStore, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vmaskmovps, INS_vmaskmovpd, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoContainment|HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX, Max, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_maxps, INS_maxpd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative) +HARDWARE_INTRINSIC(AVX, Min, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_minps, INS_minpd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative) +HARDWARE_INTRINSIC(AVX, MoveMask, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movmskps, INS_movmskpd, INS_invalid, -1, 5, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX, Multiply, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mulps, INS_mulpd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX, Or, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_orps, INS_orpd, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX, Permute, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermilps, INS_vpermilpd, INS_invalid, -1, 1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX, Permute2x128, 32, 3, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, INS_invalid, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX, PermuteVar, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermilpsvar, INS_vpermilpdvar, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, Reciprocal, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rcpps, INS_invalid, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX, ReciprocalSqrt, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rsqrtps, INS_invalid, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX, RoundCurrentDirection, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, INS_invalid, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, RoundToNearestInteger, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, INS_invalid, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, RoundToNegativeInfinity, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, INS_invalid, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, RoundToPositiveInfinity, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, INS_invalid, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, RoundToZero, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, INS_invalid, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, Shuffle, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_shufps, INS_shufpd, INS_invalid, -1, 1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX, Sqrt, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqrtps, INS_sqrtpd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, Store, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(AVX, StoreAligned, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movaps, INS_movapd, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX, StoreAlignedNonTemporal, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movntdq, INS_movntdq, INS_movntdq, INS_movntdq, INS_movntps, INS_movntpd, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX, Subtract, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_subps, INS_subpd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, TestC, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_vtestps, INS_vtestpd, INS_invalid, 6, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(AVX, TestNotZAndNotC, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_vtestps, INS_vtestpd, INS_invalid, 6, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(AVX, TestZ, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_vtestps, INS_vtestpd, INS_invalid, 6, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(AVX, UnpackHigh, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_unpckhps, INS_unpckhpd, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, UnpackLow, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_unpcklps, INS_unpcklpd, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, Xor, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_xorps, INS_xorpd, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp) #define LAST_NI_AVX NI_AVX_Xor // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -295,96 +295,96 @@ HARDWARE_INTRINSIC(AVX, Xor, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Intrinsics for AVX2, BMI1, BMI2, F16C, FMA, LZCNT, MOVBE #define FIRST_NI_AVX2 NI_AVX2_Abs -HARDWARE_INTRINSIC(AVX2, Abs, 32, 1, INS_pabsb, INS_invalid, INS_pabsw, INS_invalid, INS_pabsd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX2, Add, 32, 2, INS_paddb, INS_paddb, INS_paddw, INS_paddw, INS_paddd, INS_paddd, INS_paddq, INS_paddq, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX2, AddSaturate, 32, 2, INS_paddsb, INS_paddusb, INS_paddsw, INS_paddusw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX2, AlignRight, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_palignr, INS_palignr, INS_palignr, INS_palignr, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX2, And, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pandd, INS_pandd, INS_pandd, INS_pandd, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX2, AndNot, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_InvalidNodeId|HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(AVX2, Average, 32, 2, INS_invalid, INS_pavgb, INS_invalid, INS_pavgw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX2, BitFieldExtract, 0, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bextr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 2, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, Blend, -1, 3, INS_invalid, INS_invalid, INS_pblendw, INS_pblendw, INS_vpblendd, INS_vpblendd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, BlendVariable, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpblendvb, INS_vpblendvb, INS_vpblendvb, INS_vpblendvb, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX2, BroadcastScalarToVector128, 16, 1, INS_vpbroadcastb, INS_vpbroadcastb, INS_vpbroadcastw, INS_vpbroadcastw, INS_vpbroadcastd, INS_vpbroadcastd, INS_vpbroadcastq, INS_vpbroadcastq, INS_vbroadcastss, INS_movddup, 1, 1, HW_Category_SIMDScalar, HW_Flag_MaybeMemoryLoad) -HARDWARE_INTRINSIC(AVX2, BroadcastScalarToVector256, 32, 1, INS_vpbroadcastb, INS_vpbroadcastb, INS_vpbroadcastw, INS_vpbroadcastw, INS_vpbroadcastd, INS_vpbroadcastd, INS_vpbroadcastq, INS_vpbroadcastq, INS_vbroadcastss, INS_vbroadcastsd, 3, 3, HW_Category_SIMDScalar, HW_Flag_MaybeMemoryLoad) -HARDWARE_INTRINSIC(AVX2, BroadcastVector128ToVector256, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcasti32x4, INS_vbroadcasti32x4, INS_vbroadcasti32x4, INS_vbroadcasti32x4, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX2, CompareEqual, 32, 2, INS_pcmpeqb, INS_pcmpeqb, INS_pcmpeqw, INS_pcmpeqw, INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqq, INS_pcmpeqq, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX2, CompareGreaterThan, 32, 2, INS_pcmpgtb, INS_invalid, INS_pcmpgtw, INS_invalid, INS_pcmpgtd, INS_invalid, INS_pcmpgtq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX2, CompareLessThan, 32, 2, INS_pcmpgtb, INS_invalid, INS_pcmpgtw, INS_invalid, INS_pcmpgtd, INS_invalid, INS_pcmpgtq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX2, ConvertToInt32, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX2, ConvertToUInt32, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX2, ConvertToVector128Half, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2ph, INS_invalid, -1, 5, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX2, ConvertToVector128Single, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_vcvtph2ps, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX2, ConvertToVector256Half, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2ph, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX2, ConvertToVector256Int16, 32, 1, INS_pmovsxbw, INS_pmovzxbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_MaybeMemoryLoad) -HARDWARE_INTRINSIC(AVX2, ConvertToVector256Int32, 32, 1, INS_pmovsxbd, INS_pmovzxbd, INS_pmovsxwd, INS_pmovzxwd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_MaybeMemoryLoad) -HARDWARE_INTRINSIC(AVX2, ConvertToVector256Int64, 32, 1, INS_pmovsxbq, INS_pmovzxbq, INS_pmovsxwq, INS_pmovzxwq, INS_pmovsxdq, INS_pmovzxdq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_MaybeMemoryLoad) -HARDWARE_INTRINSIC(AVX2, ConvertToVector256Single, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_vcvtph2ps, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 7, 7, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX2, ExtractLowestSetBit, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blsi, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, ExtractVector128, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vextracti32x4, INS_vextracti32x4, INS_vextracti32x4, INS_vextracti32x4, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX2, GatherMaskVector128, 16, 5, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpgatherdd, INS_vpgatherdd, INS_vpgatherdq, INS_vpgatherdq, INS_vgatherdps, INS_vgatherdpd, -1, -1, HW_Category_IMM, HW_Flag_MaybeMemoryLoad|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_NoContainment|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, GatherMaskVector256, 32, 5, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpgatherdd, INS_vpgatherdd, INS_vpgatherdq, INS_vpgatherdq, INS_vgatherdps, INS_vgatherdpd, -1, -1, HW_Category_IMM, HW_Flag_MaybeMemoryLoad|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_NoContainment|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, GatherVector128, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpgatherdd, INS_vpgatherdd, INS_vpgatherdq, INS_vpgatherdq, INS_vgatherdps, INS_vgatherdpd, -1, -1, HW_Category_IMM, HW_Flag_MaybeMemoryLoad|HW_Flag_SpecialCodeGen|HW_Flag_NoContainment|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, GatherVector256, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpgatherdd, INS_vpgatherdd, INS_vpgatherdq, INS_vpgatherdq, INS_vgatherdps, INS_vgatherdpd, -1, -1, HW_Category_IMM, HW_Flag_MaybeMemoryLoad|HW_Flag_SpecialCodeGen|HW_Flag_NoContainment|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, GetMaskUpToLowestSetBit, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blsmsk, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(AVX2, HorizontalAdd, 32, 2, INS_invalid, INS_invalid, INS_phaddw, INS_phaddw, INS_phaddd, INS_phaddd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, HorizontalAddSaturate, 32, 2, INS_invalid, INS_invalid, INS_phaddsw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, HorizontalSubtract, 32, 2, INS_invalid, INS_invalid, INS_phsubw, INS_invalid, INS_phsubd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, HorizontalSubtractSaturate, 32, 2, INS_invalid, INS_invalid, INS_phsubsw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, InsertVector128, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vinserti32x4, INS_vinserti32x4, INS_vinserti32x4, INS_vinserti32x4, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX2, LeadingZeroCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_lzcnt, INS_lzcnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX2, LoadAlignedVector256NonTemporal, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movntdqa, INS_movntdqa, INS_movntdqa, INS_movntdqa, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX2, MaskLoad, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmaskmovd, INS_vpmaskmovd, INS_vpmaskmovq, INS_vpmaskmovq, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, MaskStore, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmaskmovd, INS_vpmaskmovd, INS_vpmaskmovq, INS_vpmaskmovq, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoContainment|HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, Max, 32, 2, INS_pmaxsb, INS_pmaxub, INS_pmaxsw, INS_pmaxuw, INS_pmaxsd, INS_pmaxud, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX2, Min, 32, 2, INS_pminsb, INS_pminub, INS_pminsw, INS_pminuw, INS_pminsd, INS_pminud, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX2, MoveMask, 32, 1, INS_pmovmskb, INS_pmovmskb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, MultipleSumAbsoluteDifferences, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_mpsadbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, Multiply, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pmuldq, INS_pmuludq, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX2, MultiplyAdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmadd213ps, INS_vfmadd213pd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX2, MultiplyAddAdjacent, 32, 2, INS_invalid, INS_invalid, INS_pmaddubsw, INS_invalid, INS_pmaddwd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX2, MultiplyAddNegated, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmadd213ps, INS_vfnmadd213pd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX2, MultiplyAddNegatedScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmadd213ss, INS_vfnmadd213sd, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX2, MultiplyAddScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmadd213ss, INS_vfmadd213sd, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX2, MultiplyAddSubtract, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmaddsub213ps, INS_vfmaddsub213pd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX2, MultiplyHigh, 32, 2, INS_invalid, INS_invalid, INS_pmulhw, INS_pmulhuw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX2, MultiplyHighRoundScale, 32, 2, INS_invalid, INS_invalid, INS_pmulhrsw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX2, MultiplyLow, 32, 2, INS_invalid, INS_invalid, INS_pmullw, INS_pmullw, INS_pmulld, INS_pmulld, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX2, MultiplyNoFlags, 0, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mulx, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_Scalar, HW_Flag_NoContainment|HW_Flag_MaybeMemoryStore|HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics|HW_Flag_MaybeCommutative) -HARDWARE_INTRINSIC(AVX2, MultiplySubtract, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmsub213ps, INS_vfmsub213pd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX2, MultiplySubtractAdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmsubadd213ps, INS_vfmsubadd213pd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX2, MultiplySubtractNegated, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmsub213ps, INS_vfnmsub213pd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX2, MultiplySubtractNegatedScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmsub213ss, INS_vfnmsub213sd, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX2, MultiplySubtractScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmsub213ss, INS_vfmsub213sd, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX2, Or, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pord, INS_pord, INS_pord, INS_pord, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX2, PackSignedSaturate, 32, 2, INS_packsswb, INS_invalid, INS_packssdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX2, PackUnsignedSaturate, 32, 2, INS_invalid, INS_packuswb, INS_invalid, INS_packusdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX2, ParallelBitDeposit, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pdep, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, ParallelBitExtract, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pext, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, Permute2x128, 32, 3, INS_vperm2i128, INS_vperm2i128, INS_vperm2i128, INS_vperm2i128, INS_vperm2i128, INS_vperm2i128, INS_vperm2i128, INS_vperm2i128, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, Permute4x64, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermq, INS_vpermq, INS_invalid, INS_vpermpd, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX2, PermuteVar8x32, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermd, INS_vpermd, INS_invalid, INS_invalid, INS_vpermps, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX2, ResetLowestSetBit, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blsr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(AVX2, ShiftLeftLogical, 32, 2, INS_invalid, INS_invalid, INS_psllw, INS_psllw, INS_pslld, INS_pslld, INS_psllq, INS_psllq, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX2, ShiftLeftLogical128BitLane, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pslldq, INS_pslldq, INS_pslldq, INS_pslldq, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX2, ShiftLeftLogicalVariable, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpsllvd, INS_vpsllvd, INS_vpsllvq, INS_vpsllvq, INS_invalid, INS_invalid, 2, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX2, ShiftRightArithmetic, 32, 2, INS_invalid, INS_invalid, INS_psraw, INS_invalid, INS_psrad, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX2, ShiftRightArithmeticVariable, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpsravd, INS_vpsravd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 2, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX2, ShiftRightLogical, 32, 2, INS_invalid, INS_invalid, INS_psrlw, INS_psrlw, INS_psrld, INS_psrld, INS_psrlq, INS_psrlq, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX2, ShiftRightLogical128BitLane, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_psrldq, INS_psrldq, INS_psrldq, INS_psrldq, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX2, ShiftRightLogicalVariable, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpsrlvd, INS_vpsrlvd, INS_vpsrlvq, INS_vpsrlvq, INS_invalid, INS_invalid, 2, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX2, Shuffle, 32, 2, INS_pshufb, INS_pshufb, INS_invalid, INS_invalid, INS_pshufd, INS_pshufd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_MaybeIMM) -HARDWARE_INTRINSIC(AVX2, ShuffleHigh, 32, 2, INS_invalid, INS_invalid, INS_pshufhw, INS_pshufhw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX2, ShuffleLow, 32, 2, INS_invalid, INS_invalid, INS_pshuflw, INS_pshuflw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX2, Sign, 32, 2, INS_psignb, INS_invalid, INS_psignw, INS_invalid, INS_psignd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, Subtract, 32, 2, INS_psubb, INS_psubb, INS_psubw, INS_psubw, INS_psubd, INS_psubd, INS_psubq, INS_psubq, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX2, SubtractSaturate, 32, 2, INS_psubsb, INS_psubusb, INS_psubsw, INS_psubusw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX2, SumAbsoluteDifferences, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_psadbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX2, TrailingZeroCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_tzcnt, INS_tzcnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX2, UnpackHigh, 32, 2, INS_punpckhbw, INS_punpckhbw, INS_punpckhwd, INS_punpckhwd, INS_punpckhdq, INS_punpckhdq, INS_punpckhqdq, INS_punpckhqdq, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX2, UnpackLow, 32, 2, INS_punpcklbw, INS_punpcklbw, INS_punpcklwd, INS_punpcklwd, INS_punpckldq, INS_punpckldq, INS_punpcklqdq, INS_punpcklqdq, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX2, Xor, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pxord, INS_pxord, INS_pxord, INS_pxord, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX2, ZeroHighBits, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bzhi, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialImport|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, Abs, 32, 1, INS_pabsb, INS_invalid, INS_pabsw, INS_invalid, INS_pabsd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX2, Add, 32, 2, INS_paddb, INS_paddb, INS_paddw, INS_paddw, INS_paddd, INS_paddd, INS_paddq, INS_paddq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX2, AddSaturate, 32, 2, INS_paddsb, INS_paddusb, INS_paddsw, INS_paddusw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX2, AlignRight, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_palignr, INS_palignr, INS_palignr, INS_palignr, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX2, And, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pandd, INS_pandd, INS_pandd, INS_pandd, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX2, AndNot, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_InvalidNodeId|HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(AVX2, Average, 32, 2, INS_invalid, INS_pavgb, INS_invalid, INS_pavgw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX2, BitFieldExtract, 0, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bextr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 2, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, Blend, -1, 3, INS_invalid, INS_invalid, INS_pblendw, INS_pblendw, INS_vpblendd, INS_vpblendd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, BlendVariable, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpblendvb, INS_vpblendvb, INS_vpblendvb, INS_vpblendvb, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX2, BroadcastScalarToVector128, 16, 1, INS_vpbroadcastb, INS_vpbroadcastb, INS_vpbroadcastw, INS_vpbroadcastw, INS_vpbroadcastd, INS_vpbroadcastd, INS_vpbroadcastq, INS_vpbroadcastq, INS_vbroadcastss, INS_movddup, INS_invalid, 1, 1, HW_Category_SIMDScalar, HW_Flag_MaybeMemoryLoad) +HARDWARE_INTRINSIC(AVX2, BroadcastScalarToVector256, 32, 1, INS_vpbroadcastb, INS_vpbroadcastb, INS_vpbroadcastw, INS_vpbroadcastw, INS_vpbroadcastd, INS_vpbroadcastd, INS_vpbroadcastq, INS_vpbroadcastq, INS_vbroadcastss, INS_vbroadcastsd, INS_invalid, 3, 3, HW_Category_SIMDScalar, HW_Flag_MaybeMemoryLoad) +HARDWARE_INTRINSIC(AVX2, BroadcastVector128ToVector256, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcasti32x4, INS_vbroadcasti32x4, INS_vbroadcasti32x4, INS_vbroadcasti32x4, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX2, CompareEqual, 32, 2, INS_pcmpeqb, INS_pcmpeqb, INS_pcmpeqw, INS_pcmpeqw, INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqq, INS_pcmpeqq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX2, CompareGreaterThan, 32, 2, INS_pcmpgtb, INS_invalid, INS_pcmpgtw, INS_invalid, INS_pcmpgtd, INS_invalid, INS_pcmpgtq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX2, CompareLessThan, 32, 2, INS_pcmpgtb, INS_invalid, INS_pcmpgtw, INS_invalid, INS_pcmpgtd, INS_invalid, INS_pcmpgtq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX2, ConvertToInt32, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX2, ConvertToUInt32, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX2, ConvertToVector128Half, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2ph, INS_invalid, INS_invalid, -1, 5, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX2, ConvertToVector128Single, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_vcvtph2ps, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX2, ConvertToVector256Half, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2ph, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX2, ConvertToVector256Int16, 32, 1, INS_pmovsxbw, INS_pmovzxbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_MaybeMemoryLoad) +HARDWARE_INTRINSIC(AVX2, ConvertToVector256Int32, 32, 1, INS_pmovsxbd, INS_pmovzxbd, INS_pmovsxwd, INS_pmovzxwd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_MaybeMemoryLoad) +HARDWARE_INTRINSIC(AVX2, ConvertToVector256Int64, 32, 1, INS_pmovsxbq, INS_pmovzxbq, INS_pmovsxwq, INS_pmovzxwq, INS_pmovsxdq, INS_pmovzxdq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_MaybeMemoryLoad) +HARDWARE_INTRINSIC(AVX2, ConvertToVector256Single, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_vcvtph2ps, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 7, 7, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX2, ExtractLowestSetBit, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blsi, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, ExtractVector128, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vextracti32x4, INS_vextracti32x4, INS_vextracti32x4, INS_vextracti32x4, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX2, GatherMaskVector128, 16, 5, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpgatherdd, INS_vpgatherdd, INS_vpgatherdq, INS_vpgatherdq, INS_vgatherdps, INS_vgatherdpd, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_MaybeMemoryLoad|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_NoContainment|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, GatherMaskVector256, 32, 5, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpgatherdd, INS_vpgatherdd, INS_vpgatherdq, INS_vpgatherdq, INS_vgatherdps, INS_vgatherdpd, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_MaybeMemoryLoad|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_NoContainment|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, GatherVector128, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpgatherdd, INS_vpgatherdd, INS_vpgatherdq, INS_vpgatherdq, INS_vgatherdps, INS_vgatherdpd, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_MaybeMemoryLoad|HW_Flag_SpecialCodeGen|HW_Flag_NoContainment|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, GatherVector256, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpgatherdd, INS_vpgatherdd, INS_vpgatherdq, INS_vpgatherdq, INS_vgatherdps, INS_vgatherdpd, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_MaybeMemoryLoad|HW_Flag_SpecialCodeGen|HW_Flag_NoContainment|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, GetMaskUpToLowestSetBit, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blsmsk, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(AVX2, HorizontalAdd, 32, 2, INS_invalid, INS_invalid, INS_phaddw, INS_phaddw, INS_phaddd, INS_phaddd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, HorizontalAddSaturate, 32, 2, INS_invalid, INS_invalid, INS_phaddsw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, HorizontalSubtract, 32, 2, INS_invalid, INS_invalid, INS_phsubw, INS_invalid, INS_phsubd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, HorizontalSubtractSaturate, 32, 2, INS_invalid, INS_invalid, INS_phsubsw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, InsertVector128, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vinserti32x4, INS_vinserti32x4, INS_vinserti32x4, INS_vinserti32x4, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX2, LeadingZeroCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_lzcnt, INS_lzcnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX2, LoadAlignedVector256NonTemporal, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movntdqa, INS_movntdqa, INS_movntdqa, INS_movntdqa, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX2, MaskLoad, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmaskmovd, INS_vpmaskmovd, INS_vpmaskmovq, INS_vpmaskmovq, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, MaskStore, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmaskmovd, INS_vpmaskmovd, INS_vpmaskmovq, INS_vpmaskmovq, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoContainment|HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, Max, 32, 2, INS_pmaxsb, INS_pmaxub, INS_pmaxsw, INS_pmaxuw, INS_pmaxsd, INS_pmaxud, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX2, Min, 32, 2, INS_pminsb, INS_pminub, INS_pminsw, INS_pminuw, INS_pminsd, INS_pminud, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX2, MoveMask, 32, 1, INS_pmovmskb, INS_pmovmskb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, MultipleSumAbsoluteDifferences, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_mpsadbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, Multiply, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pmuldq, INS_pmuludq, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX2, MultiplyAdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmadd213ps, INS_vfmadd213pd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX2, MultiplyAddAdjacent, 32, 2, INS_invalid, INS_invalid, INS_pmaddubsw, INS_invalid, INS_pmaddwd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX2, MultiplyAddNegated, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmadd213ps, INS_vfnmadd213pd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX2, MultiplyAddNegatedScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmadd213ss, INS_vfnmadd213sd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX2, MultiplyAddScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmadd213ss, INS_vfmadd213sd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX2, MultiplyAddSubtract, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmaddsub213ps, INS_vfmaddsub213pd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX2, MultiplyHigh, 32, 2, INS_invalid, INS_invalid, INS_pmulhw, INS_pmulhuw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX2, MultiplyHighRoundScale, 32, 2, INS_invalid, INS_invalid, INS_pmulhrsw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX2, MultiplyLow, 32, 2, INS_invalid, INS_invalid, INS_pmullw, INS_pmullw, INS_pmulld, INS_pmulld, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX2, MultiplyNoFlags, 0, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mulx, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_Scalar, HW_Flag_NoContainment|HW_Flag_MaybeMemoryStore|HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics|HW_Flag_MaybeCommutative) +HARDWARE_INTRINSIC(AVX2, MultiplySubtract, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmsub213ps, INS_vfmsub213pd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX2, MultiplySubtractAdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmsubadd213ps, INS_vfmsubadd213pd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX2, MultiplySubtractNegated, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmsub213ps, INS_vfnmsub213pd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX2, MultiplySubtractNegatedScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmsub213ss, INS_vfnmsub213sd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX2, MultiplySubtractScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmsub213ss, INS_vfmsub213sd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX2, Or, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pord, INS_pord, INS_pord, INS_pord, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX2, PackSignedSaturate, 32, 2, INS_packsswb, INS_invalid, INS_packssdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX2, PackUnsignedSaturate, 32, 2, INS_invalid, INS_packuswb, INS_invalid, INS_packusdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX2, ParallelBitDeposit, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pdep, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, ParallelBitExtract, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pext, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, Permute2x128, 32, 3, INS_vperm2i128, INS_vperm2i128, INS_vperm2i128, INS_vperm2i128, INS_vperm2i128, INS_vperm2i128, INS_vperm2i128, INS_vperm2i128, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, Permute4x64, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermq, INS_vpermq, INS_invalid, INS_vpermpd, INS_invalid, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX2, PermuteVar8x32, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermd, INS_vpermd, INS_invalid, INS_invalid, INS_vpermps, INS_invalid, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX2, ResetLowestSetBit, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blsr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(AVX2, ShiftLeftLogical, 32, 2, INS_invalid, INS_invalid, INS_psllw, INS_psllw, INS_pslld, INS_pslld, INS_psllq, INS_psllq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX2, ShiftLeftLogical128BitLane, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pslldq, INS_pslldq, INS_pslldq, INS_pslldq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX2, ShiftLeftLogicalVariable, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpsllvd, INS_vpsllvd, INS_vpsllvq, INS_vpsllvq, INS_invalid, INS_invalid, INS_invalid, 2, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX2, ShiftRightArithmetic, 32, 2, INS_invalid, INS_invalid, INS_psraw, INS_invalid, INS_psrad, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX2, ShiftRightArithmeticVariable, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpsravd, INS_vpsravd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 2, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX2, ShiftRightLogical, 32, 2, INS_invalid, INS_invalid, INS_psrlw, INS_psrlw, INS_psrld, INS_psrld, INS_psrlq, INS_psrlq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX2, ShiftRightLogical128BitLane, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_psrldq, INS_psrldq, INS_psrldq, INS_psrldq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX2, ShiftRightLogicalVariable, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpsrlvd, INS_vpsrlvd, INS_vpsrlvq, INS_vpsrlvq, INS_invalid, INS_invalid, INS_invalid, 2, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX2, Shuffle, 32, 2, INS_pshufb, INS_pshufb, INS_invalid, INS_invalid, INS_pshufd, INS_pshufd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_MaybeIMM) +HARDWARE_INTRINSIC(AVX2, ShuffleHigh, 32, 2, INS_invalid, INS_invalid, INS_pshufhw, INS_pshufhw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX2, ShuffleLow, 32, 2, INS_invalid, INS_invalid, INS_pshuflw, INS_pshuflw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX2, Sign, 32, 2, INS_psignb, INS_invalid, INS_psignw, INS_invalid, INS_psignd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, Subtract, 32, 2, INS_psubb, INS_psubb, INS_psubw, INS_psubw, INS_psubd, INS_psubd, INS_psubq, INS_psubq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX2, SubtractSaturate, 32, 2, INS_psubsb, INS_psubusb, INS_psubsw, INS_psubusw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX2, SumAbsoluteDifferences, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_psadbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX2, TrailingZeroCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_tzcnt, INS_tzcnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX2, UnpackHigh, 32, 2, INS_punpckhbw, INS_punpckhbw, INS_punpckhwd, INS_punpckhwd, INS_punpckhdq, INS_punpckhdq, INS_punpckhqdq, INS_punpckhqdq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX2, UnpackLow, 32, 2, INS_punpcklbw, INS_punpcklbw, INS_punpcklwd, INS_punpcklwd, INS_punpckldq, INS_punpckldq, INS_punpcklqdq, INS_punpcklqdq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX2, Xor, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pxord, INS_pxord, INS_pxord, INS_pxord, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX2, ZeroHighBits, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bzhi, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialImport|HW_Flag_NoEvexSemantics) #define LAST_NI_AVX2 NI_AVX2_ZeroHighBits // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -393,17 +393,17 @@ HARDWARE_INTRINSIC(AVX2, ZeroHighBits, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // 64-bit only Intrinsics for AVX2, BMI1, BMI2, F16C, FMA, LZCNT, MOVBE #define FIRST_NI_AVX2_X64 NI_AVX2_X64_AndNot -HARDWARE_INTRINSIC(AVX2_X64, AndNot, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_andn, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_SpecialImport|HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2_X64, BitFieldExtract, 0, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bextr, INS_invalid, INS_invalid, INS_invalid, 2, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2_X64, ExtractLowestSetBit, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blsi, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2_X64, GetMaskUpToLowestSetBit, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blsmsk, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(AVX2_X64, LeadingZeroCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_lzcnt, INS_lzcnt, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX2_X64, MultiplyNoFlags, 0, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mulx, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_Scalar, HW_Flag_NoContainment|HW_Flag_MaybeMemoryStore|HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics|HW_Flag_MaybeCommutative) -HARDWARE_INTRINSIC(AVX2_X64, ParallelBitDeposit, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pdep, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2_X64, ParallelBitExtract, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pext, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2_X64, ResetLowestSetBit, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blsr, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(AVX2_X64, TrailingZeroCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_tzcnt, INS_tzcnt, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX2_X64, ZeroHighBits, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bzhi, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialImport|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2_X64, AndNot, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_andn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_SpecialImport|HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2_X64, BitFieldExtract, 0, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bextr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 2, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2_X64, ExtractLowestSetBit, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blsi, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2_X64, GetMaskUpToLowestSetBit, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blsmsk, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(AVX2_X64, LeadingZeroCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_lzcnt, INS_lzcnt, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX2_X64, MultiplyNoFlags, 0, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mulx, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_Scalar, HW_Flag_NoContainment|HW_Flag_MaybeMemoryStore|HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics|HW_Flag_MaybeCommutative) +HARDWARE_INTRINSIC(AVX2_X64, ParallelBitDeposit, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pdep, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2_X64, ParallelBitExtract, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pext, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2_X64, ResetLowestSetBit, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blsr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(AVX2_X64, TrailingZeroCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_tzcnt, INS_tzcnt, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX2_X64, ZeroHighBits, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bzhi, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialImport|HW_Flag_NoEvexSemantics) #define LAST_NI_AVX2_X64 NI_AVX2_X64_ZeroHighBits // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -412,206 +412,206 @@ HARDWARE_INTRINSIC(AVX2_X64, ZeroHighBits, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Intrinsics for AVX512F, AVX512BW, AVX512CD, AVX512DQ, AVX512VL #define FIRST_NI_AVX512 NI_AVX512_Abs -HARDWARE_INTRINSIC(AVX512, Abs, -1, 1, INS_pabsb, INS_invalid, INS_pabsw, INS_invalid, INS_pabsd, INS_invalid, INS_vpabsq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, Add, 64, -1, INS_paddb, INS_paddb, INS_paddw, INS_paddw, INS_paddd, INS_paddd, INS_paddq, INS_paddq, INS_addps, INS_addpd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, AddSaturate, 64, 2, INS_paddsb, INS_paddusb, INS_paddsw, INS_paddusw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX512, AddScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_addss, INS_addsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, AlignRight, 64, 3, INS_palignr, INS_palignr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, AlignRight32, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_valignd, INS_valignd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, AlignRight64, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_valignq, INS_valignq, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, And, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pandd, INS_pandd, INS_vpandq, INS_vpandq, INS_andps, INS_andpd, 1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX512, AndNot, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pandnd, INS_pandnd, INS_vpandnq, INS_vpandnq, INS_andnps, INS_andnpd, 1, 1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX512, Average, 64, 2, INS_invalid, INS_pavgb, INS_invalid, INS_pavgw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX512, BlendVariable, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, BroadcastPairScalarToVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcasti32x2, INS_vbroadcasti32x2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, BroadcastPairScalarToVector256, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcasti32x2, INS_vbroadcasti32x2, INS_invalid, INS_invalid, INS_vbroadcastf32x2, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, BroadcastPairScalarToVector512, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcasti32x2, INS_vbroadcasti32x2, INS_invalid, INS_invalid, INS_vbroadcastf32x2, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, BroadcastScalarToVector512, 64, 1, INS_vpbroadcastb, INS_vpbroadcastb, INS_vpbroadcastw, INS_vpbroadcastw, INS_vpbroadcastd, INS_vpbroadcastd, INS_vpbroadcastq, INS_vpbroadcastq, INS_vbroadcastss, INS_vbroadcastsd, 3, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, BroadcastVector128ToVector512, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcasti32x4, INS_vbroadcasti32x4, INS_vbroadcasti64x2, INS_vbroadcasti64x2, INS_vbroadcastf32x4, INS_vbroadcastf64x2, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, BroadcastVector256ToVector512, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcasti32x8, INS_vbroadcasti32x8, INS_vbroadcasti64x4, INS_vbroadcasti64x4, INS_vbroadcastf32x8, INS_vbroadcastf64x4, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, Classify, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_InvalidNodeId|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, ClassifyScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_InvalidNodeId|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, Compare, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, CompareEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, CompareGreaterThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, CompareGreaterThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, CompareLessThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, CompareLessThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, CompareNotEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, CompareNotGreaterThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, CompareNotGreaterThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, CompareNotLessThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, CompareNotLessThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, CompareOrdered, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, CompareUnordered, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, Compress, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, CompressStore, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(AVX512, ConvertScalarToVector128Double, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2sd32, INS_vcvtusi2sd32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX512, ConvertScalarToVector128Single, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2ss32, INS_vcvtusi2ss32, INS_invalid, INS_invalid, INS_invalid, INS_cvtsd2ss, 5, 4, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, ConvertToInt32, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtss2si32, INS_cvtsd2si32, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, ConvertToUInt32, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtss2usi32, INS_vcvtsd2usi32, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, ConvertToUInt32WithTruncation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttss2usi32, INS_vcvttsd2usi32, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128Byte, -1, 1, INS_invalid, INS_invalid, INS_vpmovwb, INS_vpmovwb, INS_vpmovdb, INS_vpmovdb, INS_vpmovqb, INS_vpmovqb, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128ByteWithSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_vpmovuswb, INS_invalid, INS_vpmovusdb, INS_invalid, INS_vpmovusqb, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128Double, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtudq2pd, INS_vcvtqq2pd, INS_vcvtuqq2pd, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128Int16, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovdw, INS_vpmovdw, INS_vpmovqw, INS_vpmovqw, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128Int16WithSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovsdw, INS_invalid, INS_vpmovsqw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128Int32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovqd, INS_vpmovqd, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128Int32WithSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovsqd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128Int64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2qq, INS_vcvtpd2qq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128Int64WithTruncation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2qq, INS_vcvttpd2qq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128SByte, -1, 1, INS_invalid, INS_invalid, INS_vpmovwb, INS_vpmovwb, INS_vpmovdb, INS_vpmovdb, INS_vpmovqb, INS_vpmovqb, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128SByteWithSaturation, -1, 1, INS_invalid, INS_invalid, INS_vpmovswb, INS_invalid, INS_vpmovsdb, INS_invalid, INS_vpmovsqb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128Single, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtudq2ps, INS_vcvtqq2ps, INS_vcvtuqq2ps, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128UInt16, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovdw, INS_vpmovdw, INS_vpmovqw, INS_vpmovqw, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128UInt16WithSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovusdw, INS_invalid, INS_vpmovusqw, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128UInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovqd, INS_vpmovqd, INS_vcvtps2udq, INS_vcvtpd2udq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128UInt32WithSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovusqd, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128UInt32WithTruncation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2udq, INS_vcvttpd2udq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128UInt64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2uqq, INS_vcvtpd2uqq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128UInt64WithTruncation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2uqq, INS_vcvttpd2uqq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256Byte, 64, 1, INS_invalid, INS_invalid, INS_vpmovwb, INS_vpmovwb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256ByteWithSaturation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_vpmovuswb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256Double, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtudq2pd, INS_vcvtqq2pd, INS_vcvtuqq2pd, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256Int16, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovdw, INS_vpmovdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256Int16WithSaturation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovsdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256Int32, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovqd, INS_vpmovqd, INS_invalid, INS_cvtpd2dq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256Int32WithSaturation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovsqd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256Int32WithTruncation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvttpd2dq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256Int64, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2qq, INS_vcvtpd2qq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256Int64WithTruncation, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2qq, INS_vcvttpd2qq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256SByte, 64, 1, INS_invalid, INS_invalid, INS_vpmovwb, INS_vpmovwb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256SByteWithSaturation, 64, 1, INS_invalid, INS_invalid, INS_vpmovswb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256Single, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtudq2ps, INS_vcvtqq2ps, INS_vcvtuqq2ps, INS_invalid, INS_cvtpd2ps, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256UInt16, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovdw, INS_vpmovdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256UInt16WithSaturation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovusdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256UInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovqd, INS_vpmovqd, INS_vcvtps2udq, INS_vcvtpd2udq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256UInt32WithSaturation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovusqd, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256UInt32WithTruncation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2udq, INS_vcvttpd2udq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256UInt64, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2uqq, INS_vcvtpd2uqq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256UInt64WithTruncation, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2uqq, INS_vcvttpd2uqq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector512Double, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtdq2pd, INS_vcvtudq2pd, INS_vcvtqq2pd, INS_vcvtuqq2pd, INS_cvtps2pd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, ConvertToVector512Int16, 64, 1, INS_pmovsxbw, INS_pmovzxbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector512Int32, 64, -1, INS_pmovsxbd, INS_pmovzxbd, INS_pmovsxwd, INS_pmovzxwd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtps2dq, INS_invalid, 3, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, ConvertToVector512Int32WithTruncation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvttps2dq, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector512Int64, 64, -1, INS_pmovsxbq, INS_pmovzxbq, INS_pmovsxwq, INS_pmovzxwq, INS_pmovsxdq, INS_pmovzxdq, INS_invalid, INS_invalid, INS_vcvtps2qq, INS_vcvtpd2qq, 3, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, ConvertToVector512Int64WithTruncation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2qq, INS_vcvttpd2qq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector512Single, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtdq2ps, INS_vcvtudq2ps, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, ConvertToVector512UInt16, 64, 1, INS_pmovsxbw, INS_pmovzxbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector512UInt32, 64, -1, INS_pmovsxbd, INS_pmovzxbd, INS_pmovsxwd, INS_pmovzxwd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2udq, INS_invalid, 3, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, ConvertToVector512UInt32WithTruncation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2udq, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector512UInt64, 64, -1, INS_pmovsxbq, INS_pmovzxbq, INS_pmovsxwq, INS_pmovzxwq, INS_pmovsxdq, INS_pmovzxdq, INS_invalid, INS_invalid, INS_vcvtps2uqq, INS_vcvtpd2uqq, 3, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, ConvertToVector512UInt64WithTruncation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2uqq, INS_vcvttpd2uqq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, DetectConflicts, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpconflictd, INS_vpconflictd, INS_vpconflictq, INS_vpconflictq, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, Divide, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_divps, INS_divpd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, DivideScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_divss, INS_divsd, -1, -1, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, DuplicateEvenIndexed, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movsldup, INS_movddup, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, DuplicateOddIndexed, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movshdup, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, Expand, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, ExpandLoad, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, ExtractVector128, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vextracti32x4, INS_vextracti32x4, INS_vextracti64x2, INS_vextracti64x2, INS_vextractf32x4, INS_vextractf64x2, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX512, ExtractVector256, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vextracti32x8, INS_vextracti32x8, INS_vextracti64x4, INS_vextracti64x4, INS_vextractf32x8, INS_vextractf64x4, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX512, Fixup, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfixupimmps, INS_vfixupimmpd, -1, 4, HW_Category_IMM, HW_Flag_SpecialImport|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, FixupScalar, 16, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfixupimmss, INS_vfixupimmsd, -1, 4, HW_Category_IMM, HW_Flag_SpecialImport|HW_Flag_FullRangeIMM|HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX512, FusedMultiplyAdd, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmadd213ps, INS_vfmadd213pd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, FusedMultiplyAddNegated, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmadd213ps, INS_vfnmadd213pd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, FusedMultiplyAddNegatedScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmadd213ss, INS_vfnmadd213sd, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible|HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX512, FusedMultiplyAddScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmadd213ss, INS_vfmadd213sd, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible|HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX512, FusedMultiplyAddSubtract, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmaddsub213ps, INS_vfmaddsub213pd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, FusedMultiplySubtract, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmsub213ps, INS_vfmsub213pd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, FusedMultiplySubtractAdd, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmsubadd213ps, INS_vfmsubadd213pd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, FusedMultiplySubtractNegated, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmsub213ps, INS_vfnmsub213pd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, FusedMultiplySubtractNegatedScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmsub213ss, INS_vfnmsub213sd, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible|HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX512, FusedMultiplySubtractScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmsub213ss, INS_vfmsub213sd, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible|HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX512, GetExponent, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vgetexpps, INS_vgetexppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, GetExponentScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vgetexpss, INS_vgetexpsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX512, GetMantissa, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vgetmantps, INS_vgetmantpd, -1, 4, HW_Category_IMM, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, GetMantissaScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vgetmantss, INS_vgetmantsd, -1, 4, HW_Category_IMM, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX512, InsertVector128, 64, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vinserti32x4, INS_vinserti32x4, INS_vinserti64x2, INS_vinserti64x2, INS_vinsertf32x4, INS_vinsertf64x2, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX512, InsertVector256, 64, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vinserti32x8, INS_vinserti32x8, INS_vinserti64x4, INS_vinserti64x4, INS_vinsertf32x8, INS_vinsertf64x4, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX512, LeadingZeroCount, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vplzcntd, INS_vplzcntd, INS_vplzcntq, INS_vplzcntq, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, LoadAlignedVector512, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_vmovdqa64, INS_vmovdqa64, INS_movaps, INS_movapd, -1, -1, HW_Category_MemoryLoad, HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX512, LoadAlignedVector512NonTemporal, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movntdqa, INS_movntdqa, INS_movntdqa, INS_movntdqa, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX512, LoadVector512, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, MaskLoad, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, MaskLoadAligned, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, MaskStore, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(AVX512, MaskStoreAligned, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(AVX512, Max, -1, 2, INS_pmaxsb, INS_pmaxub, INS_pmaxsw, INS_pmaxuw, INS_pmaxsd, INS_pmaxud, INS_vpmaxsq, INS_vpmaxuq, INS_maxps, INS_maxpd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative) -HARDWARE_INTRINSIC(AVX512, Min, -1, 2, INS_pminsb, INS_pminub, INS_pminsw, INS_pminuw, INS_pminsd, INS_pminud, INS_vpminsq, INS_vpminuq, INS_minps, INS_minpd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative) -HARDWARE_INTRINSIC(AVX512, MoveMask, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoContainment|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, Multiply, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pmuldq, INS_pmuludq, INS_mulps, INS_mulpd, 5, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, MultiplyAddAdjacent, 64, 2, INS_invalid, INS_invalid, INS_pmaddubsw, INS_invalid, INS_pmaddwd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, MultiplyHigh, 64, 2, INS_invalid, INS_invalid, INS_pmulhw, INS_pmulhuw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX512, MultiplyHighRoundScale, 64, 2, INS_invalid, INS_invalid, INS_pmulhrsw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, MultiplyLow, -1, 2, INS_invalid, INS_invalid, INS_pmullw, INS_pmullw, INS_pmulld, INS_pmulld, INS_vpmullq, INS_vpmullq, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX512, MultiplyScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mulss, INS_mulsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, Or, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pord, INS_pord, INS_vporq, INS_vporq, INS_orps, INS_orpd, 1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX512, PackSignedSaturate, 64, 2, INS_packsswb, INS_invalid, INS_packssdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, PackUnsignedSaturate, 64, 2, INS_invalid, INS_packuswb, INS_invalid, INS_packusdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, Permute2x64, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermilpd, -1, 1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, Permute4x32, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermilps, INS_invalid, -1, 1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, Permute4x64, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermq, INS_vpermq, INS_invalid, INS_vpermpd, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, PermuteVar16x16, 32, 2, INS_invalid, INS_invalid, INS_vpermw, INS_vpermw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 6, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX512, PermuteVar16x16x2, 32, 3, INS_invalid, INS_invalid, INS_vpermt2w, INS_vpermt2w, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 6, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX512, PermuteVar16x32, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermd, INS_vpermd, INS_invalid, INS_invalid, INS_vpermps, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX512, PermuteVar16x32x2, 64, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermt2d, INS_vpermt2d, INS_invalid, INS_invalid, INS_vpermt2ps, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX512, PermuteVar2x64, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermilpdvar, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, PermuteVar2x64x2, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermt2q, INS_vpermt2q, INS_invalid, INS_vpermt2pd, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX512, PermuteVar32x16, 64, 2, INS_invalid, INS_invalid, INS_vpermw, INS_vpermw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 6, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX512, PermuteVar32x16x2, 64, 3, INS_invalid, INS_invalid, INS_vpermt2w, INS_vpermt2w, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 6, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX512, PermuteVar4x32, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermilpsvar, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, PermuteVar4x32x2, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermt2d, INS_vpermt2d, INS_invalid, INS_invalid, INS_vpermt2ps, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX512, PermuteVar4x64, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermq_reg, INS_vpermq_reg, INS_invalid, INS_vpermpd_reg, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX512, PermuteVar4x64x2, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermt2q, INS_vpermt2q, INS_invalid, INS_vpermt2pd, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX512, PermuteVar8x16 , 16, 2, INS_invalid, INS_invalid, INS_vpermw, INS_vpermw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 6, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX512, PermuteVar8x16x2, 16, 3, INS_invalid, INS_invalid, INS_vpermt2w, INS_vpermt2w, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 6, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX512, PermuteVar8x32x2, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermt2d, INS_vpermt2d, INS_invalid, INS_invalid, INS_vpermt2ps, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX512, PermuteVar8x64, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermq_reg, INS_vpermq_reg, INS_invalid, INS_vpermpd_reg, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX512, PermuteVar8x64x2, 64, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermt2q, INS_vpermt2q, INS_invalid, INS_vpermt2pd, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX512, Range, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrangeps, INS_vrangepd, -1, 4, HW_Category_IMM, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, RangeScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrangess, INS_vrangesd, -1, 4, HW_Category_IMM, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX512, Reciprocal14, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrcp14ps, INS_vrcp14pd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, Reciprocal14Scalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrcp14ss, INS_vrcp14sd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX512, ReciprocalSqrt14, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrsqrt14ps, INS_vrsqrt14pd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, ReciprocalSqrt14Scalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrsqrt14ss, INS_vrsqrt14sd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX512, Reduce, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vreduceps, INS_vreducepd, -1, 4, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, ReduceScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vreducess, INS_vreducesd, -1, 4, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX512, RotateLeft, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vprold, INS_vprold, INS_vprolq, INS_vprolq, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_MaybeNoJmpTableIMM|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, RotateLeftVariable, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vprolvd, INS_vprolvd, INS_vprolvq, INS_vprolvq, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, RotateRight, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vprord, INS_vprord, INS_vprorq, INS_vprorq, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_MaybeNoJmpTableIMM|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, RotateRightVariable, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vprorvd, INS_vprorvd, INS_vprorvq, INS_vprorvq, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, RoundScale, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrndscaleps, INS_vrndscalepd, -1, 8, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, RoundScaleScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrndscaless, INS_vrndscalesd, -1, 8, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX512, Scale, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vscalefps, INS_vscalefpd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, ScaleScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vscalefss, INS_vscalefsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, ShiftLeftLogical, 64, 2, INS_invalid, INS_invalid, INS_psllw, INS_psllw, INS_pslld, INS_pslld, INS_psllq, INS_psllq, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, ShiftLeftLogical128BitLane, 64, 2, INS_pslldq, INS_pslldq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, ShiftLeftLogicalVariable, -1, 2, INS_invalid, INS_invalid, INS_vpsllvw, INS_vpsllvw, INS_vpsllvd, INS_vpsllvd, INS_vpsllvq, INS_vpsllvq, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, ShiftRightArithmetic, -1, 2, INS_invalid, INS_invalid, INS_psraw, INS_invalid, INS_psrad, INS_invalid, INS_vpsraq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, ShiftRightArithmeticVariable, -1, 2, INS_invalid, INS_invalid, INS_vpsravw, INS_invalid, INS_vpsravd, INS_invalid, INS_vpsravq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, ShiftRightLogical, 64, 2, INS_invalid, INS_invalid, INS_psrlw, INS_psrlw, INS_psrld, INS_psrld, INS_psrlq, INS_psrlq, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, ShiftRightLogical128BitLane, 64, 2, INS_psrldq, INS_psrldq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, ShiftRightLogicalVariable, -1, 2, INS_invalid, INS_invalid, INS_vpsrlvw, INS_vpsrlvw, INS_vpsrlvd, INS_vpsrlvd, INS_vpsrlvq, INS_vpsrlvq, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, Shuffle, 64, -1, INS_pshufb, INS_pshufb, INS_invalid, INS_invalid, INS_pshufd, INS_pshufd, INS_invalid, INS_invalid, INS_shufps, INS_shufpd, 1, 1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, Shuffle2x128, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vshufi32x4, INS_vshufi32x4, INS_vshufi64x2, INS_vshufi64x2, INS_vshuff32x4, INS_vshuff64x2, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, Shuffle4x128, 64, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vshufi32x4, INS_vshufi32x4, INS_vshufi64x2, INS_vshufi64x2, INS_vshuff32x4, INS_vshuff64x2, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, ShuffleHigh, 64, 2, INS_invalid, INS_invalid, INS_pshufhw, INS_pshufhw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, ShuffleLow, 64, 2, INS_invalid, INS_invalid, INS_pshuflw, INS_pshuflw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, Sqrt, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqrtps, INS_sqrtpd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, SqrtScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqrtss, INS_sqrtsd, -1, -1, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, Store, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(AVX512, StoreAligned, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_vmovdqa64, INS_vmovdqa64, INS_movaps, INS_movapd, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX512, StoreAlignedNonTemporal, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movntdq, INS_movntdq, INS_movntdq, INS_movntdq, INS_movntps, INS_movntpd, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX512, Subtract, 64, -1, INS_psubb, INS_psubb, INS_psubw, INS_psubw, INS_psubd, INS_psubd, INS_psubq, INS_psubq, INS_subps, INS_subpd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, SubtractSaturate, 64, 2, INS_psubsb, INS_psubusb, INS_psubsw, INS_psubusw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, SubtractScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_subss, INS_subsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, SumAbsoluteDifferences, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_psadbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, SumAbsoluteDifferencesInBlock32, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_vdbpsadbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, TernaryLogic, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpternlogd, INS_vpternlogd, INS_vpternlogq, INS_vpternlogq, INS_vpternlogd, INS_vpternlogq, 1, 1, HW_Category_IMM, HW_Flag_SpecialImport|HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX512, UnpackHigh, 64, 2, INS_punpckhbw, INS_punpckhbw, INS_punpckhwd, INS_punpckhwd, INS_punpckhdq, INS_punpckhdq, INS_punpckhqdq, INS_punpckhqdq, INS_unpckhps, INS_unpckhpd, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, UnpackLow, 64, 2, INS_punpcklbw, INS_punpcklbw, INS_punpcklwd, INS_punpcklwd, INS_punpckldq, INS_punpckldq, INS_punpcklqdq, INS_punpcklqdq, INS_unpcklps, INS_unpcklpd, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, Xor, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pxord, INS_pxord, INS_vpxorq, INS_vpxorq, INS_xorps, INS_xorpd, 1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX512, Abs, -1, 1, INS_pabsb, INS_invalid, INS_pabsw, INS_invalid, INS_pabsd, INS_invalid, INS_vpabsq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, Add, 64, -1, INS_paddb, INS_paddb, INS_paddw, INS_paddw, INS_paddd, INS_paddd, INS_paddq, INS_paddq, INS_addps, INS_addpd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, AddSaturate, 64, 2, INS_paddsb, INS_paddusb, INS_paddsw, INS_paddusw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX512, AddScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_addss, INS_addsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, AlignRight, 64, 3, INS_palignr, INS_palignr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, AlignRight32, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_valignd, INS_valignd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, AlignRight64, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_valignq, INS_valignq, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, And, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pandd, INS_pandd, INS_vpandq, INS_vpandq, INS_andps, INS_andpd, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX512, AndNot, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pandnd, INS_pandnd, INS_vpandnq, INS_vpandnq, INS_andnps, INS_andnpd, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX512, Average, 64, 2, INS_invalid, INS_pavgb, INS_invalid, INS_pavgw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX512, BlendVariable, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, BroadcastPairScalarToVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcasti32x2, INS_vbroadcasti32x2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, BroadcastPairScalarToVector256, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcasti32x2, INS_vbroadcasti32x2, INS_invalid, INS_invalid, INS_vbroadcastf32x2, INS_invalid, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, BroadcastPairScalarToVector512, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcasti32x2, INS_vbroadcasti32x2, INS_invalid, INS_invalid, INS_vbroadcastf32x2, INS_invalid, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, BroadcastScalarToVector512, 64, 1, INS_vpbroadcastb, INS_vpbroadcastb, INS_vpbroadcastw, INS_vpbroadcastw, INS_vpbroadcastd, INS_vpbroadcastd, INS_vpbroadcastq, INS_vpbroadcastq, INS_vbroadcastss, INS_vbroadcastsd, INS_invalid, 3, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, BroadcastVector128ToVector512, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcasti32x4, INS_vbroadcasti32x4, INS_vbroadcasti64x2, INS_vbroadcasti64x2, INS_vbroadcastf32x4, INS_vbroadcastf64x2, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, BroadcastVector256ToVector512, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcasti32x8, INS_vbroadcasti32x8, INS_vbroadcasti64x4, INS_vbroadcasti64x4, INS_vbroadcastf32x8, INS_vbroadcastf64x4, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, Classify, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_InvalidNodeId|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, ClassifyScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_InvalidNodeId|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, Compare, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, CompareEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, CompareGreaterThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, CompareGreaterThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, CompareLessThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, CompareLessThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, CompareNotEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, CompareNotGreaterThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, CompareNotGreaterThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, CompareNotLessThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, CompareNotLessThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, CompareOrdered, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, CompareUnordered, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, Compress, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, CompressStore, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(AVX512, ConvertScalarToVector128Double, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2sd32, INS_vcvtusi2sd32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX512, ConvertScalarToVector128Single, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2ss32, INS_vcvtusi2ss32, INS_invalid, INS_invalid, INS_invalid, INS_cvtsd2ss, INS_invalid, 5, 4, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, ConvertToInt32, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtss2si32, INS_cvtsd2si32, INS_invalid, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, ConvertToUInt32, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtss2usi32, INS_vcvtsd2usi32, INS_invalid, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, ConvertToUInt32WithTruncation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttss2usi32, INS_vcvttsd2usi32, INS_invalid, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128Byte, -1, 1, INS_invalid, INS_invalid, INS_vpmovwb, INS_vpmovwb, INS_vpmovdb, INS_vpmovdb, INS_vpmovqb, INS_vpmovqb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128ByteWithSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_vpmovuswb, INS_invalid, INS_vpmovusdb, INS_invalid, INS_vpmovusqb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128Double, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtudq2pd, INS_vcvtqq2pd, INS_vcvtuqq2pd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128Int16, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovdw, INS_vpmovdw, INS_vpmovqw, INS_vpmovqw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128Int16WithSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovsdw, INS_invalid, INS_vpmovsqw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128Int32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovqd, INS_vpmovqd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128Int32WithSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovsqd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128Int64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2qq, INS_vcvtpd2qq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128Int64WithTruncation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2qq, INS_vcvttpd2qq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128SByte, -1, 1, INS_invalid, INS_invalid, INS_vpmovwb, INS_vpmovwb, INS_vpmovdb, INS_vpmovdb, INS_vpmovqb, INS_vpmovqb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128SByteWithSaturation, -1, 1, INS_invalid, INS_invalid, INS_vpmovswb, INS_invalid, INS_vpmovsdb, INS_invalid, INS_vpmovsqb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128Single, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtudq2ps, INS_vcvtqq2ps, INS_vcvtuqq2ps, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128UInt16, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovdw, INS_vpmovdw, INS_vpmovqw, INS_vpmovqw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128UInt16WithSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovusdw, INS_invalid, INS_vpmovusqw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128UInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovqd, INS_vpmovqd, INS_vcvtps2udq, INS_vcvtpd2udq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128UInt32WithSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovusqd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128UInt32WithTruncation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2udq, INS_vcvttpd2udq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128UInt64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2uqq, INS_vcvtpd2uqq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128UInt64WithTruncation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2uqq, INS_vcvttpd2uqq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256Byte, 64, 1, INS_invalid, INS_invalid, INS_vpmovwb, INS_vpmovwb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256ByteWithSaturation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_vpmovuswb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256Double, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtudq2pd, INS_vcvtqq2pd, INS_vcvtuqq2pd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256Int16, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovdw, INS_vpmovdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256Int16WithSaturation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovsdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256Int32, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovqd, INS_vpmovqd, INS_invalid, INS_cvtpd2dq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256Int32WithSaturation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovsqd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256Int32WithTruncation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvttpd2dq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256Int64, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2qq, INS_vcvtpd2qq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256Int64WithTruncation, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2qq, INS_vcvttpd2qq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256SByte, 64, 1, INS_invalid, INS_invalid, INS_vpmovwb, INS_vpmovwb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256SByteWithSaturation, 64, 1, INS_invalid, INS_invalid, INS_vpmovswb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256Single, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtudq2ps, INS_vcvtqq2ps, INS_vcvtuqq2ps, INS_invalid, INS_cvtpd2ps, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256UInt16, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovdw, INS_vpmovdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256UInt16WithSaturation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovusdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256UInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovqd, INS_vpmovqd, INS_vcvtps2udq, INS_vcvtpd2udq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256UInt32WithSaturation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovusqd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256UInt32WithTruncation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2udq, INS_vcvttpd2udq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256UInt64, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2uqq, INS_vcvtpd2uqq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256UInt64WithTruncation, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2uqq, INS_vcvttpd2uqq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector512Double, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtdq2pd, INS_vcvtudq2pd, INS_vcvtqq2pd, INS_vcvtuqq2pd, INS_cvtps2pd, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, ConvertToVector512Int16, 64, 1, INS_pmovsxbw, INS_pmovzxbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector512Int32, 64, -1, INS_pmovsxbd, INS_pmovzxbd, INS_pmovsxwd, INS_pmovzxwd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtps2dq, INS_invalid, INS_invalid, 3, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, ConvertToVector512Int32WithTruncation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvttps2dq, INS_invalid, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector512Int64, 64, -1, INS_pmovsxbq, INS_pmovzxbq, INS_pmovsxwq, INS_pmovzxwq, INS_pmovsxdq, INS_pmovzxdq, INS_invalid, INS_invalid, INS_vcvtps2qq, INS_vcvtpd2qq, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, ConvertToVector512Int64WithTruncation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2qq, INS_vcvttpd2qq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector512Single, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtdq2ps, INS_vcvtudq2ps, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, ConvertToVector512UInt16, 64, 1, INS_pmovsxbw, INS_pmovzxbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector512UInt32, 64, -1, INS_pmovsxbd, INS_pmovzxbd, INS_pmovsxwd, INS_pmovzxwd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2udq, INS_invalid, INS_invalid, 3, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, ConvertToVector512UInt32WithTruncation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2udq, INS_invalid, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector512UInt64, 64, -1, INS_pmovsxbq, INS_pmovzxbq, INS_pmovsxwq, INS_pmovzxwq, INS_pmovsxdq, INS_pmovzxdq, INS_invalid, INS_invalid, INS_vcvtps2uqq, INS_vcvtpd2uqq, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, ConvertToVector512UInt64WithTruncation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2uqq, INS_vcvttpd2uqq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, DetectConflicts, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpconflictd, INS_vpconflictd, INS_vpconflictq, INS_vpconflictq, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, Divide, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_divps, INS_divpd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, DivideScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_divss, INS_divsd, INS_invalid, -1, -1, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, DuplicateEvenIndexed, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movsldup, INS_movddup, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, DuplicateOddIndexed, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movshdup, INS_invalid, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, Expand, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, ExpandLoad, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, ExtractVector128, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vextracti32x4, INS_vextracti32x4, INS_vextracti64x2, INS_vextracti64x2, INS_vextractf32x4, INS_vextractf64x2, INS_invalid, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX512, ExtractVector256, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vextracti32x8, INS_vextracti32x8, INS_vextracti64x4, INS_vextracti64x4, INS_vextractf32x8, INS_vextractf64x4, INS_invalid, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX512, Fixup, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfixupimmps, INS_vfixupimmpd, INS_invalid, -1, 4, HW_Category_IMM, HW_Flag_SpecialImport|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, FixupScalar, 16, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfixupimmss, INS_vfixupimmsd, INS_invalid, -1, 4, HW_Category_IMM, HW_Flag_SpecialImport|HW_Flag_FullRangeIMM|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX512, FusedMultiplyAdd, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmadd213ps, INS_vfmadd213pd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, FusedMultiplyAddNegated, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmadd213ps, INS_vfnmadd213pd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, FusedMultiplyAddNegatedScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmadd213ss, INS_vfnmadd213sd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX512, FusedMultiplyAddScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmadd213ss, INS_vfmadd213sd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX512, FusedMultiplyAddSubtract, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmaddsub213ps, INS_vfmaddsub213pd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, FusedMultiplySubtract, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmsub213ps, INS_vfmsub213pd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, FusedMultiplySubtractAdd, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmsubadd213ps, INS_vfmsubadd213pd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, FusedMultiplySubtractNegated, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmsub213ps, INS_vfnmsub213pd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, FusedMultiplySubtractNegatedScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmsub213ss, INS_vfnmsub213sd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX512, FusedMultiplySubtractScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmsub213ss, INS_vfmsub213sd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX512, GetExponent, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vgetexpps, INS_vgetexppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, GetExponentScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vgetexpss, INS_vgetexpsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX512, GetMantissa, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vgetmantps, INS_vgetmantpd, INS_invalid, -1, 4, HW_Category_IMM, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, GetMantissaScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vgetmantss, INS_vgetmantsd, INS_invalid, -1, 4, HW_Category_IMM, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX512, InsertVector128, 64, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vinserti32x4, INS_vinserti32x4, INS_vinserti64x2, INS_vinserti64x2, INS_vinsertf32x4, INS_vinsertf64x2, INS_invalid, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX512, InsertVector256, 64, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vinserti32x8, INS_vinserti32x8, INS_vinserti64x4, INS_vinserti64x4, INS_vinsertf32x8, INS_vinsertf64x4, INS_invalid, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX512, LeadingZeroCount, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vplzcntd, INS_vplzcntd, INS_vplzcntq, INS_vplzcntq, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, LoadAlignedVector512, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_vmovdqa64, INS_vmovdqa64, INS_movaps, INS_movapd, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX512, LoadAlignedVector512NonTemporal, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movntdqa, INS_movntdqa, INS_movntdqa, INS_movntdqa, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX512, LoadVector512, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, MaskLoad, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, MaskLoadAligned, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, MaskStore, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(AVX512, MaskStoreAligned, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(AVX512, Max, -1, 2, INS_pmaxsb, INS_pmaxub, INS_pmaxsw, INS_pmaxuw, INS_pmaxsd, INS_pmaxud, INS_vpmaxsq, INS_vpmaxuq, INS_maxps, INS_maxpd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative) +HARDWARE_INTRINSIC(AVX512, Min, -1, 2, INS_pminsb, INS_pminub, INS_pminsw, INS_pminuw, INS_pminsd, INS_pminud, INS_vpminsq, INS_vpminuq, INS_minps, INS_minpd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative) +HARDWARE_INTRINSIC(AVX512, MoveMask, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoContainment|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, Multiply, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pmuldq, INS_pmuludq, INS_mulps, INS_mulpd, INS_invalid, 5, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, MultiplyAddAdjacent, 64, 2, INS_invalid, INS_invalid, INS_pmaddubsw, INS_invalid, INS_pmaddwd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, MultiplyHigh, 64, 2, INS_invalid, INS_invalid, INS_pmulhw, INS_pmulhuw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX512, MultiplyHighRoundScale, 64, 2, INS_invalid, INS_invalid, INS_pmulhrsw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, MultiplyLow, -1, 2, INS_invalid, INS_invalid, INS_pmullw, INS_pmullw, INS_pmulld, INS_pmulld, INS_vpmullq, INS_vpmullq, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX512, MultiplyScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mulss, INS_mulsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, Or, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pord, INS_pord, INS_vporq, INS_vporq, INS_orps, INS_orpd, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX512, PackSignedSaturate, 64, 2, INS_packsswb, INS_invalid, INS_packssdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, PackUnsignedSaturate, 64, 2, INS_invalid, INS_packuswb, INS_invalid, INS_packusdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, Permute2x64, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermilpd, INS_invalid, -1, 1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, Permute4x32, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermilps, INS_invalid, INS_invalid, -1, 1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, Permute4x64, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermq, INS_vpermq, INS_invalid, INS_vpermpd, INS_invalid, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, PermuteVar16x16, 32, 2, INS_invalid, INS_invalid, INS_vpermw, INS_vpermw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 6, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX512, PermuteVar16x16x2, 32, 3, INS_invalid, INS_invalid, INS_vpermt2w, INS_vpermt2w, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 6, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX512, PermuteVar16x32, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermd, INS_vpermd, INS_invalid, INS_invalid, INS_vpermps, INS_invalid, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX512, PermuteVar16x32x2, 64, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermt2d, INS_vpermt2d, INS_invalid, INS_invalid, INS_vpermt2ps, INS_invalid, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX512, PermuteVar2x64, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermilpdvar, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, PermuteVar2x64x2, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermt2q, INS_vpermt2q, INS_invalid, INS_vpermt2pd, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX512, PermuteVar32x16, 64, 2, INS_invalid, INS_invalid, INS_vpermw, INS_vpermw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 6, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX512, PermuteVar32x16x2, 64, 3, INS_invalid, INS_invalid, INS_vpermt2w, INS_vpermt2w, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 6, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX512, PermuteVar4x32, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermilpsvar, INS_invalid, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, PermuteVar4x32x2, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermt2d, INS_vpermt2d, INS_invalid, INS_invalid, INS_vpermt2ps, INS_invalid, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX512, PermuteVar4x64, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermq_reg, INS_vpermq_reg, INS_invalid, INS_vpermpd_reg, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX512, PermuteVar4x64x2, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermt2q, INS_vpermt2q, INS_invalid, INS_vpermt2pd, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX512, PermuteVar8x16 , 16, 2, INS_invalid, INS_invalid, INS_vpermw, INS_vpermw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 6, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX512, PermuteVar8x16x2, 16, 3, INS_invalid, INS_invalid, INS_vpermt2w, INS_vpermt2w, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 6, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX512, PermuteVar8x32x2, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermt2d, INS_vpermt2d, INS_invalid, INS_invalid, INS_vpermt2ps, INS_invalid, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX512, PermuteVar8x64, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermq_reg, INS_vpermq_reg, INS_invalid, INS_vpermpd_reg, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX512, PermuteVar8x64x2, 64, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermt2q, INS_vpermt2q, INS_invalid, INS_vpermt2pd, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX512, Range, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrangeps, INS_vrangepd, INS_invalid, -1, 4, HW_Category_IMM, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, RangeScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrangess, INS_vrangesd, INS_invalid, -1, 4, HW_Category_IMM, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX512, Reciprocal14, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrcp14ps, INS_vrcp14pd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, Reciprocal14Scalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrcp14ss, INS_vrcp14sd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX512, ReciprocalSqrt14, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrsqrt14ps, INS_vrsqrt14pd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, ReciprocalSqrt14Scalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrsqrt14ss, INS_vrsqrt14sd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX512, Reduce, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vreduceps, INS_vreducepd, INS_invalid, -1, 4, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, ReduceScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vreducess, INS_vreducesd, INS_invalid, -1, 4, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX512, RotateLeft, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vprold, INS_vprold, INS_vprolq, INS_vprolq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_MaybeNoJmpTableIMM|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, RotateLeftVariable, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vprolvd, INS_vprolvd, INS_vprolvq, INS_vprolvq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, RotateRight, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vprord, INS_vprord, INS_vprorq, INS_vprorq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_MaybeNoJmpTableIMM|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, RotateRightVariable, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vprorvd, INS_vprorvd, INS_vprorvq, INS_vprorvq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, RoundScale, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrndscaleps, INS_vrndscalepd, INS_invalid, -1, 8, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, RoundScaleScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrndscaless, INS_vrndscalesd, INS_invalid, -1, 8, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX512, Scale, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vscalefps, INS_vscalefpd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, ScaleScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vscalefss, INS_vscalefsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, ShiftLeftLogical, 64, 2, INS_invalid, INS_invalid, INS_psllw, INS_psllw, INS_pslld, INS_pslld, INS_psllq, INS_psllq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, ShiftLeftLogical128BitLane, 64, 2, INS_pslldq, INS_pslldq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, ShiftLeftLogicalVariable, -1, 2, INS_invalid, INS_invalid, INS_vpsllvw, INS_vpsllvw, INS_vpsllvd, INS_vpsllvd, INS_vpsllvq, INS_vpsllvq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, ShiftRightArithmetic, -1, 2, INS_invalid, INS_invalid, INS_psraw, INS_invalid, INS_psrad, INS_invalid, INS_vpsraq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, ShiftRightArithmeticVariable, -1, 2, INS_invalid, INS_invalid, INS_vpsravw, INS_invalid, INS_vpsravd, INS_invalid, INS_vpsravq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, ShiftRightLogical, 64, 2, INS_invalid, INS_invalid, INS_psrlw, INS_psrlw, INS_psrld, INS_psrld, INS_psrlq, INS_psrlq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, ShiftRightLogical128BitLane, 64, 2, INS_psrldq, INS_psrldq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, ShiftRightLogicalVariable, -1, 2, INS_invalid, INS_invalid, INS_vpsrlvw, INS_vpsrlvw, INS_vpsrlvd, INS_vpsrlvd, INS_vpsrlvq, INS_vpsrlvq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, Shuffle, 64, -1, INS_pshufb, INS_pshufb, INS_invalid, INS_invalid, INS_pshufd, INS_pshufd, INS_invalid, INS_invalid, INS_shufps, INS_shufpd, INS_invalid, 1, 1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, Shuffle2x128, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vshufi32x4, INS_vshufi32x4, INS_vshufi64x2, INS_vshufi64x2, INS_vshuff32x4, INS_vshuff64x2, INS_invalid, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, Shuffle4x128, 64, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vshufi32x4, INS_vshufi32x4, INS_vshufi64x2, INS_vshufi64x2, INS_vshuff32x4, INS_vshuff64x2, INS_invalid, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, ShuffleHigh, 64, 2, INS_invalid, INS_invalid, INS_pshufhw, INS_pshufhw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, ShuffleLow, 64, 2, INS_invalid, INS_invalid, INS_pshuflw, INS_pshuflw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, Sqrt, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqrtps, INS_sqrtpd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, SqrtScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqrtss, INS_sqrtsd, INS_invalid, -1, -1, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, Store, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(AVX512, StoreAligned, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_vmovdqa64, INS_vmovdqa64, INS_movaps, INS_movapd, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX512, StoreAlignedNonTemporal, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movntdq, INS_movntdq, INS_movntdq, INS_movntdq, INS_movntps, INS_movntpd, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX512, Subtract, 64, -1, INS_psubb, INS_psubb, INS_psubw, INS_psubw, INS_psubd, INS_psubd, INS_psubq, INS_psubq, INS_subps, INS_subpd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, SubtractSaturate, 64, 2, INS_psubsb, INS_psubusb, INS_psubsw, INS_psubusw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, SubtractScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_subss, INS_subsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, SumAbsoluteDifferences, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_psadbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, SumAbsoluteDifferencesInBlock32, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_vdbpsadbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, TernaryLogic, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpternlogd, INS_vpternlogd, INS_vpternlogq, INS_vpternlogq, INS_vpternlogd, INS_vpternlogq, INS_invalid, 1, 1, HW_Category_IMM, HW_Flag_SpecialImport|HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX512, UnpackHigh, 64, 2, INS_punpckhbw, INS_punpckhbw, INS_punpckhwd, INS_punpckhwd, INS_punpckhdq, INS_punpckhdq, INS_punpckhqdq, INS_punpckhqdq, INS_unpckhps, INS_unpckhpd, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, UnpackLow, 64, 2, INS_punpcklbw, INS_punpcklbw, INS_punpcklwd, INS_punpcklwd, INS_punpckldq, INS_punpckldq, INS_punpcklqdq, INS_punpcklqdq, INS_unpcklps, INS_unpcklpd, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, Xor, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pxord, INS_pxord, INS_vpxorq, INS_vpxorq, INS_xorps, INS_xorpd, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp|HW_Flag_NormalizeSmallTypeToInt) #define LAST_NI_AVX512 NI_AVX512_Xor // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -620,11 +620,11 @@ HARDWARE_INTRINSIC(AVX512, Xor, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // 64-bit only Intrinsics for AVX512F, AVX512BW, AVX512CD, AVX512DQ, AVX512VL #define FIRST_NI_AVX512_X64 NI_AVX512_X64_ConvertScalarToVector128Double -HARDWARE_INTRINSIC(AVX512_X64, ConvertScalarToVector128Double, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2sd64, INS_vcvtusi2sd64, INS_invalid, INS_invalid, 5, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_CopyUpperBits|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512_X64, ConvertScalarToVector128Single, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2ss64, INS_vcvtusi2ss64, INS_invalid, INS_invalid, 5, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_CopyUpperBits|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512_X64, ConvertToInt64, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtss2si64, INS_cvtsd2si64, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512_X64, ConvertToUInt64, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtss2usi64, INS_vcvtsd2usi64, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512_X64, ConvertToUInt64WithTruncation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttss2usi64, INS_vcvttsd2usi64, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512_X64, ConvertScalarToVector128Double, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2sd64, INS_vcvtusi2sd64, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_CopyUpperBits|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512_X64, ConvertScalarToVector128Single, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2ss64, INS_vcvtusi2ss64, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_CopyUpperBits|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512_X64, ConvertToInt64, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtss2si64, INS_cvtsd2si64, INS_invalid, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512_X64, ConvertToUInt64, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtss2usi64, INS_vcvtsd2usi64, INS_invalid, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512_X64, ConvertToUInt64WithTruncation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttss2usi64, INS_vcvttsd2usi64, INS_invalid, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) #define LAST_NI_AVX512_X64 NI_AVX512_X64_ConvertToUInt64WithTruncation // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -633,13 +633,13 @@ HARDWARE_INTRINSIC(AVX512_X64, ConvertToUInt64WithTruncation, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Intrinsics for AVX512-IFMA, AVX512-VBMI #define FIRST_NI_AVX512v2 NI_AVX512v2_MultiShift -HARDWARE_INTRINSIC(AVX512v2, MultiShift, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmultishiftqb, INS_vpmultishiftqb, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(AVX512v2, PermuteVar16x8, 16, 2, INS_vpermb, INS_vpermb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX512v2, PermuteVar16x8x2, 16, 3, INS_vpermt2b, INS_vpermt2b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX512v2, PermuteVar32x8, 32, 2, INS_vpermb, INS_vpermb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX512v2, PermuteVar32x8x2, 32, 3, INS_vpermt2b, INS_vpermt2b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX512v2, PermuteVar64x8, 64, 2, INS_vpermb, INS_vpermb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX512v2, PermuteVar64x8x2, 64, 3, INS_vpermt2b, INS_vpermt2b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX512v2, MultiShift, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmultishiftqb, INS_vpmultishiftqb, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(AVX512v2, PermuteVar16x8, 16, 2, INS_vpermb, INS_vpermb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX512v2, PermuteVar16x8x2, 16, 3, INS_vpermt2b, INS_vpermt2b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX512v2, PermuteVar32x8, 32, 2, INS_vpermb, INS_vpermb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX512v2, PermuteVar32x8x2, 32, 3, INS_vpermt2b, INS_vpermt2b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX512v2, PermuteVar64x8, 64, 2, INS_vpermb, INS_vpermb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX512v2, PermuteVar64x8x2, 64, 3, INS_vpermt2b, INS_vpermt2b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) #define LAST_NI_AVX512v2 NI_AVX512v2_PermuteVar64x8x2 // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -648,35 +648,73 @@ HARDWARE_INTRINSIC(AVX512v2, PermuteVar64x8x2, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Intrinsics for AVX512-BITALG, AVX512-VBMI2, AVX512-VPOPCNTDQ, AVX512-VNNI #define FIRST_NI_AVX512v3 NI_AVX512v3_Compress -HARDWARE_INTRINSIC(AVX512v3, Compress, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512v3, CompressStore, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(AVX512v3, Expand, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512v3, ExpandLoad, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512v3, MultiplyWideningAndAdd, 64, 3, INS_invalid, INS_vpdpbusd, INS_vpdpwssd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(AVX512v3, MultiplyWideningAndAddSaturate, 64, 3, INS_invalid, INS_vpdpbusds, INS_vpdpwssds, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(AVX512v3, Compress, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512v3, CompressStore, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(AVX512v3, Expand, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512v3, ExpandLoad, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512v3, MultiplyWideningAndAdd, 64, 3, INS_invalid, INS_vpdpbusd, INS_vpdpwssd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(AVX512v3, MultiplyWideningAndAddSaturate, 64, 3, INS_invalid, INS_vpdpbusds, INS_vpdpwssds, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg) #define LAST_NI_AVX512v3 NI_AVX512v3_MultiplyWideningAndAddSaturate +// *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** +// ISA Function name SIMD size NumArg Instructions IntCost FltCost Category Flags +// TYP_BYTE, TYP_UBYTE, TYP_SHORT, TYP_USHORT, TYP_INT, TYP_UINT, TYP_LONG, TYP_ULONG, TYP_FLOAT, TYP_DOUBLE +// *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** +// Intrinsics for AVX10v1 +#define FIRST_NI_AVX10v1 NI_AVX10v1_AddScalar +HARDWARE_INTRINSIC(AVX10v1, AddScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vaddsh, -1, 4, HW_Category_SIMDScalar, HW_Flag_EmbRoundingCompatible|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarOrderedEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcomish, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarOrderedGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcomish, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarOrderedGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcomish, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarOrderedLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcomish, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarOrderedLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcomish, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarOrderedNotEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcomish, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarUnorderedEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vucomish, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarUnorderedGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vucomish, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarUnorderedGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vucomish, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarUnorderedLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vucomish, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarUnorderedLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vucomish, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarUnorderedNotEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vucomish, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, ConvertScalarToVector128Double, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtsh2sd, 5, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX10v1, ConvertScalarToVector128Half, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtsi2sh32, INS_vcvtusi2sh32, INS_vcvtsi2sh64, INS_vcvtusi2sh64, INS_vcvtss2sh, INS_vcvtsd2sh, INS_invalid, 5, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX10v1, ConvertScalarToVector128Single, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtsh2ss, 5, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX10v1, ConvertToInt32, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtsh2si32, -1, 7, HW_Category_SIMDScalar, HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX10v1, ConvertToInt64, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtsh2si64, -1, 7, HW_Category_SIMDScalar, HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX10v1, ConvertToUInt32, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtsh2usi32, -1, 7, HW_Category_SIMDScalar, HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX10v1, ConvertToUInt64, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtsh2usi64, -1, 7, HW_Category_SIMDScalar, HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX10v1, DivideScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vdivsh, -1, -1, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX10v1, FusedMultiplyAddScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmadd213sh, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX10v1, MaxScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vmaxsh, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX10v1, MinScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vminsh, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX10v1, MultiplyScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vmulsh, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX10v1, ReciprocalScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrcpsh, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX10v1, ReciprocalSqrtScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrsqrtsh, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX10v1, RoundScaleScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrndscalesh, -1, 8, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX10v1, SqrtScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vsqrtsh, -1, -1, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX10v1, SubtractScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vsubsh, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +#define LAST_NI_AVX10v1 NI_AVX10v1_SubtractScalar + // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // ISA Function name SIMD size NumArg Instructions IntCost FltCost Category Flags // TYP_BYTE, TYP_UBYTE, TYP_SHORT, TYP_USHORT, TYP_INT, TYP_UINT, TYP_LONG, TYP_ULONG, TYP_FLOAT, TYP_DOUBLE // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Intrinsics for AVX10v2 #define FIRST_NI_AVX10v2 NI_AVX10v2_ConvertToByteWithSaturationAndZeroExtendToInt32 -HARDWARE_INTRINSIC(AVX10v2, ConvertToByteWithSaturationAndZeroExtendToInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2iubs, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX10v2, ConvertToByteWithTruncatedSaturationAndZeroExtendToInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2iubs, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX10v2, ConvertToInt32WithTruncatedSaturation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttss2sis32, INS_vcvttsd2sis32, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX10v2, ConvertToSByteWithSaturationAndZeroExtendToInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2ibs, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX10v2, ConvertToSByteWithTruncatedSaturationAndZeroExtendToInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2ibs, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX10v2, ConvertToUInt32WithTruncatedSaturation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttss2usis32, INS_vcvttsd2usis32, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX10v2, ConvertToVectorInt32WithTruncatedSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2dqs, INS_vcvttpd2dqs, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX10v2, ConvertToVectorInt64WithTruncatedSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2qqs, INS_vcvttpd2qqs, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX10v2, ConvertToVectorUInt32WithTruncatedSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2udqs, INS_vcvttpd2udqs, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX10v2, ConvertToVectorUInt64WithTruncatedSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2uqqs, INS_vcvttpd2uqqs, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX10v2, MinMax, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vminmaxps, INS_vminmaxpd, -1, 4, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX10v2, MinMaxScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vminmaxss, INS_vminmaxsd, -1, 4, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX10v2, MoveScalar, 16, -1, INS_invalid, INS_invalid, INS_vmovw_simd, INS_vmovw_simd, INS_vmovd_simd, INS_vmovd_simd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_NoContainment) -HARDWARE_INTRINSIC(AVX10v2, MultipleSumAbsoluteDifferences, 64, 3, INS_invalid, INS_invalid, INS_invalid, INS_vmpsadbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX10v2, StoreScalar, 16, 2, INS_invalid, INS_invalid, INS_vmovw_simd, INS_vmovw_simd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(AVX10v2, ConvertToByteWithSaturationAndZeroExtendToInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2iubs, INS_invalid, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX10v2, ConvertToByteWithTruncatedSaturationAndZeroExtendToInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2iubs, INS_invalid, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX10v2, ConvertToInt32WithTruncatedSaturation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttss2sis32, INS_vcvttsd2sis32, INS_invalid, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX10v2, ConvertToSByteWithSaturationAndZeroExtendToInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2ibs, INS_invalid, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX10v2, ConvertToSByteWithTruncatedSaturationAndZeroExtendToInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2ibs, INS_invalid, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX10v2, ConvertToUInt32WithTruncatedSaturation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttss2usis32, INS_vcvttsd2usis32, INS_invalid, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX10v2, ConvertToVectorInt32WithTruncatedSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2dqs, INS_vcvttpd2dqs, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX10v2, ConvertToVectorInt64WithTruncatedSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2qqs, INS_vcvttpd2qqs, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX10v2, ConvertToVectorUInt32WithTruncatedSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2udqs, INS_vcvttpd2udqs, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX10v2, ConvertToVectorUInt64WithTruncatedSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2uqqs, INS_vcvttpd2uqqs, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX10v2, MinMax, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vminmaxps, INS_vminmaxpd, INS_invalid, -1, 4, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX10v2, MinMaxScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vminmaxss, INS_vminmaxsd, INS_invalid, -1, 4, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX10v2, MoveScalar, 16, -1, INS_invalid, INS_invalid, INS_vmovw_simd, INS_vmovw_simd, INS_vmovd_simd, INS_vmovd_simd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_NoContainment) +HARDWARE_INTRINSIC(AVX10v2, MultipleSumAbsoluteDifferences, 64, 3, INS_invalid, INS_invalid, INS_invalid, INS_vmpsadbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX10v2, StoreScalar, 16, 2, INS_invalid, INS_invalid, INS_vmovw_simd, INS_vmovw_simd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromSecondArg) #define LAST_NI_AVX10v2 NI_AVX10v2_StoreScalar // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -685,14 +723,14 @@ HARDWARE_INTRINSIC(AVX10v2, StoreScalar, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // 64-bit only intrinsics for AVX10v2 #define FIRST_NI_AVX10v2_X64 NI_AVX10v2_X64_ConvertToInt64WithTruncatedSaturation -HARDWARE_INTRINSIC(AVX10v2_X64, ConvertToInt64WithTruncatedSaturation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttss2sis64, INS_vcvttsd2sis64, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX10v2_X64, ConvertToUInt64WithTruncatedSaturation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttss2usis64, INS_vcvttsd2usis64, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX10v2_X64, ConvertToInt64WithTruncatedSaturation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttss2sis64, INS_vcvttsd2sis64, INS_invalid, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX10v2_X64, ConvertToUInt64WithTruncatedSaturation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttss2usis64, INS_vcvttsd2usis64, INS_invalid, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) #define LAST_NI_AVX10v2_X64 NI_AVX10v2_X64_ConvertToUInt64WithTruncatedSaturation #define FIRST_NI_AVX512BMM NI_AVX512BMM_BitMultiplyMatrix16x16WithOrReduction -HARDWARE_INTRINSIC(AVX512BMM, BitMultiplyMatrix16x16WithOrReduction, -1, -1, INS_invalid, INS_invalid, INS_vbmacor16x16x16, INS_vbmacor16x16x16, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX512BMM, BitMultiplyMatrix16x16WithXorReduction, -1, -1, INS_invalid, INS_invalid, INS_vbmacxor16x16x16, INS_vbmacxor16x16x16, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX512BMM, ReverseBits, -1, -1, INS_invalid, INS_vbitrev, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512BMM, BitMultiplyMatrix16x16WithOrReduction, -1, -1, INS_invalid, INS_invalid, INS_vbmacor16x16x16, INS_vbmacor16x16x16, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX512BMM, BitMultiplyMatrix16x16WithXorReduction, -1, -1, INS_invalid, INS_invalid, INS_vbmacxor16x16x16, INS_vbmacxor16x16x16, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX512BMM, ReverseBits, -1, -1, INS_invalid, INS_vbitrev, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) #define LAST_NI_AVX512BMM NI_AVX512BMM_ReverseBits // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -701,8 +739,8 @@ HARDWARE_INTRINSIC(AVX512BMM, ReverseBits, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Intrinsics for AVXVNNI #define FIRST_NI_AVXVNNI NI_AVXVNNI_MultiplyWideningAndAdd -HARDWARE_INTRINSIC(AVXVNNI, MultiplyWideningAndAdd, -1, 3, INS_invalid, INS_vpdpbusd, INS_vpdpwssd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVXVNNI, MultiplyWideningAndAddSaturate, -1, 3, INS_invalid, INS_vpdpbusds, INS_vpdpwssds, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVXVNNI, MultiplyWideningAndAdd, -1, 3, INS_invalid, INS_vpdpbusd, INS_vpdpwssd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVXVNNI, MultiplyWideningAndAddSaturate, -1, 3, INS_invalid, INS_vpdpbusds, INS_vpdpwssds, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoEvexSemantics) #define LAST_NI_AVXVNNI NI_AVXVNNI_MultiplyWideningAndAddSaturate // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -711,8 +749,8 @@ HARDWARE_INTRINSIC(AVXVNNI, MultiplyWideningAndAddSaturate, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // AVXVNNIINT Intrinsics #define FIRST_NI_AVXVNNIINT NI_AVXVNNIINT_MultiplyWideningAndAdd -HARDWARE_INTRINSIC(AVXVNNIINT, MultiplyWideningAndAdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVXVNNIINT, MultiplyWideningAndAddSaturate, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVXVNNIINT, MultiplyWideningAndAdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVXVNNIINT, MultiplyWideningAndAddSaturate, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) #define LAST_NI_AVXVNNIINT NI_AVXVNNIINT_MultiplyWideningAndAddSaturate @@ -722,8 +760,8 @@ HARDWARE_INTRINSIC(AVXVNNIINT, MultiplyWideningAndAddSaturate, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // AVXVNNIINT_V512 Intrinsics #define FIRST_NI_AVXVNNIINT_V512 NI_AVXVNNIINT_V512_MultiplyWideningAndAdd -HARDWARE_INTRINSIC(AVXVNNIINT_V512, MultiplyWideningAndAdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVXVNNIINT_V512, MultiplyWideningAndAddSaturate, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVXVNNIINT_V512, MultiplyWideningAndAdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVXVNNIINT_V512, MultiplyWideningAndAddSaturate, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) #define LAST_NI_AVXVNNIINT_V512 NI_AVXVNNIINT_V512_MultiplyWideningAndAddSaturate // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -732,13 +770,13 @@ HARDWARE_INTRINSIC(AVXVNNIINT_V512, MultiplyWideningAndAddSaturate, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Intrinsics for AES, PCLMULQDQ #define FIRST_NI_AES NI_AES_CarrylessMultiply -HARDWARE_INTRINSIC(AES, CarrylessMultiply, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pclmulqdq, INS_pclmulqdq, INS_invalid, INS_invalid, 7, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AES, Decrypt, 16, 2, INS_invalid, INS_aesdec, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AES, DecryptLast, 16, 2, INS_invalid, INS_aesdeclast, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AES, Encrypt, 16, 2, INS_invalid, INS_aesenc, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AES, EncryptLast, 16, 2, INS_invalid, INS_aesenclast, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AES, InverseMixColumns, 16, 1, INS_invalid, INS_aesimc, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 8, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AES, KeygenAssist, 16, 2, INS_invalid, INS_aeskeygenassist, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 6, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AES, CarrylessMultiply, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pclmulqdq, INS_pclmulqdq, INS_invalid, INS_invalid, INS_invalid, 7, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AES, Decrypt, 16, 2, INS_invalid, INS_aesdec, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AES, DecryptLast, 16, 2, INS_invalid, INS_aesdeclast, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AES, Encrypt, 16, 2, INS_invalid, INS_aesenc, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AES, EncryptLast, 16, 2, INS_invalid, INS_aesenclast, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AES, InverseMixColumns, 16, 1, INS_invalid, INS_aesimc, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 8, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AES, KeygenAssist, 16, 2, INS_invalid, INS_aeskeygenassist, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 6, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) #define LAST_NI_AES NI_AES_KeygenAssist // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -747,7 +785,7 @@ HARDWARE_INTRINSIC(AES, KeygenAssist, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Intrinsics for VAES, VPCLMULQDQ #define FIRST_NI_AES_V256 NI_AES_V256_CarrylessMultiply -HARDWARE_INTRINSIC(AES_V256, CarrylessMultiply, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pclmulqdq, INS_pclmulqdq, INS_invalid, INS_invalid, 7, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AES_V256, CarrylessMultiply, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pclmulqdq, INS_pclmulqdq, INS_invalid, INS_invalid, INS_invalid, 7, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) #define LAST_NI_AES_V256 NI_AES_V256_CarrylessMultiply // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -756,7 +794,7 @@ HARDWARE_INTRINSIC(AES_V256, CarrylessMultiply, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // AVX Intrinsics for VAES, VPCLMULQDQ #define FIRST_NI_AES_V512 NI_AES_V512_CarrylessMultiply -HARDWARE_INTRINSIC(AES_V512, CarrylessMultiply, 64, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pclmulqdq, INS_pclmulqdq, INS_invalid, INS_invalid, 7, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AES_V512, CarrylessMultiply, 64, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pclmulqdq, INS_pclmulqdq, INS_invalid, INS_invalid, INS_invalid, 7, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) #define LAST_NI_AES_V512 NI_AES_V512_CarrylessMultiply // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -765,7 +803,7 @@ HARDWARE_INTRINSIC(AES_V512, CarrylessMultiply, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // AVX512 Intrinsics for X86Serialize #define FIRST_NI_X86Serialize NI_X86Serialize_Serialize -HARDWARE_INTRINSIC(X86Serialize, Serialize, 0, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Barrier) +HARDWARE_INTRINSIC(X86Serialize, Serialize, 0, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Barrier) #define LAST_NI_X86Serialize NI_X86Serialize_Serialize // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -774,9 +812,9 @@ HARDWARE_INTRINSIC(X86Serialize, Serialize, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Intrinsics for GFNI #define FIRST_NI_GFNI NI_GFNI_GaloisFieldAffineTransform -HARDWARE_INTRINSIC(GFNI, GaloisFieldAffineTransform, 16, 3, INS_invalid, INS_gf2p8affineqb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_gf2p8affineqb, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(GFNI, GaloisFieldAffineTransformInverse, 16, 3, INS_invalid, INS_gf2p8affineinvqb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_gf2p8affineinvqb, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(GFNI, GaloisFieldMultiply, 16, 2, INS_invalid, INS_gf2p8mulb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(GFNI, GaloisFieldAffineTransform, 16, 3, INS_invalid, INS_gf2p8affineqb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_gf2p8affineqb, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(GFNI, GaloisFieldAffineTransformInverse, 16, 3, INS_invalid, INS_gf2p8affineinvqb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_gf2p8affineinvqb, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(GFNI, GaloisFieldMultiply, 16, 2, INS_invalid, INS_gf2p8mulb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) #define LAST_NI_GFNI NI_GFNI_GaloisFieldMultiply // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -785,9 +823,9 @@ HARDWARE_INTRINSIC(GFNI, GaloisFieldMultiply, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // AVX Intrinsics for GFNI #define FIRST_NI_GFNI_V256 NI_GFNI_V256_GaloisFieldAffineTransform -HARDWARE_INTRINSIC(GFNI_V256, GaloisFieldAffineTransform, 32, 3, INS_invalid, INS_gf2p8affineqb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_gf2p8affineqb, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(GFNI_V256, GaloisFieldAffineTransformInverse, 32, 3, INS_invalid, INS_gf2p8affineinvqb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_gf2p8affineinvqb, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(GFNI_V256, GaloisFieldMultiply, 32, 2, INS_invalid, INS_gf2p8mulb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(GFNI_V256, GaloisFieldAffineTransform, 32, 3, INS_invalid, INS_gf2p8affineqb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_gf2p8affineqb, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(GFNI_V256, GaloisFieldAffineTransformInverse, 32, 3, INS_invalid, INS_gf2p8affineinvqb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_gf2p8affineinvqb, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(GFNI_V256, GaloisFieldMultiply, 32, 2, INS_invalid, INS_gf2p8mulb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) #define LAST_NI_GFNI_V256 NI_GFNI_V256_GaloisFieldMultiply // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -796,9 +834,9 @@ HARDWARE_INTRINSIC(GFNI_V256, GaloisFieldMultiply, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // AVX512 Intrinsics for GFNI #define FIRST_NI_GFNI_V512 NI_GFNI_V512_GaloisFieldAffineTransform -HARDWARE_INTRINSIC(GFNI_V512, GaloisFieldAffineTransform, 64, 3, INS_invalid, INS_gf2p8affineqb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_gf2p8affineqb, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(GFNI_V512, GaloisFieldAffineTransformInverse, 64, 3, INS_invalid, INS_gf2p8affineinvqb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_gf2p8affineinvqb, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(GFNI_V512, GaloisFieldMultiply, 64, 2, INS_invalid, INS_gf2p8mulb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(GFNI_V512, GaloisFieldAffineTransform, 64, 3, INS_invalid, INS_gf2p8affineqb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_gf2p8affineqb, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(GFNI_V512, GaloisFieldAffineTransformInverse, 64, 3, INS_invalid, INS_gf2p8affineinvqb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_gf2p8affineinvqb, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(GFNI_V512, GaloisFieldMultiply, 64, 2, INS_invalid, INS_gf2p8mulb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) #define LAST_NI_GFNI_V512 NI_GFNI_V512_GaloisFieldMultiply // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -806,53 +844,55 @@ HARDWARE_INTRINSIC(GFNI_V512, GaloisFieldMultiply, // TYP_BYTE, TYP_UBYTE, TYP_SHORT, TYP_USHORT, TYP_INT, TYP_UINT, TYP_LONG, TYP_ULONG, TYP_FLOAT, TYP_DOUBLE // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Special intrinsics that are generated during lowering -HARDWARE_INTRINSIC(X86Base, COMIS, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_comiss, INS_comisd, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, PTEST, 16, 2, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, UCOMIS, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ucomiss, INS_ucomisd, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(AVX, PTEST, -1, 2, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_vtestps, INS_vtestpd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, AndNotVector, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pandnd, INS_pandnd, INS_pandnd, INS_pandnd, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX2, AndNotScalar, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_andn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX512, KORTEST, 0, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_Special, HW_Flag_NoContainment) -HARDWARE_INTRINSIC(AVX512, KTEST, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_Special, HW_Flag_NoContainment) -HARDWARE_INTRINSIC(AVX512, PTESTM, 0, 2, INS_vptestmb, INS_vptestmb, INS_vptestmw, INS_vptestmw, INS_vptestmd, INS_vptestmd, INS_vptestmq, INS_vptestmq, INS_vptestmd, INS_vptestmq, 4, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX512, PTESTNM, 0, 2, INS_vptestnmb, INS_vptestnmb, INS_vptestnmw, INS_vptestnmw, INS_vptestnmd, INS_vptestnmd, INS_vptestnmq, INS_vptestnmq, INS_vptestnmd, INS_vptestnmq, 4, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(X86Base, COMIS, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_comiss, INS_comisd, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, PTEST, 16, 2, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, UCOMIS, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ucomiss, INS_ucomisd, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(AVX10v1, VCOMISH, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcomish, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(AVX10v1, VUCOMISH, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vucomish, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(AVX, PTEST, -1, 2, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_vtestps, INS_vtestpd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, AndNotVector, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pandnd, INS_pandnd, INS_pandnd, INS_pandnd, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX2, AndNotScalar, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_andn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX512, KORTEST, 0, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_Special, HW_Flag_NoContainment) +HARDWARE_INTRINSIC(AVX512, KTEST, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_Special, HW_Flag_NoContainment) +HARDWARE_INTRINSIC(AVX512, PTESTM, 0, 2, INS_vptestmb, INS_vptestmb, INS_vptestmw, INS_vptestmw, INS_vptestmd, INS_vptestmd, INS_vptestmq, INS_vptestmq, INS_vptestmd, INS_vptestmq, INS_invalid, 4, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX512, PTESTNM, 0, 2, INS_vptestnmb, INS_vptestnmb, INS_vptestnmw, INS_vptestnmw, INS_vptestnmd, INS_vptestnmd, INS_vptestnmq, INS_vptestnmq, INS_vptestnmd, INS_vptestnmq, INS_invalid, 4, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX512, AddMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, AndMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, AndNotMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, BlendVariableMask, -1, 3, INS_vpblendmb, INS_vpblendmb, INS_vpblendmw, INS_vpblendmw, INS_vpblendmd, INS_vpblendmd, INS_vpblendmq, INS_vpblendmq, INS_vblendmps, INS_vblendmpd, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, ClassifyMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfpclassps, INS_vfpclasspd, -1, 3, HW_Category_IMM, HW_Flag_ReturnsPerElementMask|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, ClassifyScalarMask, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfpclassss, INS_vfpclasssd, -1, 3, HW_Category_IMM, HW_Flag_ReturnsPerElementMask|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, CompareMask, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpps, INS_vcmppd, -1, 4, HW_Category_IMM, HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(AVX512, CompareEqualMask, -1, 2, INS_vpcmpeqb, INS_vpcmpeqb, INS_vpcmpeqw, INS_vpcmpeqw, INS_vpcmpeqd, INS_vpcmpeqd, INS_vpcmpeqq, INS_vpcmpeqq, INS_vcmpps, INS_vcmppd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AVX512, CompareGreaterThanMask, -1, 2, INS_vpcmpgtb, INS_vpcmpub, INS_vpcmpgtw, INS_vpcmpuw, INS_vpcmpgtd, INS_vpcmpud, INS_vpcmpgtq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(AVX512, CompareGreaterThanOrEqualMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(AVX512, CompareLessThanMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(AVX512, CompareLessThanOrEqualMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(AVX512, CompareNotEqualMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AVX512, CompareNotGreaterThanMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(AVX512, CompareNotGreaterThanOrEqualMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(AVX512, CompareNotLessThanMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(AVX512, CompareNotLessThanOrEqualMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(AVX512, CompareOrderedMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpps, INS_vcmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(AVX512, CompareScalarMask, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpss, INS_vcmpsd, -1, 4, HW_Category_IMM, HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(AVX512, CompareUnorderedMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpps, INS_vcmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(AVX512, CompressMask, -1, 3, INS_vpcompressb, INS_vpcompressb, INS_vpcompressw, INS_vpcompressw, INS_vpcompressd, INS_vpcompressd, INS_vpcompressq, INS_vpcompressq, INS_vcompressps, INS_vcompresspd, 3, 3, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(AVX512, CompressStoreMask, -1, 3, INS_vpcompressb, INS_vpcompressb, INS_vpcompressw, INS_vpcompressw, INS_vpcompressd, INS_vpcompressd, INS_vpcompressq, INS_vpcompressq, INS_vcompressps, INS_vcompresspd, -1, -1, HW_Category_MemoryStore, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, ConvertMaskToVector, -1, 1, INS_vpmovm2b, INS_vpmovm2b, INS_vpmovm2w, INS_vpmovm2w, INS_vpmovm2d, INS_vpmovm2d, INS_vpmovm2q, INS_vpmovm2q, INS_vpmovm2d, INS_vpmovm2q, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(AVX512, ConvertVectorToMask, -1, 1, INS_vpmovb2m, INS_vpmovb2m, INS_vpmovw2m, INS_vpmovw2m, INS_vpmovd2m, INS_vpmovd2m, INS_vpmovq2m, INS_vpmovq2m, INS_vpmovd2m, INS_vpmovq2m, 3, 3, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(AVX512, ExpandLoadMask, -1, 3, INS_vpexpandb, INS_vpexpandb, INS_vpexpandw, INS_vpexpandw, INS_vpexpandd, INS_vpexpandd, INS_vpexpandq, INS_vpexpandq, INS_vexpandps, INS_vexpandpd, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, ExpandMask, -1, 3, INS_vpexpandb, INS_vpexpandb, INS_vpexpandw, INS_vpexpandw, INS_vpexpandd, INS_vpexpandd, INS_vpexpandq, INS_vpexpandq, INS_vexpandps, INS_vexpandpd, 3, 3, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, MaskLoadMask, -1, 3, INS_vmovdqu8, INS_vmovdqu8, INS_vmovdqu16, INS_vmovdqu16, INS_movdqu32, INS_movdqu32, INS_vmovdqu64, INS_vmovdqu64, INS_movups, INS_movupd, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, MaskLoadAlignedMask, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_vmovdqa64, INS_vmovdqa64, INS_movaps, INS_movapd, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, MaskStoreMask, -1, 3, INS_vmovdqu8, INS_vmovdqu8, INS_vmovdqu16, INS_vmovdqu16, INS_movdqu32, INS_movdqu32, INS_vmovdqu64, INS_vmovdqu64, INS_movups, INS_movupd, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(AVX512, MaskStoreAlignedMask, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_vmovdqa64, INS_vmovdqa64, INS_movaps, INS_movapd, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(AVX512, NotMask, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, OrMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ShiftLeftMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ShiftRightMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, XorMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, XnorMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, AddMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, AndMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, AndNotMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, BlendVariableMask, -1, 3, INS_vpblendmb, INS_vpblendmb, INS_vpblendmw, INS_vpblendmw, INS_vpblendmd, INS_vpblendmd, INS_vpblendmq, INS_vpblendmq, INS_vblendmps, INS_vblendmpd, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, ClassifyMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfpclassps, INS_vfpclasspd, INS_invalid, -1, 3, HW_Category_IMM, HW_Flag_ReturnsPerElementMask|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, ClassifyScalarMask, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfpclassss, INS_vfpclasssd, INS_invalid, -1, 3, HW_Category_IMM, HW_Flag_ReturnsPerElementMask|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, CompareMask, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpps, INS_vcmppd, INS_invalid, -1, 4, HW_Category_IMM, HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(AVX512, CompareEqualMask, -1, 2, INS_vpcmpeqb, INS_vpcmpeqb, INS_vpcmpeqw, INS_vpcmpeqw, INS_vpcmpeqd, INS_vpcmpeqd, INS_vpcmpeqq, INS_vpcmpeqq, INS_vcmpps, INS_vcmppd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AVX512, CompareGreaterThanMask, -1, 2, INS_vpcmpgtb, INS_vpcmpub, INS_vpcmpgtw, INS_vpcmpuw, INS_vpcmpgtd, INS_vpcmpud, INS_vpcmpgtq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(AVX512, CompareGreaterThanOrEqualMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(AVX512, CompareLessThanMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(AVX512, CompareLessThanOrEqualMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(AVX512, CompareNotEqualMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AVX512, CompareNotGreaterThanMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(AVX512, CompareNotGreaterThanOrEqualMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(AVX512, CompareNotLessThanMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(AVX512, CompareNotLessThanOrEqualMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(AVX512, CompareOrderedMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpps, INS_vcmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(AVX512, CompareScalarMask, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpss, INS_vcmpsd, INS_invalid, -1, 4, HW_Category_IMM, HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(AVX512, CompareUnorderedMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpps, INS_vcmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(AVX512, CompressMask, -1, 3, INS_vpcompressb, INS_vpcompressb, INS_vpcompressw, INS_vpcompressw, INS_vpcompressd, INS_vpcompressd, INS_vpcompressq, INS_vpcompressq, INS_vcompressps, INS_vcompresspd, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(AVX512, CompressStoreMask, -1, 3, INS_vpcompressb, INS_vpcompressb, INS_vpcompressw, INS_vpcompressw, INS_vpcompressd, INS_vpcompressd, INS_vpcompressq, INS_vpcompressq, INS_vcompressps, INS_vcompresspd, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, ConvertMaskToVector, -1, 1, INS_vpmovm2b, INS_vpmovm2b, INS_vpmovm2w, INS_vpmovm2w, INS_vpmovm2d, INS_vpmovm2d, INS_vpmovm2q, INS_vpmovm2q, INS_vpmovm2d, INS_vpmovm2q, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(AVX512, ConvertVectorToMask, -1, 1, INS_vpmovb2m, INS_vpmovb2m, INS_vpmovw2m, INS_vpmovw2m, INS_vpmovd2m, INS_vpmovd2m, INS_vpmovq2m, INS_vpmovq2m, INS_vpmovd2m, INS_vpmovq2m, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(AVX512, ExpandLoadMask, -1, 3, INS_vpexpandb, INS_vpexpandb, INS_vpexpandw, INS_vpexpandw, INS_vpexpandd, INS_vpexpandd, INS_vpexpandq, INS_vpexpandq, INS_vexpandps, INS_vexpandpd, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, ExpandMask, -1, 3, INS_vpexpandb, INS_vpexpandb, INS_vpexpandw, INS_vpexpandw, INS_vpexpandd, INS_vpexpandd, INS_vpexpandq, INS_vpexpandq, INS_vexpandps, INS_vexpandpd, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, MaskLoadMask, -1, 3, INS_vmovdqu8, INS_vmovdqu8, INS_vmovdqu16, INS_vmovdqu16, INS_movdqu32, INS_movdqu32, INS_vmovdqu64, INS_vmovdqu64, INS_movups, INS_movupd, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, MaskLoadAlignedMask, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_vmovdqa64, INS_vmovdqa64, INS_movaps, INS_movapd, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, MaskStoreMask, -1, 3, INS_vmovdqu8, INS_vmovdqu8, INS_vmovdqu16, INS_vmovdqu16, INS_movdqu32, INS_movdqu32, INS_vmovdqu64, INS_vmovdqu64, INS_movups, INS_movupd, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(AVX512, MaskStoreAlignedMask, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_vmovdqa64, INS_vmovdqa64, INS_movaps, INS_movapd, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(AVX512, NotMask, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, OrMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ShiftLeftMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ShiftRightMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, XorMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, XnorMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) #endif // FEATURE_HW_INTRINSIC #undef HARDWARE_INTRINSIC diff --git a/src/coreclr/jit/namedintrinsiclist.h b/src/coreclr/jit/namedintrinsiclist.h index 0560d07c142b0b..47b5b66590f2df 100644 --- a/src/coreclr/jit/namedintrinsiclist.h +++ b/src/coreclr/jit/namedintrinsiclist.h @@ -25,7 +25,40 @@ enum NamedIntrinsic : unsigned short NI_System_SpanHelpers_Memmove, + NI_System_Half_FusedMultiplyAdd, + NI_System_Half_Max, + NI_System_Half_Min, + NI_System_Half_ReciprocalEstimate, + NI_System_Half_ReciprocalSqrtEstimate, + NI_System_Half_Round, + NI_System_Half_Sqrt, + + NI_System_Half_op_Addition, + NI_System_Half_op_Decrement, + NI_System_Half_op_Division, + NI_System_Half_op_Equality, NI_System_Half_op_Explicit, + NI_System_Half_op_GreaterThan, + NI_System_Half_op_GreaterThanOrEqual, + NI_System_Half_op_Increment, + NI_System_Half_op_Inequality, + NI_System_Half_op_LessThan, + NI_System_Half_op_LessThanOrEqual, + NI_System_Half_op_Multiply, + NI_System_Half_op_Subtraction, + + NI_System_Half_get_MinValue, + NI_System_Half_get_MaxValue, + NI_System_Half_get_Epsilon, + NI_System_Half_get_NaN, + NI_System_Half_get_PositiveInfinity, + NI_System_Half_get_NegativeInfinity, + NI_System_Half_get_One, + NI_System_Half_get_Zero, + + NI_System_Half_Ceiling, + NI_System_Half_Floor, + NI_System_Half_Truncate, NI_SYSTEM_MATH_START, NI_System_Math_Abs, @@ -184,7 +217,7 @@ enum NamedIntrinsic : unsigned short NI_HW_INTRINSIC_START, // clang-format off -#define HARDWARE_INTRINSIC(isa, name, simdSize, numArgs, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, intCost, fltCost, category, flag) \ +#define HARDWARE_INTRINSIC(isa, name, simdSize, numArgs, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, intCost, fltCost, category, flag) \ NI_##isa##_##name, #include "hwintrinsiclist.h" // clang-format on diff --git a/src/coreclr/jit/valuenumfuncs.h b/src/coreclr/jit/valuenumfuncs.h index 7ff916a2def4a7..84bd04b8a89b6d 100644 --- a/src/coreclr/jit/valuenumfuncs.h +++ b/src/coreclr/jit/valuenumfuncs.h @@ -192,8 +192,8 @@ ValueNumFuncDef(SimdType, 2, false, false) // A value number function to compos #if defined(FEATURE_HW_INTRINSICS) ValueNumFuncDef(HWI_INTRINSIC_START, -1, false, false) -#define HARDWARE_INTRINSIC(isa, name, simdSize, numArgs, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, intCost, fltCost, category, flag) \ -ValueNumFuncDef(HWI_##isa##_##name, ((numArgs == -1) ? -1 : (numArgs + 1)), ((flag) & HW_Flag_Commutative) >> 0, false) // All of the HARDWARE_INTRINSICS for x86/x64 +#define HARDWARE_INTRINSIC(isa, name, simdSize, numArgs, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, intCost, fltCost, category, flag) \ +ValueNumFuncDef(HWI_##isa##_##name, ((numArgs == -1) ? -1 : (numArgs + 1)), ((flag) & HW_Flag_Commutative) >> 0, false) // All of the HARDWARE_INTRINSICS #include "hwintrinsiclist.h" ValueNumFuncDef(HWI_INTRINSIC_END, -1, false, false) From 0e31957008f9b36a7d3139077c86f842d05c1533 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Thu, 9 Jul 2026 22:53:30 -0700 Subject: [PATCH 03/22] Wire up FP16 scalar instruction flags, EVEX map asserts, and perfscore Add INS_Flags_IsDstSrcSrcAVXInstruction to the FP16 scalar arithmetic and conversion instructions and INS_Flags_IsDstDstSrcAVXInstruction to vfmadd213sh, so the emitter encodes their EVEX vvvv src register. Fix the MAP5 escape on vminsh/vminph/vmovsh (were 0x00, must be 0x05). Add FIRST/LAST_AVX10V1_FMA_INSTR markers and recognize that range in Is3OpRmwInstruction. Relax the EVEX MAP5/MAP6 debug asserts to also accept AVX10v1, and add perfscore characteristics plus the missing PERFSCORE_THROUGHPUT_1P5X/4P5C macros. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/emit.h | 2 + src/coreclr/jit/emitxarch.cpp | 110 +++++++++++++++++++++++++++++++--- src/coreclr/jit/instrsxarch.h | 44 +++++++------- 3 files changed, 127 insertions(+), 29 deletions(-) diff --git a/src/coreclr/jit/emit.h b/src/coreclr/jit/emit.h index 780f06bd1b138c..b4d349853bef20 100644 --- a/src/coreclr/jit/emit.h +++ b/src/coreclr/jit/emit.h @@ -2075,12 +2075,14 @@ class emitter #define PERFSCORE_THROUGHPUT_4X 0.25f // Quad issue #define PERFSCORE_THROUGHPUT_3X (1.0f / 3.0f) // Three issue #define PERFSCORE_THROUGHPUT_2X 0.5f // Dual issue +#define PERFSCORE_THROUGHPUT_1P5X 0.67f // Dual issue #define PERFSCORE_THROUGHPUT_1C 1.0f // Single Issue #define PERFSCORE_THROUGHPUT_2C 2.0f // slower - 2 cycles #define PERFSCORE_THROUGHPUT_3C 3.0f // slower - 3 cycles #define PERFSCORE_THROUGHPUT_4C 4.0f // slower - 4 cycles +#define PERFSCORE_THROUGHPUT_4P5C 4.5f // slower - 4.5 cycles #define PERFSCORE_THROUGHPUT_5C 5.0f // slower - 5 cycles #define PERFSCORE_THROUGHPUT_6C 6.0f // slower - 6 cycles #define PERFSCORE_THROUGHPUT_7C 7.0f // slower - 7 cycles diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index 2d8b9328edcbb8..e6b11ff04a6458 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -129,7 +129,8 @@ bool emitter::Is3OpRmwInstruction(instruction ins) return ((ins >= FIRST_FMA_INSTRUCTION) && (ins <= LAST_FMA_INSTRUCTION)) || (IsAVXVNNIFamilyInstruction(ins)) || ((ins >= FIRST_AVX512BMM_INSTRUCTION) && (ins <= LAST_AVX512BMM_INSTRUCTION)) || - ((ins >= FIRST_AVXIFMA_INSTRUCTION) && (ins <= LAST_AVXIFMA_INSTRUCTION)); + ((ins >= FIRST_AVXIFMA_INSTRUCTION) && (ins <= LAST_AVXIFMA_INSTRUCTION)) || + ((ins >= FIRST_AVX10V1_FMA_INSTR) && (ins <= LAST_AVX10V1_FMA_INSTR)); } } } @@ -3185,7 +3186,8 @@ emitter::code_t emitter::emitExtractEvexPrefix(instruction ins, code_t& code) co // 1. An escape byte 0F (For isa before AVX10.2) // 2. A map number from 0 to 7 (For AVX10.2 and above) leadingBytes = check; - assert((leadingBytes == 0x0F) || ((m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX10v2) || + assert((leadingBytes == 0x0F) || ((m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX10v1) || + m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX10v2) || (m_compiler->compIsaSupportedDebugOnly(InstructionSet_APX))) && (leadingBytes >= 0x00) && (leadingBytes <= 0x07))); @@ -3212,7 +3214,8 @@ emitter::code_t emitter::emitExtractEvexPrefix(instruction ins, code_t& code) co // 0x0000RM11. leadingBytes = (code >> 16) & 0xFF; assert(leadingBytes == 0x0F || - ((m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX10v2) || + ((m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX10v1) || + m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX10v2) || m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX512BMM)) && leadingBytes >= 0x00 && leadingBytes <= 0x07) || (IsApxExtendedEvexInstruction(ins) && leadingBytes == 0)); @@ -3268,14 +3271,16 @@ emitter::code_t emitter::emitExtractEvexPrefix(instruction ins, code_t& code) co case 0x05: { - assert(m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX10v2)); + assert(m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX10v1) || + m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX10v2)); evexPrefix |= (0x05 << 16); break; } case 0x06: { - assert(m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX512BMM)); + assert(m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX10v1) || + m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX512BMM)); evexPrefix |= (0x6 << 16); break; } @@ -3872,7 +3877,8 @@ unsigned emitter::emitGetAdjustedSize(instrDesc* id, code_t code) const } emitAttr attr = id->idOpSize(); - if ((attr == EA_2BYTE) && (ins != INS_movzx) && (ins != INS_movsx) && !TakesApxExtendedEvexPrefix(id)) + if ((attr == EA_2BYTE) && (ins != INS_movzx) && (ins != INS_movsx) && !IsSimdInstruction(ins) && + !TakesApxExtendedEvexPrefix(id)) { // Most 16-bit operand instructions will need a 0x66 prefix. prefixAdjustedSize++; @@ -3891,7 +3897,7 @@ unsigned emitter::emitGetAdjustedSize(instrDesc* id, code_t code) const emitAttr attr = id->idOpSize(); - if ((attr == EA_2BYTE) && (ins != INS_movzx) && (ins != INS_movsx)) + if ((attr == EA_2BYTE) && (ins != INS_movzx) && (ins != INS_movsx) && !IsSimdInstruction(ins)) { // Most 16-bit operand instructions will need a 0x66 prefix. adjustedSize++; @@ -5503,7 +5509,7 @@ UNATIVE_OFFSET emitter::emitInsSizeAM(instrDesc* id, code_t code) assert((attrSize == EA_4BYTE) || (attrSize == EA_PTRSIZE) // Only for x64 || (attrSize == EA_16BYTE) || (attrSize == EA_32BYTE) || (attrSize == EA_64BYTE) // only for x64 - || (ins == INS_movzx) || (ins == INS_movsx) || + || (ins == INS_movzx) || (ins == INS_movsx) || (ins == INS_vmovsh) || (ins == INS_cmpxchg) // kmov instructions reach this path with EA_8BYTE size, even on x86 || IsKMOVInstruction(ins) @@ -7542,6 +7548,7 @@ bool emitter::IsMovInstruction(instruction ins) case INS_movq: case INS_movsd_simd: case INS_movss: + case INS_vmovsh: case INS_movsx: case INS_movupd: case INS_movups: @@ -7702,6 +7709,13 @@ bool emitter::HasSideEffect(instruction ins, emitAttr size) break; } + case INS_vmovsh: + { + // Clears the upper bits + hasSideEffect = true; + break; + } + case INS_movsx: case INS_movzx: { @@ -7978,6 +7992,7 @@ bool emitter::emitIns_Mov( case INS_vmovdqu64: case INS_movsd_simd: case INS_movss: + case INS_vmovsh: case INS_movupd: case INS_movups: { @@ -20753,6 +20768,85 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins break; } + case INS_vaddsh: + case INS_vsubsh: + case INS_vmulsh: + case INS_vfmadd213sh: + case INS_vmaxsh: + case INS_vminsh: + case INS_vcvtsh2ss: + { + insLatency = PERFSCORE_LATENCY_4C; + insThroughput = PERFSCORE_THROUGHPUT_2X; + break; + } + + case INS_vdivsh: + { + insLatency = PERFSCORE_LATENCY_14C; + insThroughput = PERFSCORE_THROUGHPUT_4C; + break; + } + + case INS_vsqrtsh: + { + insLatency = PERFSCORE_LATENCY_14C; + insThroughput = PERFSCORE_THROUGHPUT_4P5C; + break; + } + + case INS_vrsqrtsh: + case INS_vcomish: + case INS_vucomish: + case INS_vrcpsh: + { + insLatency = PERFSCORE_LATENCY_4C; + insThroughput = PERFSCORE_THROUGHPUT_1C; + break; + } + + case INS_vrndscalesh: + { + insLatency = PERFSCORE_LATENCY_8C; + insThroughput = PERFSCORE_THROUGHPUT_1C; + break; + } + + case INS_vcvtss2sh: + { + insLatency = PERFSCORE_LATENCY_6C; + insThroughput = PERFSCORE_THROUGHPUT_1P5X; + break; + } + + case INS_vcvtsd2sh: + { + insLatency = PERFSCORE_LATENCY_7C; + insThroughput = PERFSCORE_THROUGHPUT_1C; + break; + } + + case INS_vcvtsh2sd: + { + insLatency = PERFSCORE_LATENCY_10C; + insThroughput = PERFSCORE_THROUGHPUT_1C; + break; + } + + case INS_vcvtsi2sh32: + case INS_vcvtsi2sh64: + case INS_vcvtsh2si32: + case INS_vcvtsh2si64: + case INS_vcvtusi2sh32: + case INS_vcvtusi2sh64: + case INS_vcvtsh2usi32: + case INS_vcvtsh2usi64: + { + insLatency = PERFSCORE_LATENCY_7C; + insThroughput = PERFSCORE_THROUGHPUT_1C; + break; + } + case INS_vpmovdb: case INS_vpmovdw: case INS_vpmovqb: diff --git a/src/coreclr/jit/instrsxarch.h b/src/coreclr/jit/instrsxarch.h index 959aab27c0a5f9..503cafedda81d8 100644 --- a/src/coreclr/jit/instrsxarch.h +++ b/src/coreclr/jit/instrsxarch.h @@ -986,7 +986,7 @@ INST3(vpshufbitqmb, "vpshufbitqmb", IUM_WR, BAD_CODE, BAD_ // Instructions for AVX512-BF16, AVX512-FP16 INST3(vaddph, "vaddph", IUM_WR, BAD_CODE, BAD_CODE, PCKFLTMAP(0x05, 0x58), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Add Packed FP16 Values -INST3(vaddsh, "vaddsh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x58), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Add Scalar FP16 Values +INST3(vaddsh, "vaddsh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x58), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Add Scalar FP16 Values INST3(vcmpph, "vcmpph", IUM_WR, BAD_CODE, BAD_CODE, PSSE3A(0x00, 0xC2), 3C, 1C, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Compare Packed FP16 Values INST3(vcmpsh, "vcmpsh", IUM_WR, BAD_CODE, BAD_CODE, PSSE3A(0xF3, 0xC2), 3C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Compare Scalar FP16 Values INST3(vcomish, "vcomish", IUM_RD, BAD_CODE, BAD_CODE, PCKFLTMAP(0x05, 0x2F), 3C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | REX_W0 | Encoding_EVEX) // Compare Scalar Ordered FP16 Values and Set EFLAGS @@ -1004,16 +1004,16 @@ INST3(vcvtph2uw, "vcvtph2uw", IUM_WR, BAD_CODE, BAD_ INST3(vcvtph2w, "vcvtph2w", IUM_WR, BAD_CODE, BAD_CODE, PCKDBLMAP(0x05, 0x7D), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Convert Packed FP16 Values to Packed Signed WORD Integers INST3(vcvtps2phx, "vcvtps2phx", IUM_WR, BAD_CODE, BAD_CODE, PCKDBLMAP(0x05, 0x1D), ILLEGAL, ILLEGAL, INS_TT_FULL, Input_32Bit | KMask_Base4 | REX_W0 | Encoding_EVEX) // Convert Packed Single Precision FP Values to Packed FP16 Values INST3(vcvtqq2ph, "vcvtqq2ph", IUM_WR, BAD_CODE, BAD_CODE, PCKFLTMAP(0x05, 0x5B), ILLEGAL, ILLEGAL, INS_TT_FULL, Input_64Bit | KMask_Base2 | REX_W1 | Encoding_EVEX) // Convert Packed Signed QWORD Integers to Packed FP16 Values -INST3(vcvtsd2sh, "vcvtsd2sh", IUM_WR, BAD_CODE, BAD_CODE, SSEDBLMAP(0x05, 0x5A), 7C, 1C, INS_TT_TUPLE1_SCALAR, Input_64Bit | KMask_Base1 | REX_W1 | Encoding_EVEX) // Convert Scalar Double Precision FP Value to Scalar FP16 Value -INST3(vcvtsh2sd, "vcvtsh2sd", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x5A), 7C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Convert Scalar FP16 Value to Scalar Double Precision FP Value +INST3(vcvtsd2sh, "vcvtsd2sh", IUM_WR, BAD_CODE, BAD_CODE, SSEDBLMAP(0x05, 0x5A), 7C, 1C, INS_TT_TUPLE1_SCALAR, Input_64Bit | KMask_Base1 | REX_W1 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Convert Scalar Double Precision FP Value to Scalar FP16 Value +INST3(vcvtsh2sd, "vcvtsh2sd", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x5A), 7C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Convert Scalar FP16 Value to Scalar Double Precision FP Value INST3(vcvtsh2si32, "vcvtsh2si", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x2D), 8C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Convert Scalar FP16 Value to Scalar Signed DWORD Integer INST3(vcvtsh2si64, "vcvtsh2si", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x2D), 8C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W1 | Encoding_EVEX) // Convert Scalar FP16 Value to Scalar Signed QWORD Integer -INST3(vcvtsh2ss, "vcvtsh2ss", IUM_WR, BAD_CODE, BAD_CODE, PCKFLTMAP(0x06, 0x13), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Convert Scalar FP16 Value to Scalar Single Precision FP Value +INST3(vcvtsh2ss, "vcvtsh2ss", IUM_WR, BAD_CODE, BAD_CODE, PCKFLTMAP(0x06, 0x13), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Convert Scalar FP16 Value to Scalar Single Precision FP Value INST3(vcvtsh2usi32, "vcvtsh2usi", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x79), 7C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Convert Scalar FP16 Value to Scalar Unsigned DWORD Integer INST3(vcvtsh2usi64, "vcvtsh2usi", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x79), 7C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W1 | Encoding_EVEX) // Convert Scalar FP16 Value to Scalar Unsigned QWORD Integer -INST3(vcvtsi2sh32, "vcvtsi2sh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x2A), 7C, 1C, INS_TT_TUPLE1_SCALAR, Input_32Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Convert Scalar Signed DWORD Integer to Scalar FP16 Value -INST3(vcvtsi2sh64, "vcvtsi2sh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x2A), 7C, 1C, INS_TT_TUPLE1_SCALAR, Input_64Bit | KMask_Base1 | REX_W1 | Encoding_EVEX) // Convert Scalar Signed QWORD Integer to Scalar FP16 Value -INST3(vcvtss2sh, "vcvtss2sh", IUM_WR, BAD_CODE, BAD_CODE, PCKFLTMAP(0x05, 0x1D), 4C, 1C, INS_TT_TUPLE1_SCALAR, Input_32Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Convert Scalar Single Precision FP Value to Scalar FP16 Value +INST3(vcvtsi2sh32, "vcvtsi2sh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x2A), 7C, 1C, INS_TT_TUPLE1_SCALAR, Input_32Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Convert Scalar Signed DWORD Integer to Scalar FP16 Value +INST3(vcvtsi2sh64, "vcvtsi2sh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x2A), 7C, 1C, INS_TT_TUPLE1_SCALAR, Input_64Bit | KMask_Base1 | REX_W1 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Convert Scalar Signed QWORD Integer to Scalar FP16 Value +INST3(vcvtss2sh, "vcvtss2sh", IUM_WR, BAD_CODE, BAD_CODE, PCKFLTMAP(0x05, 0x1D), 4C, 1C, INS_TT_TUPLE1_SCALAR, Input_32Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Convert Scalar Single Precision FP Value to Scalar FP16 Value INST3(vcvttph2dq, "vcvttph2dq", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x5B), 8C, 1C, INS_TT_HALF_MEM, Input_16Bit | KMask_Base4 | REX_W0 | Encoding_EVEX) // Convert with Truncation Packed FP16 Values to Packed Signed DWORD Integers INST3(vcvttph2qq, "vcvttph2qq", IUM_WR, BAD_CODE, BAD_CODE, PCKDBLMAP(0x05, 0x7A), ILLEGAL, ILLEGAL, INS_TT_QUARTER_MEM, Input_16Bit | KMask_Base2 | REX_W0 | Encoding_EVEX) // Convert with Truncation Packed FP16 Values to Packed Signed QWORD Integers INST3(vcvttph2udq, "vcvttph2udq", IUM_WR, BAD_CODE, BAD_CODE, PCKFLTMAP(0x05, 0x78), 8C, 1C, INS_TT_HALF_MEM, Input_16Bit | KMask_Base4 | REX_W0 | Encoding_EVEX) // Convert with Truncation Packed FP16 Values to Packed Unsigned DWORD Integers @@ -1026,21 +1026,22 @@ INST3(vcvttsh2usi32, "vcvttsh2usi", IUM_WR, BAD_CODE, BAD_ INST3(vcvttsh2usi64, "vcvttsh2usi", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x78), 8C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W1 | Encoding_EVEX) // Convert with Truncation Scalar FP16 Value to Scalar Unsigned QWORD Integer INST3(vcvtudq2ph, "vcvtudq2ph", IUM_WR, BAD_CODE, BAD_CODE, SSEDBLMAP(0x05, 0x7A), ILLEGAL, ILLEGAL, INS_TT_FULL, Input_32Bit | KMask_Base4 | REX_W0 | Encoding_EVEX) // Convert Packed Single Precision FP Values to Packed FP16 Values INST3(vcvtuqq2ph, "vcvtuqq2ph", IUM_WR, BAD_CODE, BAD_CODE, SSEDBLMAP(0x05, 0x7A), ILLEGAL, ILLEGAL, INS_TT_FULL, Input_64Bit | KMask_Base2 | REX_W1 | Encoding_EVEX) // Convert Packed Single Precision FP Values to Packed FP16 Values -INST3(vcvtusi2sh32, "vcvtusi2sh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x7B), 7C, 1C, INS_TT_TUPLE1_SCALAR, Input_32Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Convert Scalar Unsigned DWORD Integer to Scalar FP16 Value -INST3(vcvtusi2sh64, "vcvtusi2sh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x7B), 7C, 1C, INS_TT_TUPLE1_SCALAR, Input_64Bit | KMask_Base1 | REX_W1 | Encoding_EVEX) // Convert Scalar Unsigned QWORD Integer to Scalar FP16 Value +INST3(vcvtusi2sh32, "vcvtusi2sh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x7B), 7C, 1C, INS_TT_TUPLE1_SCALAR, Input_32Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Convert Scalar Unsigned DWORD Integer to Scalar FP16 Value +INST3(vcvtusi2sh64, "vcvtusi2sh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x7B), 7C, 1C, INS_TT_TUPLE1_SCALAR, Input_64Bit | KMask_Base1 | REX_W1 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Convert Scalar Unsigned QWORD Integer to Scalar FP16 Value INST3(vcvtuw2ph, "vcvtuw2ph", IUM_WR, BAD_CODE, BAD_CODE, SSEDBLMAP(0x05, 0x7D), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Convert Packed Single Precision FP Values to Packed FP16 Values INST3(vcvtw2ph, "vcvtw2ph", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x7D), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Convert Packed Single Precision FP Values to Packed FP16 Values INST3(vdivph, "vdivph", IUM_WR, BAD_CODE, BAD_CODE, PCKFLTMAP(0x05, 0x5E), ILLEGAL, ILLEGAL, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Divide Packed FP16 Values -INST3(vdivsh, "vdivsh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x5E), ILLEGAL, ILLEGAL, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Divide Scalar FP16 Values +INST3(vdivsh, "vdivsh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x5E), ILLEGAL, ILLEGAL, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Divide Scalar FP16 Values INST3(vdpbf16ps, "vdpbf16ps", IUM_WR, BAD_CODE, BAD_CODE, PSSE38(0xF3, 0x52), 8C, 2C, INS_TT_FULL, Input_16Bit | KMask_Base4 | REX_W0 | Encoding_EVEX) // Dot Product of BF16 Pairs Accumulated Into Packed Single Precision INST3(vfcmaddcph, "vfcmaddcph", IUM_RW, BAD_CODE, BAD_CODE, SSEDBLMAP(0x06, 0x56), 8C, 1C, INS_TT_FULL, Input_32Bit | KMask_Base4 | REX_W0 | Encoding_EVEX) // Complex Multiply and Accumulate Packed FP16 Values INST3(vfcmaddcsh, "vfcmaddcsh", IUM_RW, BAD_CODE, BAD_CODE, SSEDBLMAP(0x06, 0x57), 8C, 1C, INS_TT_TUPLE1_SCALAR, Input_32Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Complex Multiply and Accumulate Scalar FP16 Values INST3(vfcmulcph, "vfcmulcph", IUM_WR, BAD_CODE, BAD_CODE, SSEDBLMAP(0x06, 0xD6), 8C, 1C, INS_TT_FULL, Input_32Bit | KMask_Base4 | REX_W0 | Encoding_EVEX) // Complex Multiply Packed FP16 Values INST3(vfcmulcsh, "vfcmulcsh", IUM_WR, BAD_CODE, BAD_CODE, SSEDBLMAP(0x06, 0xD7), 8C, 1C, INS_TT_TUPLE1_SCALAR, Input_32Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Complex Multiply Scalar FP16 Values +#define FIRST_AVX10V1_FMA_INSTR INS_vfmadd132ph INST3(vfmadd132ph, "vvfmadd132ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x98), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Multiply-Add of Packed FP16 Values INST3(vfmadd132sh, "vvfmadd132sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x99), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Multiply-Add of Scalar FP16 Values INST3(vfmadd213ph, "vvfmadd213ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xA8), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Multiply-Add of Packed FP16 Values -INST3(vfmadd213sh, "vvfmadd213sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xA9), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Multiply-Add of Scalar FP16 Values +INST3(vfmadd213sh, "vvfmadd213sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xA9), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstDstSrcAVXInstruction) // Fused Multiply-Add of Scalar FP16 Values INST3(vfmadd231ph, "vvfmadd231ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xB8), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Multiply-Add of Packed FP16 Values INST3(vfmadd231sh, "vvfmadd231sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xB9), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Multiply-Add of Scalar FP16 Values INST3(vfmaddcph, "vfmaddcph", IUM_RW, BAD_CODE, BAD_CODE, SSEFLTMAP(0x06, 0x56), 8C, 1C, INS_TT_FULL, Input_32Bit | KMask_Base4 | REX_W0 | Encoding_EVEX) // Complex Multiply and Accumulate Packed FP16 Values @@ -1071,6 +1072,7 @@ INST3(vfnmsub213ph, "vvfnmsub213ph", IUM_RW, BAD_CODE, BAD_ INST3(vfnmsub213sh, "vvfnmsub213sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xAF), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Subtract of Scalar FP16 Values INST3(vfnmsub231ph, "vvfnmsub231ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xBE), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Subtract of Packed FP16 Values INST3(vfnmsub231sh, "vvfnmsub231sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xBF), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Subtract of Scalar FP16 Values +#define LAST_AVX10V1_FMA_INSTR INS_vfnmsub231sh INST3(vfpclassph, "vfpclassph", IUM_WR, BAD_CODE, BAD_CODE, PSSE3A(0x00, 0x66), 3C, 1C, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Test Types of Packed FP16 Values INST3(vfpclasssh, "vfpclasssh", IUM_WR, BAD_CODE, BAD_CODE, PSSE3A(0x00, 0x67), 3C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Test Types of Scalar FP16 Values INST3(vgetexpph, "vgetexpph", IUM_WR, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x42), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Convert Exponents of Packed FP16 Values to FP16 Values @@ -1078,27 +1080,27 @@ INST3(vgetexpsh, "vgetexpsh", IUM_WR, BAD_CODE, BAD_ INST3(vgetmantph, "vgetmantph", IUM_WR, BAD_CODE, BAD_CODE, PSSE3A(0x00, 0x26), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Extract Normalized Mantissas from Packed FP16 Values INST3(vgetmantsh, "vgetmantsh", IUM_WR, BAD_CODE, BAD_CODE, PSSE3A(0x00, 0x27), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Extract Normalized Mantissas from Scalar FP16 Values INST3(vmaxph, "vmaxph", IUM_WR, BAD_CODE, BAD_CODE, PCKFLTMAP(0x05, 0x5F), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Return Maximum of Packed FP16 Values -INST3(vmaxsh, "vmaxsh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x5F), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Return Maximum of Scalar FP16 Values -INST3(vminsh, "vminsh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x00, 0x5D), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Return Minimum of Scalar FP16 Values -INST3(vminph, "vminph", IUM_WR, BAD_CODE, BAD_CODE, PCKFLTMAP(0x00, 0x5D), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Return Minimum of Packed FP16 Values -INST3(vmovsh, "vmovsh", IUM_WR, SSEFLTMAP(0x00, 0x11), BAD_CODE, SSEFLTMAP(0x00, 0x10), ILLEGAL, ILLEGAL, INS_TT_TUPLE1_SCALAR, Input_16Bit | REX_W0 | Encoding_EVEX) // Move Scalar FP16 Value +INST3(vmaxsh, "vmaxsh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x5F), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Return Maximum of Scalar FP16 Values +INST3(vminsh, "vminsh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x5D), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Return Minimum of Scalar FP16 Values +INST3(vminph, "vminph", IUM_WR, BAD_CODE, BAD_CODE, PCKFLTMAP(0x05, 0x5D), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Return Minimum of Packed FP16 Values +INST3(vmovsh, "vmovsh", IUM_WR, SSEFLTMAP(0x05, 0x11), BAD_CODE, SSEFLTMAP(0x05, 0x10), ILLEGAL, ILLEGAL, INS_TT_TUPLE1_SCALAR, Input_16Bit | REX_W0 | Encoding_EVEX) // Move Scalar FP16 Value INST3(vmovw, "vmovw", IUM_WR, PCKDBLMAP(0x06, 0x7E), BAD_CODE, PCKDBLMAP(0x00, 0x6E), ILLEGAL, ILLEGAL, INS_TT_TUPLE1_SCALAR, Input_16Bit | REX_WIG | Encoding_EVEX) // Move Word INST3(vmulph, "vmulph", IUM_WR, BAD_CODE, BAD_CODE, PCKFLTMAP(0x05, 0x59), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Multiply Packed FP16 Values -INST3(vmulsh, "vmulsh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x59), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Multiply Scalar FP16 Values +INST3(vmulsh, "vmulsh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x59), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Multiply Scalar FP16 Values INST3(vrcpph, "vrcpph", IUM_WR, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x4C), ILLEGAL, ILLEGAL, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Compute REciprocals of Packed FP16 Values -INST3(vrcpsh, "vrcpsh", IUM_WR, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x4D), 4C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Compute REciprocals of Scalar FP16 Values +INST3(vrcpsh, "vrcpsh", IUM_WR, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x4D), 4C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Compute REciprocals of Scalar FP16 Values INST3(vreduceph, "vreduceph", IUM_WR, BAD_CODE, BAD_CODE, PSSE3A(0x00, 0x56), 12C, 2C, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Perform Reduction Transformation on Packed FP16 Values INST3(vreducesh, "vreducesh", IUM_WR, BAD_CODE, BAD_CODE, PSSE3A(0x00, 0x57), 12C, 2C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Perform Reduction Transformation on Scalar FP16 Values INST3(vrndscaleph, "vrndscaleph", IUM_WR, BAD_CODE, BAD_CODE, PSSE3A(0x00, 0x08), 8C, 1C, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Round Packed FP16 Values to Include a Given Number of Fraction Bits -INST3(vrndscalesh, "vrndscalesh", IUM_WR, BAD_CODE, BAD_CODE, PSSE3A(0x00, 0x0A), 8C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Round Scalar FP16 Values to Include a Given Number of Fraction Bits +INST3(vrndscalesh, "vrndscalesh", IUM_WR, BAD_CODE, BAD_CODE, PSSE3A(0x00, 0x0A), 8C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Round Scalar FP16 Values to Include a Given Number of Fraction Bits INST3(vrsqrtph, "vrsqrtph", IUM_WR, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x4E), ILLEGAL, ILLEGAL, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Compute REciprocals of Square Roots of Packed FP16 Values -INST3(vrsqrtsh, "vrsqrtsh", IUM_WR, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x4F), 5C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Compute REciprocals of Square Roots of Scalar FP16 Values +INST3(vrsqrtsh, "vrsqrtsh", IUM_WR, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x4F), 5C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Compute REciprocals of Square Roots of Scalar FP16 Values INST3(vscalefph, "vscalefph", IUM_WR, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x2C), 8C, 1C, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Scale Packed FP16 Values with FP16 Values INST3(vscalefsh, "vscalefsh", IUM_WR, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x2D), 8C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Scale Scalar FP16 Values with FP16 Values INST3(vsqrtph, "vsqrtph", IUM_WR, BAD_CODE, BAD_CODE, PCKFLTMAP(0x05, 0x51), ILLEGAL, ILLEGAL, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Compute Square Root of Packed FP16 Values -INST3(vsqrtsh, "vsqrtsh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x51), 15C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Compute Square Root of Scalar FP16 Values +INST3(vsqrtsh, "vsqrtsh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x51), 15C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Compute Square Root of Scalar FP16 Values INST3(vsubph, "vsubph", IUM_WR, BAD_CODE, BAD_CODE, PCKFLTMAP(0x05, 0x5C), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Subtract Packed FP16 Values -INST3(vsubsh, "vsubsh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x5C), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Subtract Scalar FP16 Values +INST3(vsubsh, "vsubsh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x5C), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Subtract Scalar FP16 Values INST3(vucomish, "vucomish", IUM_RD, BAD_CODE, BAD_CODE, PCKFLTMAP(0x05, 0x2E), 3C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | REX_W0 | Encoding_EVEX) // Compare Scalar Unordered FP16 Values and Set EFLAGS // AVX512-VP2INTERSECT From 36b6796d4305554d70e85619fc6b123197f82044 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Thu, 9 Jul 2026 23:15:34 -0700 Subject: [PATCH 04/22] Wire up FP16 scalar codegen, lowering, and LSRA for AVX10v1 Group the FP16 FMA instruction families so genFmaIntrinsic's 132/231 form derivation stays valid, dispatch AVX10v1 through genAvxFamilyIntrinsic and the FMA jumptable fallback, lower the Half compare-scalar intrinsics to VCOMISH/ VUCOMISH, contain RoundScaleScalar, and build FusedMultiplyAddScalar as an FMA. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/hwintrinsiccodegenxarch.cpp | 2 + src/coreclr/jit/instrsxarch.h | 16 ++++---- src/coreclr/jit/lowerxarch.cpp | 42 +++++++++++++++++++++ src/coreclr/jit/lsraxarch.cpp | 1 + 4 files changed, 53 insertions(+), 8 deletions(-) diff --git a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp index 3e8c533ff6c825..c18175fd8f0dc1 100644 --- a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp @@ -1036,6 +1036,7 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) case InstructionSet_AVX512: case InstructionSet_AVX512_X64: case InstructionSet_AVX512v2: + case InstructionSet_AVX10v1: case InstructionSet_AVX10v2: case InstructionSet_AVX10v2_X64: case InstructionSet_AVXVNNIINT: @@ -1890,6 +1891,7 @@ void CodeGen::genNonTableDrivenHWIntrinsicsJumpTableFallback(GenTreeHWIntrinsic* case NI_AVX512_FusedMultiplyAdd: case NI_AVX512_FusedMultiplyAddScalar: + case NI_AVX10v1_FusedMultiplyAddScalar: case NI_AVX512_FusedMultiplyAddNegated: case NI_AVX512_FusedMultiplyAddNegatedScalar: case NI_AVX512_FusedMultiplyAddSubtract: diff --git a/src/coreclr/jit/instrsxarch.h b/src/coreclr/jit/instrsxarch.h index 503cafedda81d8..37524439b16311 100644 --- a/src/coreclr/jit/instrsxarch.h +++ b/src/coreclr/jit/instrsxarch.h @@ -1039,10 +1039,10 @@ INST3(vfcmulcph, "vfcmulcph", IUM_WR, BAD_CODE, BAD_ INST3(vfcmulcsh, "vfcmulcsh", IUM_WR, BAD_CODE, BAD_CODE, SSEDBLMAP(0x06, 0xD7), 8C, 1C, INS_TT_TUPLE1_SCALAR, Input_32Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Complex Multiply Scalar FP16 Values #define FIRST_AVX10V1_FMA_INSTR INS_vfmadd132ph INST3(vfmadd132ph, "vvfmadd132ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x98), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Multiply-Add of Packed FP16 Values -INST3(vfmadd132sh, "vvfmadd132sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x99), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Multiply-Add of Scalar FP16 Values INST3(vfmadd213ph, "vvfmadd213ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xA8), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Multiply-Add of Packed FP16 Values -INST3(vfmadd213sh, "vvfmadd213sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xA9), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstDstSrcAVXInstruction) // Fused Multiply-Add of Scalar FP16 Values INST3(vfmadd231ph, "vvfmadd231ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xB8), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Multiply-Add of Packed FP16 Values +INST3(vfmadd132sh, "vvfmadd132sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x99), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Multiply-Add of Scalar FP16 Values +INST3(vfmadd213sh, "vvfmadd213sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xA9), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstDstSrcAVXInstruction) // Fused Multiply-Add of Scalar FP16 Values INST3(vfmadd231sh, "vvfmadd231sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xB9), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Multiply-Add of Scalar FP16 Values INST3(vfmaddcph, "vfmaddcph", IUM_RW, BAD_CODE, BAD_CODE, SSEFLTMAP(0x06, 0x56), 8C, 1C, INS_TT_FULL, Input_32Bit | KMask_Base4 | REX_W0 | Encoding_EVEX) // Complex Multiply and Accumulate Packed FP16 Values INST3(vfmaddcsh, "vfmaddcsh", IUM_RW, BAD_CODE, BAD_CODE, SSEFLTMAP(0x06, 0x57), 8C, 1C, INS_TT_TUPLE1_SCALAR, Input_32Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Complex Multiply and Accumulate Scalar FP16 Values @@ -1050,10 +1050,10 @@ INST3(vfmaddsub132ph, "vvfmaddsub132ph", IUM_RW, BAD_CODE, BAD_ INST3(vfmaddsub213ph, "vvfmaddsub213ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xA6), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Multiply-Alternating Add/Subtract of Packed FP16 Values INST3(vfmaddsub231ph, "vvfmaddsub231ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xB6), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Multiply-Alternating Add/Subtract of Packed FP16 Values INST3(vfmsub132ph, "vvfmsub132ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x9A), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Multiply-Subtract of Packed FP16 Values -INST3(vfmsub132sh, "vvfmsub132sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x9B), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Multiply-Subtract of Scalar FP16 Values INST3(vfmsub213ph, "vvfmsub213ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xAA), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Multiply-Subtract of Packed FP16 Values -INST3(vfmsub213sh, "vvfmsub213sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xAB), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Multiply-Subtract of Scalar FP16 Values INST3(vfmsub231ph, "vvfmsub231ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xBA), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Multiply-Subtract of Packed FP16 Values +INST3(vfmsub132sh, "vvfmsub132sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x9B), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Multiply-Subtract of Scalar FP16 Values +INST3(vfmsub213sh, "vvfmsub213sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xAB), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Multiply-Subtract of Scalar FP16 Values INST3(vfmsub231sh, "vvfmsub231sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xBB), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Multiply-Subtract of Scalar FP16 Values INST3(vfmsubadd132ph, "vvfmsubadd132ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x97), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Multiply-Alternating Subtract/Add of Packed FP16 Values INST3(vfmsubadd213ph, "vvfmsubadd213ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xA7), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Multiply-Alternating Subtract/Add of Packed FP16 Values @@ -1061,16 +1061,16 @@ INST3(vfmsubadd231ph, "vvfmsubadd231ph", IUM_RW, BAD_CODE, BAD_ INST3(vfmulcph, "vfmulcph", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x06, 0xD6), 8C, 1C, INS_TT_FULL, Input_32Bit | KMask_Base4 | REX_W0 | Encoding_EVEX) // Complex Multiply Packed FP16 Values INST3(vfmulcsh, "vfmulcsh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x06, 0xD7), 8C, 1C, INS_TT_TUPLE1_SCALAR, Input_32Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Complex Multiply Scalar FP16 Values INST3(vfnmadd132ph, "vvfnmadd132ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x9C), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Add of Packed FP16 Values -INST3(vfnmadd132sh, "vvfnmadd132sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x9D), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Add of Scalar FP16 Values INST3(vfnmadd213ph, "vvfnmadd213ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xAC), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Add of Packed FP16 Values -INST3(vfnmadd213sh, "vvfnmadd213sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xAD), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Add of Scalar FP16 Values INST3(vfnmadd231ph, "vvfnmadd231ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xBC), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Add of Packed FP16 Values +INST3(vfnmadd132sh, "vvfnmadd132sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x9D), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Add of Scalar FP16 Values +INST3(vfnmadd213sh, "vvfnmadd213sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xAD), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Add of Scalar FP16 Values INST3(vfnmadd231sh, "vvfnmadd231sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xBD), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Add of Scalar FP16 Values INST3(vfnmsub132ph, "vvfnmsub132ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x9E), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Subtract of Packed FP16 Values -INST3(vfnmsub132sh, "vvfnmsub132sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x9F), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Subtract of Scalar FP16 Values INST3(vfnmsub213ph, "vvfnmsub213ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xAE), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Subtract of Packed FP16 Values -INST3(vfnmsub213sh, "vvfnmsub213sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xAF), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Subtract of Scalar FP16 Values INST3(vfnmsub231ph, "vvfnmsub231ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xBE), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Subtract of Packed FP16 Values +INST3(vfnmsub132sh, "vvfnmsub132sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x9F), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Subtract of Scalar FP16 Values +INST3(vfnmsub213sh, "vvfnmsub213sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xAF), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Subtract of Scalar FP16 Values INST3(vfnmsub231sh, "vvfnmsub231sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xBF), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Subtract of Scalar FP16 Values #define LAST_AVX10V1_FMA_INSTR INS_vfnmsub231sh INST3(vfpclassph, "vfpclassph", IUM_WR, BAD_CODE, BAD_CODE, PSSE3A(0x00, 0x66), 3C, 1C, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Test Types of Packed FP16 Values diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index 0359b806c6e1bc..e4b41d92fa57b7 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -1113,6 +1113,8 @@ void Lowering::LowerHWIntrinsicCC(GenTreeHWIntrinsic* node, NamedIntrinsic newIn { case NI_X86Base_COMIS: case NI_X86Base_UCOMIS: + case NI_AVX10v1_VCOMISH: + case NI_AVX10v1_VUCOMISH: // In some cases we can generate better code if we swap the operands: // - If the condition is not one of the "preferred" floating point conditions we can swap // the operands and change the condition to avoid generating an extra JP/JNP branch. @@ -2380,6 +2382,44 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) LowerHWIntrinsicCC(node, NI_X86Base_UCOMIS, GenCondition::FGT); break; + case NI_AVX10v1_CompareScalarOrderedEqual: + LowerHWIntrinsicCC(node, NI_AVX10v1_VCOMISH, GenCondition::FEQ); + break; + case NI_AVX10v1_CompareScalarOrderedNotEqual: + LowerHWIntrinsicCC(node, NI_AVX10v1_VCOMISH, GenCondition::FNEU); + break; + case NI_AVX10v1_CompareScalarOrderedLessThan: + LowerHWIntrinsicCC(node, NI_AVX10v1_VCOMISH, GenCondition::FLT); + break; + case NI_AVX10v1_CompareScalarOrderedLessThanOrEqual: + LowerHWIntrinsicCC(node, NI_AVX10v1_VCOMISH, GenCondition::FLE); + break; + case NI_AVX10v1_CompareScalarOrderedGreaterThan: + LowerHWIntrinsicCC(node, NI_AVX10v1_VCOMISH, GenCondition::FGT); + break; + case NI_AVX10v1_CompareScalarOrderedGreaterThanOrEqual: + LowerHWIntrinsicCC(node, NI_AVX10v1_VCOMISH, GenCondition::FGE); + break; + + case NI_AVX10v1_CompareScalarUnorderedEqual: + LowerHWIntrinsicCC(node, NI_AVX10v1_VUCOMISH, GenCondition::FEQ); + break; + case NI_AVX10v1_CompareScalarUnorderedNotEqual: + LowerHWIntrinsicCC(node, NI_AVX10v1_VUCOMISH, GenCondition::FNEU); + break; + case NI_AVX10v1_CompareScalarUnorderedLessThan: + LowerHWIntrinsicCC(node, NI_AVX10v1_VUCOMISH, GenCondition::FLT); + break; + case NI_AVX10v1_CompareScalarUnorderedLessThanOrEqual: + LowerHWIntrinsicCC(node, NI_AVX10v1_VUCOMISH, GenCondition::FLE); + break; + case NI_AVX10v1_CompareScalarUnorderedGreaterThan: + LowerHWIntrinsicCC(node, NI_AVX10v1_VUCOMISH, GenCondition::FGT); + break; + case NI_AVX10v1_CompareScalarUnorderedGreaterThanOrEqual: + LowerHWIntrinsicCC(node, NI_AVX10v1_VUCOMISH, GenCondition::FGE); + break; + case NI_X86Base_TestC: LowerHWIntrinsicCC(node, NI_X86Base_PTEST, GenCondition::C); break; @@ -9535,6 +9575,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) case NI_AVX512_GetMantissaScalar: case NI_AVX512_RoundScaleScalar: case NI_AVX512_ReduceScalar: + case NI_AVX10v1_RoundScaleScalar: { // These intrinsics have both 2 and 3-operand overloads. // @@ -10224,6 +10265,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) case NI_AES_CarrylessMultiply: case NI_AES_V256_CarrylessMultiply: case NI_AES_V512_CarrylessMultiply: + case NI_AVX10v1_RoundScaleScalar: case NI_AVX10v2_MinMax: case NI_AVX10v2_MinMaxScalar: case NI_AVX10v2_MultipleSumAbsoluteDifferences: diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index 58a9e2cbef9cc6..6cfd53c2e4206e 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -2489,6 +2489,7 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou case NI_AVX512_FusedMultiplySubtractAdd: case NI_AVX512_FusedMultiplySubtractNegated: case NI_AVX512_FusedMultiplySubtractNegatedScalar: + case NI_AVX10v1_FusedMultiplyAddScalar: { // While this operation is RMW, it is also almost freely reorderable // and so we do not need to set the operands as delay free unless From 0a10f92124f8f6319509bdf6d47112bedadd569d Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Thu, 9 Jul 2026 23:15:48 -0700 Subject: [PATCH 05/22] Accelerate System.Half arithmetic via AVX10v1 using the F16C premise Recognize the Half operators, Min/Max, Sqrt, reciprocal estimates, rounding, comparisons, and FusedMultiplyAdd as named intrinsics and expand them by moving the Half struct bits into/out of SIMD scalars (impSimdCreateScalarHalf/ impSimdToScalarHalf) and selecting the AVX10v1 *sh scalar instructions via a TYP_HALF SIMD base type. Half is never a GenTree/arg/return type, so the ABI is unchanged. Add lookupHalfNamedIntrinsic/lookupHalfIntrinsic/lookupHalfRoundingMode and mark the corresponding Half.cs members [Intrinsic]. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/compiler.h | 5 + src/coreclr/jit/importercalls.cpp | 311 +++++++++++++++++- .../System.Private.CoreLib/src/System/Half.cs | 20 ++ 3 files changed, 335 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 5e7ef8beb9c7ee..98a2d997695866 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -5322,6 +5322,11 @@ class Compiler NamedIntrinsic lookupPrimitiveFloatNamedIntrinsic(CORINFO_METHOD_HANDLE method, const char* methodName); NamedIntrinsic lookupPrimitiveIntNamedIntrinsic(CORINFO_METHOD_HANDLE method, const char* methodName); + NamedIntrinsic lookupHalfNamedIntrinsic(CORINFO_METHOD_HANDLE method, const char* methodName); +#if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_XARCH) + NamedIntrinsic lookupHalfIntrinsic(NamedIntrinsic ni); + int lookupHalfRoundingMode(NamedIntrinsic ni); +#endif // FEATURE_HW_INTRINSICS && TARGET_XARCH GenTree* impUnsupportedNamedIntrinsic(unsigned helper, CORINFO_METHOD_HANDLE method, CORINFO_SIG_INFO* sig, diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 61db69f06ddb25..b79200cd99999d 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -4578,6 +4578,121 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, break; } + case NI_System_Half_op_Addition: + case NI_System_Half_op_Subtraction: + case NI_System_Half_op_Multiply: + case NI_System_Half_op_Division: + case NI_System_Half_Max: + case NI_System_Half_Min: + { +#if defined(TARGET_XARCH) + if (compOpportunisticallyDependsOn(InstructionSet_AVX10v1)) + { + NamedIntrinsic opId = lookupHalfIntrinsic(ni); + assert(opId != NI_Illegal); + + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + + op2 = impSimdCreateScalarHalf(op2); + op1 = impSimdCreateScalarHalf(op1); + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, opId, TYP_HALF, 16); + retNode = impSimdToScalarHalf(retNode, clsHnd); + } +#endif // TARGET_XARCH + break; + } + + case NI_System_Half_Sqrt: + case NI_System_Half_ReciprocalEstimate: + case NI_System_Half_ReciprocalSqrtEstimate: + { +#if defined(TARGET_XARCH) + if (compOpportunisticallyDependsOn(InstructionSet_AVX10v1)) + { + NamedIntrinsic opId = lookupHalfIntrinsic(ni); + assert(opId != NI_Illegal); + + GenTree* op1 = impPopStack().val; + + // These scalar ops compute their result from lane 0 of the second operand and take the + // upper bits from the first. We only consume lane 0, so a zeroed upper-bits source is fine. + GenTree* op2 = gtNewZeroConNode(TYP_SIMD16); + op1 = impSimdCreateScalarHalf(op1); + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op2, op1, opId, TYP_HALF, 16); + retNode = impSimdToScalarHalf(retNode, clsHnd); + } +#endif // TARGET_XARCH + break; + } + + case NI_System_Half_FusedMultiplyAdd: + { +#if defined(TARGET_XARCH) + if (compOpportunisticallyDependsOn(InstructionSet_AVX10v1)) + { + GenTree* op3 = impPopStack().val; + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + + op3 = impSimdCreateScalarHalf(op3); + op2 = impSimdCreateScalarHalf(op2); + op1 = impSimdCreateScalarHalf(op1); + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, op3, NI_AVX10v1_FusedMultiplyAddScalar, + TYP_HALF, 16); + retNode = impSimdToScalarHalf(retNode, clsHnd); + } +#endif // TARGET_XARCH + break; + } + + case NI_System_Half_Round: + case NI_System_Half_Ceiling: + case NI_System_Half_Floor: + case NI_System_Half_Truncate: + { +#if defined(TARGET_XARCH) + // todo-xarch-half: We only optimize the single-argument overloads for now. + if (compOpportunisticallyDependsOn(InstructionSet_AVX10v1) && (sig->numArgs == 1)) + { + GenTree* op1 = impPopStack().val; + + int halfRoundingMode = lookupHalfRoundingMode(ni); + + GenTree* op2 = gtNewZeroConNode(TYP_SIMD16); + op1 = impSimdCreateScalarHalf(op1); + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op2, op1, gtNewIconNode(halfRoundingMode, TYP_INT), + NI_AVX10v1_RoundScaleScalar, TYP_HALF, 16); + retNode = impSimdToScalarHalf(retNode, clsHnd); + } +#endif // TARGET_XARCH + break; + } + + case NI_System_Half_op_GreaterThan: + case NI_System_Half_op_GreaterThanOrEqual: + case NI_System_Half_op_LessThan: + case NI_System_Half_op_LessThanOrEqual: + case NI_System_Half_op_Equality: + case NI_System_Half_op_Inequality: + { +#if defined(TARGET_XARCH) + if (compOpportunisticallyDependsOn(InstructionSet_AVX10v1)) + { + NamedIntrinsic opId = lookupHalfIntrinsic(ni); + assert(opId != NI_Illegal); + + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + + op2 = impSimdCreateScalarHalf(op2); + op1 = impSimdCreateScalarHalf(op1); + retNode = gtNewSimdHWIntrinsicNode(TYP_INT, op1, op2, opId, TYP_HALF, 16); + } +#endif // TARGET_XARCH + break; + } + case NI_System_Math_FusedMultiplyAdd: { assert(varTypeIsFloating(callType)); @@ -10901,7 +11016,7 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method) { if (strcmp(className, "Half") == 0) { - result = lookupPrimitiveFloatNamedIntrinsic(method, methodName); + result = lookupHalfNamedIntrinsic(method, methodName); } break; } @@ -12234,6 +12349,200 @@ NamedIntrinsic Compiler::lookupPrimitiveFloatNamedIntrinsic(CORINFO_METHOD_HANDL return result; } +//------------------------------------------------------------------------ +// lookupHalfNamedIntrinsic: map a System.Half method to its jit named intrinsic value +// +// Arguments: +// method -- method handle for method +// methodName -- name of the method +// +// Return Value: +// Id for the named intrinsic, or Illegal if none. +// +// Notes: +// method should have CORINFO_FLG_INTRINSIC set in its attributes, +// otherwise it is not a named jit intrinsic. +// +NamedIntrinsic Compiler::lookupHalfNamedIntrinsic(CORINFO_METHOD_HANDLE method, const char* methodName) +{ + NamedIntrinsic result = NI_Illegal; + + if (strcmp(methodName, "op_Addition") == 0) + { + result = NI_System_Half_op_Addition; + } + else if (strcmp(methodName, "op_Subtraction") == 0) + { + result = NI_System_Half_op_Subtraction; + } + else if (strcmp(methodName, "op_Multiply") == 0) + { + result = NI_System_Half_op_Multiply; + } + else if (strcmp(methodName, "op_Division") == 0) + { + result = NI_System_Half_op_Division; + } + else if (strcmp(methodName, "op_Equality") == 0) + { + result = NI_System_Half_op_Equality; + } + else if (strcmp(methodName, "op_Inequality") == 0) + { + result = NI_System_Half_op_Inequality; + } + else if (strcmp(methodName, "op_GreaterThan") == 0) + { + result = NI_System_Half_op_GreaterThan; + } + else if (strcmp(methodName, "op_GreaterThanOrEqual") == 0) + { + result = NI_System_Half_op_GreaterThanOrEqual; + } + else if (strcmp(methodName, "op_LessThan") == 0) + { + result = NI_System_Half_op_LessThan; + } + else if (strcmp(methodName, "op_LessThanOrEqual") == 0) + { + result = NI_System_Half_op_LessThanOrEqual; + } + else if (strcmp(methodName, "op_Explicit") == 0) + { + result = NI_System_Half_op_Explicit; + } + else if (strcmp(methodName, "Max") == 0) + { + result = NI_System_Half_Max; + } + else if (strcmp(methodName, "Min") == 0) + { + result = NI_System_Half_Min; + } + else if (strcmp(methodName, "Sqrt") == 0) + { + result = NI_System_Half_Sqrt; + } + else if (strcmp(methodName, "ReciprocalEstimate") == 0) + { + result = NI_System_Half_ReciprocalEstimate; + } + else if (strcmp(methodName, "ReciprocalSqrtEstimate") == 0) + { + result = NI_System_Half_ReciprocalSqrtEstimate; + } + else if (strcmp(methodName, "FusedMultiplyAdd") == 0) + { + result = NI_System_Half_FusedMultiplyAdd; + } + else if (strcmp(methodName, "Round") == 0) + { + result = NI_System_Half_Round; + } + else if (strcmp(methodName, "Ceiling") == 0) + { + result = NI_System_Half_Ceiling; + } + else if (strcmp(methodName, "Floor") == 0) + { + result = NI_System_Half_Floor; + } + else if (strcmp(methodName, "Truncate") == 0) + { + result = NI_System_Half_Truncate; + } + + return result; +} + +#if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_XARCH) +//------------------------------------------------------------------------ +// lookupHalfIntrinsic: map a System.Half named intrinsic to the AVX10v1 scalar +// hardware intrinsic that implements it +// +// Arguments: +// ni -- the System.Half named intrinsic +// +// Return Value: +// The corresponding NI_AVX10v1_* intrinsic, or NI_Illegal if none. +// +NamedIntrinsic Compiler::lookupHalfIntrinsic(NamedIntrinsic ni) +{ + assert(compOpportunisticallyDependsOn(InstructionSet_AVX10v1)); + + switch (ni) + { + case NI_System_Half_op_Addition: + return NI_AVX10v1_AddScalar; + case NI_System_Half_op_Subtraction: + return NI_AVX10v1_SubtractScalar; + case NI_System_Half_op_Multiply: + return NI_AVX10v1_MultiplyScalar; + case NI_System_Half_op_Division: + return NI_AVX10v1_DivideScalar; + case NI_System_Half_Max: + return NI_AVX10v1_MaxScalar; + case NI_System_Half_Min: + return NI_AVX10v1_MinScalar; + case NI_System_Half_Sqrt: + return NI_AVX10v1_SqrtScalar; + case NI_System_Half_ReciprocalEstimate: + return NI_AVX10v1_ReciprocalScalar; + case NI_System_Half_ReciprocalSqrtEstimate: + return NI_AVX10v1_ReciprocalSqrtScalar; + case NI_System_Half_FusedMultiplyAdd: + return NI_AVX10v1_FusedMultiplyAddScalar; + case NI_System_Half_op_GreaterThan: + return NI_AVX10v1_CompareScalarOrderedGreaterThan; + case NI_System_Half_op_GreaterThanOrEqual: + return NI_AVX10v1_CompareScalarOrderedGreaterThanOrEqual; + case NI_System_Half_op_LessThan: + return NI_AVX10v1_CompareScalarOrderedLessThan; + case NI_System_Half_op_LessThanOrEqual: + return NI_AVX10v1_CompareScalarOrderedLessThanOrEqual; + case NI_System_Half_op_Equality: + return NI_AVX10v1_CompareScalarOrderedEqual; + case NI_System_Half_op_Inequality: + return NI_AVX10v1_CompareScalarOrderedNotEqual; + case NI_System_Half_Round: + case NI_System_Half_Ceiling: + case NI_System_Half_Floor: + case NI_System_Half_Truncate: + return NI_AVX10v1_RoundScaleScalar; + default: + return NI_Illegal; + } +} + +//------------------------------------------------------------------------ +// lookupHalfRoundingMode: map a System.Half rounding named intrinsic to the +// immediate rounding mode used by RoundScaleScalar +// +// Arguments: +// ni -- the System.Half named intrinsic +// +// Return Value: +// The rounding mode immediate (0=nearest, 1=+inf, 2=-inf, 3=zero). +// +int Compiler::lookupHalfRoundingMode(NamedIntrinsic ni) +{ + switch (ni) + { + case NI_System_Half_Round: + return 0; // Round to nearest + case NI_System_Half_Ceiling: + return 1; // Round towards +infinity + case NI_System_Half_Floor: + return 2; // Round towards -infinity + case NI_System_Half_Truncate: + return 3; // Round towards zero + default: + noway_assert(!"Should have one of the above Half intrinsics"); + return -1; + } +} +#endif // FEATURE_HW_INTRINSICS && TARGET_XARCH + //------------------------------------------------------------------------ // lookupPrimitiveIntNamedIntrinsic: map method to jit named intrinsic value // diff --git a/src/libraries/System.Private.CoreLib/src/System/Half.cs b/src/libraries/System.Private.CoreLib/src/System/Half.cs index 8fe7b7be916ddd..7dd41648016599 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Half.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Half.cs @@ -155,6 +155,7 @@ internal static ushort ExtractTrailingSignificandFromBits(ushort bits) } /// + [Intrinsic] public static bool operator <(Half left, Half right) { if (IsNaN(left) || IsNaN(right)) @@ -177,12 +178,14 @@ internal static ushort ExtractTrailingSignificandFromBits(ushort bits) } /// + [Intrinsic] public static bool operator >(Half left, Half right) { return right < left; } /// + [Intrinsic] public static bool operator <=(Half left, Half right) { if (IsNaN(left) || IsNaN(right)) @@ -205,12 +208,14 @@ internal static ushort ExtractTrailingSignificandFromBits(ushort bits) } /// + [Intrinsic] public static bool operator >=(Half left, Half right) { return right <= left; } /// + [Intrinsic] public static bool operator ==(Half left, Half right) { if (IsNaN(left) || IsNaN(right)) @@ -224,6 +229,7 @@ internal static ushort ExtractTrailingSignificandFromBits(ushort bits) } /// + [Intrinsic] public static bool operator !=(Half left, Half right) { return !(left == right); @@ -1187,6 +1193,7 @@ private static double CreateDoubleNaN(bool sign, ulong significand) // /// + [Intrinsic] public static Half operator +(Half left, Half right) => (Half)((float)left + (float)right); // @@ -1280,6 +1287,7 @@ public static bool IsPow2(Half value) // /// + [Intrinsic] public static Half operator /(Half left, Half right) => (Half)((float)left / (float)right); // @@ -1309,6 +1317,7 @@ public static bool IsPow2(Half value) // /// + [Intrinsic] public static Half Ceiling(Half x) => (Half)MathF.Ceiling((float)x); /// @@ -1320,9 +1329,11 @@ public static TInteger ConvertToIntegerNative(Half value) where TInteger : IBinaryInteger => TInteger.CreateSaturating(value); /// + [Intrinsic] public static Half Floor(Half x) => (Half)MathF.Floor((float)x); /// + [Intrinsic] public static Half Round(Half x) => (Half)MathF.Round((float)x); /// @@ -1335,6 +1346,7 @@ public static TInteger ConvertToIntegerNative(Half value) public static Half Round(Half x, int digits, MidpointRounding mode) => (Half)MathF.Round((float)x, digits, mode); /// + [Intrinsic] public static Half Truncate(Half x) => (Half)MathF.Truncate((float)x); /// @@ -1508,6 +1520,7 @@ public static Half BitIncrement(Half x) } /// + [Intrinsic] public static Half FusedMultiplyAdd(Half left, Half right, Half addend) => (Half)MathF.FusedMultiplyAdd((float)left, (float)right, (float)addend); /// @@ -1545,9 +1558,11 @@ public static int ILogB(Half x) public static Half Lerp(Half value1, Half value2, Half amount) => (Half)float.Lerp((float)value1, (float)value2, (float)amount); /// + [Intrinsic] public static Half ReciprocalEstimate(Half x) => (Half)MathF.ReciprocalEstimate((float)x); /// + [Intrinsic] public static Half ReciprocalSqrtEstimate(Half x) => (Half)MathF.ReciprocalSqrtEstimate((float)x); /// @@ -1631,6 +1646,7 @@ public static int ILogB(Half x) // /// + [Intrinsic] public static Half operator *(Half left, Half right) => (Half)((float)left * (float)right); // @@ -1664,6 +1680,7 @@ public static Half CopySign(Half value, Half sign) } /// + [Intrinsic] public static Half Max(Half x, Half y) => (Half)float.Max((float)x, (float)y); /// @@ -1692,6 +1709,7 @@ public static Half MaxNumber(Half x, Half y) } /// + [Intrinsic] public static Half Min(Half x, Half y) => (Half)float.Min((float)x, (float)y); /// @@ -2195,6 +2213,7 @@ private static bool TryConvertTo(Half value, [MaybeNullWhen(false)] out public static Half RootN(Half x, int n) => (Half)float.RootN((float)x, n); /// + [Intrinsic] public static Half Sqrt(Half x) => (Half)MathF.Sqrt((float)x); // @@ -2219,6 +2238,7 @@ private static bool TryConvertTo(Half value, [MaybeNullWhen(false)] out // /// + [Intrinsic] public static Half operator -(Half left, Half right) => (Half)((float)left - (float)right); // From 8fffc3d6b4217867b39af8b990d321f0b7cdda81 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Thu, 9 Jul 2026 23:45:11 -0700 Subject: [PATCH 06/22] Add TYP_HALF ins slot to the ARM64 SVE hwintrinsic table The uniform 11-slot HARDWARE_INTRINSIC macro requires every list file to provide the TYP_HALF instruction column. The SVE list was missed, causing a C4003 not-enough-arguments error when building the arm64 altjit variant. Add the INS_invalid TYP_HALF column to each SVE entry. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/hwintrinsiclistarm64sve.h | 1068 ++++++++++----------- 1 file changed, 534 insertions(+), 534 deletions(-) diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index 81ccf47a89f8bf..6c15735e515834 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -16,310 +16,310 @@ // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // SVE Intrinsics #define FIRST_NI_Sve NI_Sve_Abs -HARDWARE_INTRINSIC(Sve, Abs, -1, -1, INS_sve_abs, INS_invalid, INS_sve_abs, INS_invalid, INS_sve_abs, INS_invalid, INS_sve_abs, INS_invalid, INS_sve_fabs, INS_sve_fabs, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, AbsoluteCompareGreaterThan, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_facgt, INS_sve_facgt, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, AbsoluteCompareGreaterThanOrEqual, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_facge, INS_sve_facge, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, AbsoluteCompareLessThan, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_faclt, INS_sve_faclt, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, AbsoluteCompareLessThanOrEqual, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_facle, INS_sve_facle, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, AbsoluteDifference, -1, -1, INS_sve_sabd, INS_sve_uabd, INS_sve_sabd, INS_sve_uabd, INS_sve_sabd, INS_sve_uabd, INS_sve_sabd, INS_sve_uabd, INS_sve_fabd, INS_sve_fabd, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, Add, -1, -1, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_fadd, INS_sve_fadd, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, AddAcross, -1, 1, INS_sve_saddv, INS_sve_uaddv, INS_sve_saddv, INS_sve_uaddv, INS_sve_saddv, INS_sve_uaddv, INS_sve_uaddv, INS_sve_uaddv, INS_sve_faddv, INS_sve_faddv, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) -HARDWARE_INTRINSIC(Sve, AddRotateComplex, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcadd, INS_sve_fcadd, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve, AddSaturate, -1, 2, INS_sve_sqadd, INS_sve_uqadd, INS_sve_sqadd, INS_sve_uqadd, INS_sve_sqadd, INS_sve_uqadd, INS_sve_sqadd, INS_sve_uqadd, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve, AddSequentialAcross, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fadda, INS_sve_fadda, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_ReduceOperation) -HARDWARE_INTRINSIC(Sve, And, -1, -1, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_HasAllMaskVariant) -HARDWARE_INTRINSIC(Sve, AndAcross, -1, -1, INS_sve_andv, INS_sve_andv, INS_sve_andv, INS_sve_andv, INS_sve_andv, INS_sve_andv, INS_sve_andv, INS_sve_andv, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) -HARDWARE_INTRINSIC(Sve, BitwiseClear, -1, -1, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_HasAllMaskVariant) -HARDWARE_INTRINSIC(Sve, BooleanNot, -1, -1, INS_sve_cnot, INS_sve_cnot, INS_sve_cnot, INS_sve_cnot, INS_sve_cnot, INS_sve_cnot, INS_sve_cnot, INS_sve_cnot, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, Compact, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_compact, INS_sve_compact, INS_sve_compact, INS_sve_compact, INS_sve_compact, INS_sve_compact, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, CompareEqual, -1, -1, INS_sve_cmpeq, INS_sve_cmpeq, INS_sve_cmpeq, INS_sve_cmpeq, INS_sve_cmpeq, INS_sve_cmpeq, INS_sve_cmpeq, INS_sve_cmpeq, INS_sve_fcmeq, INS_sve_fcmeq, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, CompareGreaterThan, -1, -1, INS_sve_cmpgt, INS_sve_cmphi, INS_sve_cmpgt, INS_sve_cmphi, INS_sve_cmpgt, INS_sve_cmphi, INS_sve_cmpgt, INS_sve_cmphi, INS_sve_fcmgt, INS_sve_fcmgt, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, CompareGreaterThanOrEqual, -1, -1, INS_sve_cmpge, INS_sve_cmphs, INS_sve_cmpge, INS_sve_cmphs, INS_sve_cmpge, INS_sve_cmphs, INS_sve_cmpge, INS_sve_cmphs, INS_sve_fcmge, INS_sve_fcmge, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, CompareLessThan, -1, -1, INS_sve_cmplt, INS_sve_cmplo, INS_sve_cmplt, INS_sve_cmplo, INS_sve_cmplt, INS_sve_cmplo, INS_sve_cmplt, INS_sve_cmplo, INS_sve_fcmlt, INS_sve_fcmlt, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, CompareLessThanOrEqual, -1, -1, INS_sve_cmple, INS_sve_cmpls, INS_sve_cmple, INS_sve_cmpls, INS_sve_cmple, INS_sve_cmpls, INS_sve_cmple, INS_sve_cmpls, INS_sve_fcmle, INS_sve_fcmle, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, CompareNotEqualTo, -1, -1, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_fcmne, INS_sve_fcmne, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, CompareUnordered, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcmuo, INS_sve_fcmuo, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, Compute16BitAddresses, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_adr, INS_invalid, INS_sve_adr, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, Compute32BitAddresses, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_adr, INS_invalid, INS_sve_adr, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, Compute64BitAddresses, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_adr, INS_invalid, INS_sve_adr, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, Compute8BitAddresses, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_adr, INS_invalid, INS_sve_adr, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, ConditionalExtractAfterLastActiveElement, -1, 3, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, ConditionalExtractAfterLastActiveElementAndReplicate, -1, 3, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, ConditionalExtractLastActiveElement, -1, 3, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, ConditionalExtractLastActiveElementAndReplicate, -1, 3, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, ConditionalSelect, -1, 3, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_SupportsContainment|HW_Flag_HasAllMaskVariant) -HARDWARE_INTRINSIC(Sve, ConvertToDouble, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_scvtf, INS_sve_ucvtf, INS_sve_scvtf, INS_sve_ucvtf, INS_sve_fcvt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, ConvertToInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtzs, INS_sve_fcvtzs, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, ConvertToInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtzs, INS_sve_fcvtzs, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, ConvertToSingle, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_scvtf, INS_sve_ucvtf, INS_sve_scvtf, INS_sve_ucvtf, INS_invalid, INS_sve_fcvt, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, ConvertToUInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtzu, INS_sve_fcvtzu, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, ConvertToUInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtzu, INS_sve_fcvtzu, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, Count16BitElements, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_cnth, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(Sve, Count32BitElements, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_cntw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(Sve, Count64BitElements, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_cntd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(Sve, Count8BitElements, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_cntb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(Sve, CreateBreakAfterMask, -1, 2, INS_sve_brka, INS_sve_brka, INS_sve_brka, INS_sve_brka, INS_sve_brka, INS_sve_brka, INS_sve_brka, INS_sve_brka, INS_sve_brka, INS_sve_brka, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, CreateBreakAfterPropagateMask, -1, 3, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, CreateBreakBeforeMask, -1, 2, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, CreateBreakBeforePropagateMask, -1, 3, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, CreateBreakPropagateMask, -1, -1, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_HasRMWSemantics|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, CreateFalseMaskByte, -1, 0, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateFalseMaskDouble, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateFalseMaskInt16, -1, 0, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateFalseMaskInt32, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateFalseMaskInt64, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateFalseMaskSByte, -1, 0, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateFalseMaskSingle, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateFalseMaskUInt16, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateFalseMaskUInt32, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateFalseMaskUInt64, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateMaskForFirstActiveElement, -1, 2, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, CreateMaskForNextActiveElement, -1, 2, INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, CreateTrueMaskByte, -1, 1, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateTrueMaskDouble, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateTrueMaskInt16, -1, 1, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateTrueMaskInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateTrueMaskInt64, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateTrueMaskSByte, -1, 1, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateTrueMaskSingle, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateTrueMaskUInt16, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateTrueMaskUInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateTrueMaskUInt64, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskByte, -1, 2, INS_invalid, INS_sve_whilelt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskDouble, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilelt, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_whilelt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilelt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilelt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskSByte, -1, 2, INS_sve_whilelt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskSingle, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilelt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilelt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilelt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilelt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskByte, -1, 2, INS_invalid, INS_sve_whilele, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskDouble, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilele, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_whilele, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilele, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilele, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskSByte, -1, 2, INS_sve_whilele, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskSingle, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilele, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilele, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilele, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilele, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, Divide, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sdiv, INS_sve_udiv, INS_sve_sdiv, INS_sve_udiv, INS_sve_fdiv, INS_sve_fdiv, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, DotProduct, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sdot, INS_sve_udot, INS_sve_sdot, INS_sve_udot, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, DotProductBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sdot, INS_sve_udot, INS_sve_sdot, INS_sve_udot, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve, DuplicateSelectedScalarToVector, -1, 2, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve, ExtractAfterLastActiveElement, -1, 2, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, ExtractAfterLastActiveElementScalar, 0, 2, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, ExtractLastActiveElement, -1, 2, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, ExtractLastActiveElementScalar, 0, 2, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, ExtractVector, -1, 3, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, FloatingPointExponentialAccelerator, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fexpa, INS_invalid, INS_sve_fexpa, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Sve, FusedMultiplyAdd, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmla, INS_sve_fmla, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, FusedMultiplyAddBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmla, INS_sve_fmla, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_FmaIntrinsic|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve, FusedMultiplyAddNegated, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fnmla, INS_sve_fnmla, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, FusedMultiplySubtract, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmls, INS_sve_fmls, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, FusedMultiplySubtractBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmls, INS_sve_fmls, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_FmaIntrinsic|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve, FusedMultiplySubtractNegated, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fnmls, INS_sve_fnmls, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, GatherPrefetch16Bit, -1, -1, INS_invalid, INS_invalid, INS_sve_prfh, INS_sve_prfh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherPrefetch32Bit, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_prfw, INS_sve_prfw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherPrefetch64Bit, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_prfd, INS_sve_prfd, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherPrefetch8Bit, -1, -1, INS_sve_prfb, INS_sve_prfb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherVector, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1d, INS_sve_ld1d, INS_sve_ld1w, INS_sve_ld1d, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, GatherVectorByteZeroExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_sve_ld1b, INS_sve_ld1b, INS_sve_ld1b, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, GatherVectorByteZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherVectorFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_sve_ldff1d, INS_sve_ldff1d, INS_sve_ldff1w, INS_sve_ldff1d, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherVectorInt16SignExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_sve_ld1sh, INS_sve_ld1sh, INS_sve_ld1sh, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, GatherVectorInt16SignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherVectorInt16WithByteOffsetsSignExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_sve_ld1sh, INS_sve_ld1sh, INS_sve_ld1sh, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, GatherVectorInt16WithByteOffsetsSignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherVectorInt32SignExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sw, INS_sve_ld1sw, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, GatherVectorInt32SignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sw, INS_sve_ldff1sw, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherVectorInt32WithByteOffsetsSignExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sw, INS_sve_ld1sw, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, GatherVectorInt32WithByteOffsetsSignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sw, INS_sve_ldff1sw, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherVectorSByteSignExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_sve_ld1sb, INS_sve_ld1sb, INS_sve_ld1sb, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, GatherVectorSByteSignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherVectorUInt16WithByteOffsetsZeroExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_sve_ld1h, INS_sve_ld1h, INS_sve_ld1h, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, GatherVectorUInt16WithByteOffsetsZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherVectorUInt16ZeroExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_sve_ld1h, INS_sve_ld1h, INS_sve_ld1h, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, GatherVectorUInt16ZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherVectorUInt32WithByteOffsetsZeroExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, GatherVectorUInt32WithByteOffsetsZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherVectorUInt32ZeroExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, GatherVectorUInt32ZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherVectorWithByteOffsetFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_sve_ldff1d, INS_sve_ldff1d, INS_sve_ldff1w, INS_sve_ldff1d, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherVectorWithByteOffsets, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1d, INS_sve_ld1d, INS_sve_ld1w, INS_sve_ld1d, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, GetActiveElementCount, -1, 2, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation) -HARDWARE_INTRINSIC(Sve, GetFfrByte, -1, 0, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GetFfrDouble, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GetFfrInt16, -1, 0, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GetFfrInt32, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GetFfrInt64, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GetFfrSByte, -1, 0, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GetFfrSingle, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GetFfrUInt16, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GetFfrUInt32, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GetFfrUInt64, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, InsertIntoShiftedVector, -1, 2, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, LeadingSignCount, -1, -1, INS_sve_cls, INS_invalid, INS_sve_cls, INS_invalid, INS_sve_cls, INS_invalid, INS_sve_cls, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, LeadingZeroCount, -1, -1, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, Load2xVectorAndUnzip, -1, 2, INS_sve_ld2b, INS_sve_ld2b, INS_sve_ld2h, INS_sve_ld2h, INS_sve_ld2w, INS_sve_ld2w, INS_sve_ld2d, INS_sve_ld2d, INS_sve_ld2w, INS_sve_ld2d, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Sve, Load3xVectorAndUnzip, -1, 2, INS_sve_ld3b, INS_sve_ld3b, INS_sve_ld3h, INS_sve_ld3h, INS_sve_ld3w, INS_sve_ld3w, INS_sve_ld3d, INS_sve_ld3d, INS_sve_ld3w, INS_sve_ld3d, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Sve, Load4xVectorAndUnzip, -1, 2, INS_sve_ld4b, INS_sve_ld4b, INS_sve_ld4h, INS_sve_ld4h, INS_sve_ld4w, INS_sve_ld4w, INS_sve_ld4d, INS_sve_ld4d, INS_sve_ld4w, INS_sve_ld4d, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Sve, LoadVector, -1, 2, INS_sve_ld1b, INS_sve_ld1b, INS_sve_ld1h, INS_sve_ld1h, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1d, INS_sve_ld1d, INS_sve_ld1w, INS_sve_ld1d, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVector128AndReplicateToVector, -1, 2, INS_sve_ld1rqb, INS_sve_ld1rqb, INS_sve_ld1rqh, INS_sve_ld1rqh, INS_sve_ld1rqw, INS_sve_ld1rqw, INS_sve_ld1rqd, INS_sve_ld1rqd, INS_sve_ld1rqw, INS_sve_ld1rqd, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToInt16, -1, -1, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToUInt16, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToUInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToUInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorFirstFaulting, -1, -1, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1w, INS_sve_ldff1w, INS_sve_ldff1d, INS_sve_ldff1d, INS_sve_ldff1w, INS_sve_ldff1d, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToUInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToUInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorInt32NonFaultingSignExtendToInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorInt32NonFaultingSignExtendToUInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sw, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorInt32SignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sw, INS_sve_ldff1sw, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorInt32SignExtendToInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorInt32SignExtendToUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sw, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorNonFaulting, -1, -1, INS_sve_ldnf1b, INS_sve_ldnf1b, INS_sve_ldnf1h, INS_sve_ldnf1h, INS_sve_ldnf1w, INS_sve_ldnf1w, INS_sve_ldnf1d, INS_sve_ldnf1d, INS_sve_ldnf1w, INS_sve_ldnf1d, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorNonTemporal, -1, 2, INS_sve_ldnt1b, INS_sve_ldnt1b, INS_sve_ldnt1h, INS_sve_ldnt1h, INS_sve_ldnt1w, INS_sve_ldnt1w, INS_sve_ldnt1d, INS_sve_ldnt1d, INS_sve_ldnt1w, INS_sve_ldnt1d, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToInt16, -1, -1, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToUInt16, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToUInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToUInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToUInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToUInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt32NonFaultingZeroExtendToInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1w, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt32NonFaultingZeroExtendToUInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1w, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, Max, -1, -1, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_fmax, INS_sve_fmax, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, MaxAcross, -1, -1, INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_fmaxv, INS_sve_fmaxv, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) -HARDWARE_INTRINSIC(Sve, MaxNumber, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmaxnm, INS_sve_fmaxnm, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, MaxNumberAcross, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmaxnmv, INS_sve_fmaxnmv, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) -HARDWARE_INTRINSIC(Sve, Min, -1, -1, INS_sve_smin, INS_sve_umin, INS_sve_smin, INS_sve_umin, INS_sve_smin, INS_sve_umin, INS_sve_smin, INS_sve_umin, INS_sve_fmin, INS_sve_fmin, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, MinAcross, -1, -1, INS_sve_sminv, INS_sve_uminv, INS_sve_sminv, INS_sve_uminv, INS_sve_sminv, INS_sve_uminv, INS_sve_sminv, INS_sve_uminv, INS_sve_fminv, INS_sve_fminv, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) -HARDWARE_INTRINSIC(Sve, MinNumber, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fminnm, INS_sve_fminnm, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, MinNumberAcross, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fminnmv, INS_sve_fminnmv, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) -HARDWARE_INTRINSIC(Sve, Multiply, -1, 2, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_fmul, INS_sve_fmul, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, MultiplyAdd, -1, -1, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, MultiplyAddRotateComplex, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcmla, INS_sve_fcmla, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve, MultiplyAddRotateComplexBySelectedScalar, -1, 5, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcmla, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, MultiplyBySelectedScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmul, INS_sve_fmul, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve, MultiplyExtended, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmulx, INS_sve_fmulx, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, MultiplySubtract, -1, -1, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, Negate, -1, -1, INS_sve_neg, INS_invalid, INS_sve_neg, INS_invalid, INS_sve_neg, INS_invalid, INS_sve_neg, INS_invalid, INS_sve_fneg, INS_sve_fneg, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, Not, -1, -1, INS_sve_not, INS_sve_not, INS_sve_not, INS_sve_not, INS_sve_not, INS_sve_not, INS_sve_not, INS_sve_not, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation) -HARDWARE_INTRINSIC(Sve, Or, -1, -1, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_HasAllMaskVariant) -HARDWARE_INTRINSIC(Sve, OrAcross, -1, -1, INS_sve_orv, INS_sve_orv, INS_sve_orv, INS_sve_orv, INS_sve_orv, INS_sve_orv, INS_sve_orv, INS_sve_orv, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) -HARDWARE_INTRINSIC(Sve, PopCount, -1, -1, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, Prefetch16Bit, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_sve_prfh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, Prefetch32Bit, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_prfw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, Prefetch64Bit, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_prfd, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, Prefetch8Bit, -1, 3, INS_invalid, INS_sve_prfb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, ReciprocalEstimate, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frecpe, INS_sve_frecpe, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve, ReciprocalExponent, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frecpx, INS_sve_frecpx, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, ReciprocalSqrtEstimate, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frsqrte, INS_sve_frsqrte, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve, ReciprocalSqrtStep, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frsqrts, INS_sve_frsqrts, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve, ReciprocalStep, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frecps, INS_sve_frecps, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve, ReverseBits, -1, -1, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, ReverseElement, -1, 1, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant) -HARDWARE_INTRINSIC(Sve, ReverseElement16, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_revh, INS_sve_revh, INS_sve_revh, INS_sve_revh, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, ReverseElement32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_revw, INS_sve_revw, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, ReverseElement8, -1, -1, INS_invalid, INS_invalid, INS_sve_revb, INS_sve_revb, INS_sve_revb, INS_sve_revb, INS_sve_revb, INS_sve_revb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, RoundAwayFromZero, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frinta, INS_sve_frinta, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, RoundToNearest, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frintn, INS_sve_frintn, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, RoundToNegativeInfinity, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frintm, INS_sve_frintm, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, RoundToPositiveInfinity, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frintp, INS_sve_frintp, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, RoundToZero, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frintz, INS_sve_frintz, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, SaturatingDecrementBy16BitElementCount, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdech, INS_sve_uqdech, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SaturatingDecrementBy32BitElementCount, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdecw, INS_sve_uqdecw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SaturatingDecrementBy64BitElementCount, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdecd, INS_sve_uqdecd, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SaturatingDecrementBy8BitElementCount, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdecb, INS_sve_uqdecb, INS_sve_sqdecb, INS_sve_uqdecb, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SaturatingDecrementByActiveElementCount, -1, 2, INS_invalid, INS_sve_sqdecp, INS_sve_sqdecp, INS_sve_sqdecp, INS_sve_sqdecp, INS_sve_sqdecp, INS_sve_sqdecp, INS_sve_sqdecp, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy16BitElementCount, -1, 3, INS_invalid, INS_invalid, INS_sve_sqinch, INS_sve_uqinch, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy32BitElementCount, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqincw, INS_sve_uqincw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy64BitElementCount, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqincd, INS_sve_uqincd, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy8BitElementCount, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqincb, INS_sve_uqincb, INS_sve_sqincb, INS_sve_uqincb, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SaturatingIncrementByActiveElementCount, -1, 2, INS_invalid, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, Scale, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fscale, INS_sve_fscale, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, Scatter, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1w, INS_sve_st1w, INS_sve_st1d, INS_sve_st1d, INS_sve_st1w, INS_sve_st1d, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, Scatter16BitNarrowing, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1h, INS_sve_st1h, INS_sve_st1h, INS_sve_st1h, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, Scatter16BitWithByteOffsetsNarrowing, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1h, INS_sve_st1h, INS_sve_st1h, INS_sve_st1h, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, Scatter32BitNarrowing, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1w, INS_sve_st1w, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, Scatter32BitWithByteOffsetsNarrowing, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1w, INS_sve_st1w, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, Scatter8BitNarrowing, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1b, INS_sve_st1b, INS_sve_st1b, INS_sve_st1b, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, Scatter8BitWithByteOffsetsNarrowing, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1b, INS_sve_st1b, INS_sve_st1b, INS_sve_st1b, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, ScatterWithByteOffsets, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1w, INS_sve_st1w, INS_sve_st1d, INS_sve_st1d, INS_sve_st1w, INS_sve_st1d, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, SetFfr, -1, 1, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialSideEffect_Other|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, ShiftLeftLogical, -1, -1, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, ShiftRightArithmetic, -1, -1, INS_sve_asr, INS_invalid, INS_sve_asr, INS_invalid, INS_sve_asr, INS_invalid, INS_sve_asr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, ShiftRightArithmeticForDivide, -1, -1, INS_sve_asrd, INS_invalid, INS_sve_asrd, INS_invalid, INS_sve_asrd, INS_invalid, INS_sve_asrd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve, ShiftRightLogical, -1, -1, INS_invalid, INS_sve_lsr, INS_invalid, INS_sve_lsr, INS_invalid, INS_sve_lsr, INS_invalid, INS_sve_lsr, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SignExtend16, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sxth, INS_invalid, INS_sve_sxth, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, SignExtend32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sxtw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, SignExtend8, -1, -1, INS_invalid, INS_invalid, INS_sve_sxtb, INS_invalid, INS_sve_sxtb, INS_invalid, INS_sve_sxtb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, SignExtendWideningLower, -1, 1, INS_sve_sunpklo, INS_invalid, INS_sve_sunpklo, INS_invalid, INS_sve_sunpklo, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Sve, SignExtendWideningUpper, -1, 1, INS_sve_sunpkhi, INS_invalid, INS_sve_sunpkhi, INS_invalid, INS_sve_sunpkhi, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Sve, Splice, -1, 3, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, Sqrt, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fsqrt, INS_sve_fsqrt, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, StoreAndZip, -1, 3, INS_sve_st1b, INS_sve_st1b, INS_sve_st1h, INS_sve_st1h, INS_sve_st1w, INS_sve_st1w, INS_sve_st1d, INS_sve_st1d, INS_sve_st1w, INS_sve_st1d, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, StoreNarrowing, -1, 3, INS_sve_st1b, INS_sve_st1b, INS_sve_st1h, INS_sve_st1h, INS_sve_st1w, INS_sve_st1w, INS_sve_st1d, INS_sve_st1d, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, StoreNonTemporal, -1, 3, INS_sve_stnt1b, INS_sve_stnt1b, INS_sve_stnt1h, INS_sve_stnt1h, INS_sve_stnt1w, INS_sve_stnt1w, INS_sve_stnt1d, INS_sve_stnt1d, INS_sve_stnt1w, INS_sve_stnt1d, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, Subtract, -1, 2, INS_sve_sub, INS_sve_sub, INS_sve_sub, INS_sve_sub, INS_sve_sub, INS_sve_sub, INS_sve_sub, INS_sve_sub, INS_sve_fsub, INS_sve_fsub, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, SubtractSaturate, -1, 2, INS_sve_sqsub, INS_sve_uqsub, INS_sve_sqsub, INS_sve_uqsub, INS_sve_sqsub, INS_sve_uqsub, INS_sve_sqsub, INS_sve_uqsub, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve, TestAnyTrue, -1, 2, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, TestFirstTrue, -1, 2, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, TestLastTrue, -1, 2, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, TransposeEven, -1, 2, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant) -HARDWARE_INTRINSIC(Sve, TransposeOdd, -1, 2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant) -HARDWARE_INTRINSIC(Sve, TrigonometricMultiplyAddCoefficient, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ftmad, INS_sve_ftmad, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, TrigonometricSelectCoefficient, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ftssel, INS_sve_ftssel, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Sve, TrigonometricStartingValue, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ftsmul, INS_sve_ftsmul, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Sve, UnzipEven, -1, 2, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant) -HARDWARE_INTRINSIC(Sve, UnzipOdd, -1, 2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant) -HARDWARE_INTRINSIC(Sve, VectorTableLookup, -1, 2, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve, Xor, -1, -1, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_HasAllMaskVariant) -HARDWARE_INTRINSIC(Sve, XorAcross, -1, -1, INS_sve_eorv, INS_sve_eorv, INS_sve_eorv, INS_sve_eorv, INS_sve_eorv, INS_sve_eorv, INS_sve_eorv, INS_sve_eorv, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) -HARDWARE_INTRINSIC(Sve, ZeroExtend16, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_uxth, INS_invalid, INS_sve_uxth, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, ZeroExtend32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_uxtw, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, ZeroExtend8, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_sve_uxtb, INS_invalid, INS_sve_uxtb, INS_invalid, INS_sve_uxtb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, ZeroExtendWideningLower, -1, 1, INS_invalid, INS_sve_uunpklo, INS_invalid, INS_sve_uunpklo, INS_invalid, INS_sve_uunpklo, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Sve, ZeroExtendWideningUpper, -1, 1, INS_invalid, INS_sve_uunpkhi, INS_invalid, INS_sve_uunpkhi, INS_invalid, INS_sve_uunpkhi, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Sve, ZipHigh, -1, 2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant) -HARDWARE_INTRINSIC(Sve, ZipLow, -1, 2, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant) +HARDWARE_INTRINSIC(Sve, Abs, -1, -1, INS_sve_abs, INS_invalid, INS_sve_abs, INS_invalid, INS_sve_abs, INS_invalid, INS_sve_abs, INS_invalid, INS_sve_fabs, INS_sve_fabs, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, AbsoluteCompareGreaterThan, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_facgt, INS_sve_facgt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, AbsoluteCompareGreaterThanOrEqual, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_facge, INS_sve_facge, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, AbsoluteCompareLessThan, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_faclt, INS_sve_faclt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, AbsoluteCompareLessThanOrEqual, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_facle, INS_sve_facle, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, AbsoluteDifference, -1, -1, INS_sve_sabd, INS_sve_uabd, INS_sve_sabd, INS_sve_uabd, INS_sve_sabd, INS_sve_uabd, INS_sve_sabd, INS_sve_uabd, INS_sve_fabd, INS_sve_fabd, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, Add, -1, -1, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_fadd, INS_sve_fadd, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, AddAcross, -1, 1, INS_sve_saddv, INS_sve_uaddv, INS_sve_saddv, INS_sve_uaddv, INS_sve_saddv, INS_sve_uaddv, INS_sve_uaddv, INS_sve_uaddv, INS_sve_faddv, INS_sve_faddv, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) +HARDWARE_INTRINSIC(Sve, AddRotateComplex, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcadd, INS_sve_fcadd, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve, AddSaturate, -1, 2, INS_sve_sqadd, INS_sve_uqadd, INS_sve_sqadd, INS_sve_uqadd, INS_sve_sqadd, INS_sve_uqadd, INS_sve_sqadd, INS_sve_uqadd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve, AddSequentialAcross, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fadda, INS_sve_fadda, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_ReduceOperation) +HARDWARE_INTRINSIC(Sve, And, -1, -1, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_HasAllMaskVariant) +HARDWARE_INTRINSIC(Sve, AndAcross, -1, -1, INS_sve_andv, INS_sve_andv, INS_sve_andv, INS_sve_andv, INS_sve_andv, INS_sve_andv, INS_sve_andv, INS_sve_andv, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) +HARDWARE_INTRINSIC(Sve, BitwiseClear, -1, -1, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_HasAllMaskVariant) +HARDWARE_INTRINSIC(Sve, BooleanNot, -1, -1, INS_sve_cnot, INS_sve_cnot, INS_sve_cnot, INS_sve_cnot, INS_sve_cnot, INS_sve_cnot, INS_sve_cnot, INS_sve_cnot, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, Compact, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_compact, INS_sve_compact, INS_sve_compact, INS_sve_compact, INS_sve_compact, INS_sve_compact, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, CompareEqual, -1, -1, INS_sve_cmpeq, INS_sve_cmpeq, INS_sve_cmpeq, INS_sve_cmpeq, INS_sve_cmpeq, INS_sve_cmpeq, INS_sve_cmpeq, INS_sve_cmpeq, INS_sve_fcmeq, INS_sve_fcmeq, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, CompareGreaterThan, -1, -1, INS_sve_cmpgt, INS_sve_cmphi, INS_sve_cmpgt, INS_sve_cmphi, INS_sve_cmpgt, INS_sve_cmphi, INS_sve_cmpgt, INS_sve_cmphi, INS_sve_fcmgt, INS_sve_fcmgt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, CompareGreaterThanOrEqual, -1, -1, INS_sve_cmpge, INS_sve_cmphs, INS_sve_cmpge, INS_sve_cmphs, INS_sve_cmpge, INS_sve_cmphs, INS_sve_cmpge, INS_sve_cmphs, INS_sve_fcmge, INS_sve_fcmge, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, CompareLessThan, -1, -1, INS_sve_cmplt, INS_sve_cmplo, INS_sve_cmplt, INS_sve_cmplo, INS_sve_cmplt, INS_sve_cmplo, INS_sve_cmplt, INS_sve_cmplo, INS_sve_fcmlt, INS_sve_fcmlt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, CompareLessThanOrEqual, -1, -1, INS_sve_cmple, INS_sve_cmpls, INS_sve_cmple, INS_sve_cmpls, INS_sve_cmple, INS_sve_cmpls, INS_sve_cmple, INS_sve_cmpls, INS_sve_fcmle, INS_sve_fcmle, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, CompareNotEqualTo, -1, -1, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_fcmne, INS_sve_fcmne, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, CompareUnordered, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcmuo, INS_sve_fcmuo, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, Compute16BitAddresses, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_adr, INS_invalid, INS_sve_adr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, Compute32BitAddresses, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_adr, INS_invalid, INS_sve_adr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, Compute64BitAddresses, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_adr, INS_invalid, INS_sve_adr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, Compute8BitAddresses, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_adr, INS_invalid, INS_sve_adr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, ConditionalExtractAfterLastActiveElement, -1, 3, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, ConditionalExtractAfterLastActiveElementAndReplicate, -1, 3, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, ConditionalExtractLastActiveElement, -1, 3, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, ConditionalExtractLastActiveElementAndReplicate, -1, 3, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, ConditionalSelect, -1, 3, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_SupportsContainment|HW_Flag_HasAllMaskVariant) +HARDWARE_INTRINSIC(Sve, ConvertToDouble, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_scvtf, INS_sve_ucvtf, INS_sve_scvtf, INS_sve_ucvtf, INS_sve_fcvt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ConvertToInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtzs, INS_sve_fcvtzs, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ConvertToInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtzs, INS_sve_fcvtzs, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ConvertToSingle, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_scvtf, INS_sve_ucvtf, INS_sve_scvtf, INS_sve_ucvtf, INS_invalid, INS_sve_fcvt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ConvertToUInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtzu, INS_sve_fcvtzu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ConvertToUInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtzu, INS_sve_fcvtzu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, Count16BitElements, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_cnth, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(Sve, Count32BitElements, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_cntw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(Sve, Count64BitElements, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_cntd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(Sve, Count8BitElements, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_cntb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(Sve, CreateBreakAfterMask, -1, 2, INS_sve_brka, INS_sve_brka, INS_sve_brka, INS_sve_brka, INS_sve_brka, INS_sve_brka, INS_sve_brka, INS_sve_brka, INS_sve_brka, INS_sve_brka, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, CreateBreakAfterPropagateMask, -1, 3, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, CreateBreakBeforeMask, -1, 2, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, CreateBreakBeforePropagateMask, -1, 3, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, CreateBreakPropagateMask, -1, -1, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_HasRMWSemantics|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, CreateFalseMaskByte, -1, 0, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateFalseMaskDouble, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateFalseMaskInt16, -1, 0, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateFalseMaskInt32, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateFalseMaskInt64, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateFalseMaskSByte, -1, 0, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateFalseMaskSingle, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateFalseMaskUInt16, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateFalseMaskUInt32, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateFalseMaskUInt64, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateMaskForFirstActiveElement, -1, 2, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, CreateMaskForNextActiveElement, -1, 2, INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, CreateTrueMaskByte, -1, 1, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateTrueMaskDouble, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateTrueMaskInt16, -1, 1, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateTrueMaskInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateTrueMaskInt64, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateTrueMaskSByte, -1, 1, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateTrueMaskSingle, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateTrueMaskUInt16, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateTrueMaskUInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateTrueMaskUInt64, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskByte, -1, 2, INS_invalid, INS_sve_whilelt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskDouble, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilelt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_whilelt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilelt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilelt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskSByte, -1, 2, INS_sve_whilelt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskSingle, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilelt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilelt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilelt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilelt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskByte, -1, 2, INS_invalid, INS_sve_whilele, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskDouble, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilele, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_whilele, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilele, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilele, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskSByte, -1, 2, INS_sve_whilele, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskSingle, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilele, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilele, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilele, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilele, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, Divide, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sdiv, INS_sve_udiv, INS_sve_sdiv, INS_sve_udiv, INS_sve_fdiv, INS_sve_fdiv, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, DotProduct, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sdot, INS_sve_udot, INS_sve_sdot, INS_sve_udot, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, DotProductBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sdot, INS_sve_udot, INS_sve_sdot, INS_sve_udot, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve, DuplicateSelectedScalarToVector, -1, 2, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve, ExtractAfterLastActiveElement, -1, 2, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, ExtractAfterLastActiveElementScalar, 0, 2, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, ExtractLastActiveElement, -1, 2, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, ExtractLastActiveElementScalar, 0, 2, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, ExtractVector, -1, 3, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, FloatingPointExponentialAccelerator, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fexpa, INS_invalid, INS_sve_fexpa, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Sve, FusedMultiplyAdd, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmla, INS_sve_fmla, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, FusedMultiplyAddBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmla, INS_sve_fmla, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_FmaIntrinsic|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve, FusedMultiplyAddNegated, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fnmla, INS_sve_fnmla, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, FusedMultiplySubtract, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmls, INS_sve_fmls, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, FusedMultiplySubtractBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmls, INS_sve_fmls, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_FmaIntrinsic|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve, FusedMultiplySubtractNegated, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fnmls, INS_sve_fnmls, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, GatherPrefetch16Bit, -1, -1, INS_invalid, INS_invalid, INS_sve_prfh, INS_sve_prfh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherPrefetch32Bit, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_prfw, INS_sve_prfw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherPrefetch64Bit, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_prfd, INS_sve_prfd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherPrefetch8Bit, -1, -1, INS_sve_prfb, INS_sve_prfb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherVector, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1d, INS_sve_ld1d, INS_sve_ld1w, INS_sve_ld1d, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, GatherVectorByteZeroExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_sve_ld1b, INS_sve_ld1b, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, GatherVectorByteZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherVectorFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_sve_ldff1d, INS_sve_ldff1d, INS_sve_ldff1w, INS_sve_ldff1d, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherVectorInt16SignExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_sve_ld1sh, INS_sve_ld1sh, INS_sve_ld1sh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, GatherVectorInt16SignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherVectorInt16WithByteOffsetsSignExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_sve_ld1sh, INS_sve_ld1sh, INS_sve_ld1sh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, GatherVectorInt16WithByteOffsetsSignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherVectorInt32SignExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sw, INS_sve_ld1sw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, GatherVectorInt32SignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sw, INS_sve_ldff1sw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherVectorInt32WithByteOffsetsSignExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sw, INS_sve_ld1sw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, GatherVectorInt32WithByteOffsetsSignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sw, INS_sve_ldff1sw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherVectorSByteSignExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_sve_ld1sb, INS_sve_ld1sb, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, GatherVectorSByteSignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherVectorUInt16WithByteOffsetsZeroExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_sve_ld1h, INS_sve_ld1h, INS_sve_ld1h, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, GatherVectorUInt16WithByteOffsetsZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherVectorUInt16ZeroExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_sve_ld1h, INS_sve_ld1h, INS_sve_ld1h, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, GatherVectorUInt16ZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherVectorUInt32WithByteOffsetsZeroExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, GatherVectorUInt32WithByteOffsetsZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherVectorUInt32ZeroExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, GatherVectorUInt32ZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherVectorWithByteOffsetFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_sve_ldff1d, INS_sve_ldff1d, INS_sve_ldff1w, INS_sve_ldff1d, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherVectorWithByteOffsets, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1d, INS_sve_ld1d, INS_sve_ld1w, INS_sve_ld1d, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, GetActiveElementCount, -1, 2, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation) +HARDWARE_INTRINSIC(Sve, GetFfrByte, -1, 0, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GetFfrDouble, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GetFfrInt16, -1, 0, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GetFfrInt32, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GetFfrInt64, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GetFfrSByte, -1, 0, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GetFfrSingle, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GetFfrUInt16, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GetFfrUInt32, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GetFfrUInt64, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, InsertIntoShiftedVector, -1, 2, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, LeadingSignCount, -1, -1, INS_sve_cls, INS_invalid, INS_sve_cls, INS_invalid, INS_sve_cls, INS_invalid, INS_sve_cls, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, LeadingZeroCount, -1, -1, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, Load2xVectorAndUnzip, -1, 2, INS_sve_ld2b, INS_sve_ld2b, INS_sve_ld2h, INS_sve_ld2h, INS_sve_ld2w, INS_sve_ld2w, INS_sve_ld2d, INS_sve_ld2d, INS_sve_ld2w, INS_sve_ld2d, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Sve, Load3xVectorAndUnzip, -1, 2, INS_sve_ld3b, INS_sve_ld3b, INS_sve_ld3h, INS_sve_ld3h, INS_sve_ld3w, INS_sve_ld3w, INS_sve_ld3d, INS_sve_ld3d, INS_sve_ld3w, INS_sve_ld3d, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Sve, Load4xVectorAndUnzip, -1, 2, INS_sve_ld4b, INS_sve_ld4b, INS_sve_ld4h, INS_sve_ld4h, INS_sve_ld4w, INS_sve_ld4w, INS_sve_ld4d, INS_sve_ld4d, INS_sve_ld4w, INS_sve_ld4d, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Sve, LoadVector, -1, 2, INS_sve_ld1b, INS_sve_ld1b, INS_sve_ld1h, INS_sve_ld1h, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1d, INS_sve_ld1d, INS_sve_ld1w, INS_sve_ld1d, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVector128AndReplicateToVector, -1, 2, INS_sve_ld1rqb, INS_sve_ld1rqb, INS_sve_ld1rqh, INS_sve_ld1rqh, INS_sve_ld1rqw, INS_sve_ld1rqw, INS_sve_ld1rqd, INS_sve_ld1rqd, INS_sve_ld1rqw, INS_sve_ld1rqd, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToInt16, -1, -1, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToUInt16, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToUInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToUInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorFirstFaulting, -1, -1, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1w, INS_sve_ldff1w, INS_sve_ldff1d, INS_sve_ldff1d, INS_sve_ldff1w, INS_sve_ldff1d, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToUInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToUInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorInt32NonFaultingSignExtendToInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorInt32NonFaultingSignExtendToUInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorInt32SignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sw, INS_sve_ldff1sw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorInt32SignExtendToInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorInt32SignExtendToUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorNonFaulting, -1, -1, INS_sve_ldnf1b, INS_sve_ldnf1b, INS_sve_ldnf1h, INS_sve_ldnf1h, INS_sve_ldnf1w, INS_sve_ldnf1w, INS_sve_ldnf1d, INS_sve_ldnf1d, INS_sve_ldnf1w, INS_sve_ldnf1d, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorNonTemporal, -1, 2, INS_sve_ldnt1b, INS_sve_ldnt1b, INS_sve_ldnt1h, INS_sve_ldnt1h, INS_sve_ldnt1w, INS_sve_ldnt1w, INS_sve_ldnt1d, INS_sve_ldnt1d, INS_sve_ldnt1w, INS_sve_ldnt1d, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToInt16, -1, -1, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToUInt16, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToUInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToUInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToUInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToUInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt32NonFaultingZeroExtendToInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1w, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt32NonFaultingZeroExtendToUInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1w, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, Max, -1, -1, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_fmax, INS_sve_fmax, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, MaxAcross, -1, -1, INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_fmaxv, INS_sve_fmaxv, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) +HARDWARE_INTRINSIC(Sve, MaxNumber, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmaxnm, INS_sve_fmaxnm, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, MaxNumberAcross, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmaxnmv, INS_sve_fmaxnmv, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) +HARDWARE_INTRINSIC(Sve, Min, -1, -1, INS_sve_smin, INS_sve_umin, INS_sve_smin, INS_sve_umin, INS_sve_smin, INS_sve_umin, INS_sve_smin, INS_sve_umin, INS_sve_fmin, INS_sve_fmin, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, MinAcross, -1, -1, INS_sve_sminv, INS_sve_uminv, INS_sve_sminv, INS_sve_uminv, INS_sve_sminv, INS_sve_uminv, INS_sve_sminv, INS_sve_uminv, INS_sve_fminv, INS_sve_fminv, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) +HARDWARE_INTRINSIC(Sve, MinNumber, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fminnm, INS_sve_fminnm, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, MinNumberAcross, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fminnmv, INS_sve_fminnmv, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) +HARDWARE_INTRINSIC(Sve, Multiply, -1, 2, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_fmul, INS_sve_fmul, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, MultiplyAdd, -1, -1, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, MultiplyAddRotateComplex, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcmla, INS_sve_fcmla, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve, MultiplyAddRotateComplexBySelectedScalar, -1, 5, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcmla, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, MultiplyBySelectedScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmul, INS_sve_fmul, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve, MultiplyExtended, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmulx, INS_sve_fmulx, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, MultiplySubtract, -1, -1, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, Negate, -1, -1, INS_sve_neg, INS_invalid, INS_sve_neg, INS_invalid, INS_sve_neg, INS_invalid, INS_sve_neg, INS_invalid, INS_sve_fneg, INS_sve_fneg, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, Not, -1, -1, INS_sve_not, INS_sve_not, INS_sve_not, INS_sve_not, INS_sve_not, INS_sve_not, INS_sve_not, INS_sve_not, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation) +HARDWARE_INTRINSIC(Sve, Or, -1, -1, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_HasAllMaskVariant) +HARDWARE_INTRINSIC(Sve, OrAcross, -1, -1, INS_sve_orv, INS_sve_orv, INS_sve_orv, INS_sve_orv, INS_sve_orv, INS_sve_orv, INS_sve_orv, INS_sve_orv, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) +HARDWARE_INTRINSIC(Sve, PopCount, -1, -1, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, Prefetch16Bit, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_sve_prfh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, Prefetch32Bit, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_prfw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, Prefetch64Bit, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_prfd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, Prefetch8Bit, -1, 3, INS_invalid, INS_sve_prfb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, ReciprocalEstimate, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frecpe, INS_sve_frecpe, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve, ReciprocalExponent, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frecpx, INS_sve_frecpx, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ReciprocalSqrtEstimate, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frsqrte, INS_sve_frsqrte, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve, ReciprocalSqrtStep, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frsqrts, INS_sve_frsqrts, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve, ReciprocalStep, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frecps, INS_sve_frecps, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve, ReverseBits, -1, -1, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ReverseElement, -1, 1, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant) +HARDWARE_INTRINSIC(Sve, ReverseElement16, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_revh, INS_sve_revh, INS_sve_revh, INS_sve_revh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ReverseElement32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_revw, INS_sve_revw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ReverseElement8, -1, -1, INS_invalid, INS_invalid, INS_sve_revb, INS_sve_revb, INS_sve_revb, INS_sve_revb, INS_sve_revb, INS_sve_revb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, RoundAwayFromZero, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frinta, INS_sve_frinta, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, RoundToNearest, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frintn, INS_sve_frintn, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, RoundToNegativeInfinity, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frintm, INS_sve_frintm, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, RoundToPositiveInfinity, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frintp, INS_sve_frintp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, RoundToZero, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frintz, INS_sve_frintz, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, SaturatingDecrementBy16BitElementCount, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdech, INS_sve_uqdech, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, SaturatingDecrementBy32BitElementCount, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdecw, INS_sve_uqdecw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, SaturatingDecrementBy64BitElementCount, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdecd, INS_sve_uqdecd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, SaturatingDecrementBy8BitElementCount, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdecb, INS_sve_uqdecb, INS_sve_sqdecb, INS_sve_uqdecb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, SaturatingDecrementByActiveElementCount, -1, 2, INS_invalid, INS_sve_sqdecp, INS_sve_sqdecp, INS_sve_sqdecp, INS_sve_sqdecp, INS_sve_sqdecp, INS_sve_sqdecp, INS_sve_sqdecp, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy16BitElementCount, -1, 3, INS_invalid, INS_invalid, INS_sve_sqinch, INS_sve_uqinch, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy32BitElementCount, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqincw, INS_sve_uqincw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy64BitElementCount, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqincd, INS_sve_uqincd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy8BitElementCount, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqincb, INS_sve_uqincb, INS_sve_sqincb, INS_sve_uqincb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, SaturatingIncrementByActiveElementCount, -1, 2, INS_invalid, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, Scale, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fscale, INS_sve_fscale, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, Scatter, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1w, INS_sve_st1w, INS_sve_st1d, INS_sve_st1d, INS_sve_st1w, INS_sve_st1d, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, Scatter16BitNarrowing, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1h, INS_sve_st1h, INS_sve_st1h, INS_sve_st1h, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, Scatter16BitWithByteOffsetsNarrowing, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1h, INS_sve_st1h, INS_sve_st1h, INS_sve_st1h, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, Scatter32BitNarrowing, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1w, INS_sve_st1w, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, Scatter32BitWithByteOffsetsNarrowing, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1w, INS_sve_st1w, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, Scatter8BitNarrowing, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1b, INS_sve_st1b, INS_sve_st1b, INS_sve_st1b, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, Scatter8BitWithByteOffsetsNarrowing, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1b, INS_sve_st1b, INS_sve_st1b, INS_sve_st1b, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ScatterWithByteOffsets, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1w, INS_sve_st1w, INS_sve_st1d, INS_sve_st1d, INS_sve_st1w, INS_sve_st1d, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, SetFfr, -1, 1, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialSideEffect_Other|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, ShiftLeftLogical, -1, -1, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, ShiftRightArithmetic, -1, -1, INS_sve_asr, INS_invalid, INS_sve_asr, INS_invalid, INS_sve_asr, INS_invalid, INS_sve_asr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, ShiftRightArithmeticForDivide, -1, -1, INS_sve_asrd, INS_invalid, INS_sve_asrd, INS_invalid, INS_sve_asrd, INS_invalid, INS_sve_asrd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve, ShiftRightLogical, -1, -1, INS_invalid, INS_sve_lsr, INS_invalid, INS_sve_lsr, INS_invalid, INS_sve_lsr, INS_invalid, INS_sve_lsr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, SignExtend16, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sxth, INS_invalid, INS_sve_sxth, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, SignExtend32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sxtw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, SignExtend8, -1, -1, INS_invalid, INS_invalid, INS_sve_sxtb, INS_invalid, INS_sve_sxtb, INS_invalid, INS_sve_sxtb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, SignExtendWideningLower, -1, 1, INS_sve_sunpklo, INS_invalid, INS_sve_sunpklo, INS_invalid, INS_sve_sunpklo, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Sve, SignExtendWideningUpper, -1, 1, INS_sve_sunpkhi, INS_invalid, INS_sve_sunpkhi, INS_invalid, INS_sve_sunpkhi, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Sve, Splice, -1, 3, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, Sqrt, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fsqrt, INS_sve_fsqrt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, StoreAndZip, -1, 3, INS_sve_st1b, INS_sve_st1b, INS_sve_st1h, INS_sve_st1h, INS_sve_st1w, INS_sve_st1w, INS_sve_st1d, INS_sve_st1d, INS_sve_st1w, INS_sve_st1d, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, StoreNarrowing, -1, 3, INS_sve_st1b, INS_sve_st1b, INS_sve_st1h, INS_sve_st1h, INS_sve_st1w, INS_sve_st1w, INS_sve_st1d, INS_sve_st1d, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, StoreNonTemporal, -1, 3, INS_sve_stnt1b, INS_sve_stnt1b, INS_sve_stnt1h, INS_sve_stnt1h, INS_sve_stnt1w, INS_sve_stnt1w, INS_sve_stnt1d, INS_sve_stnt1d, INS_sve_stnt1w, INS_sve_stnt1d, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, Subtract, -1, 2, INS_sve_sub, INS_sve_sub, INS_sve_sub, INS_sve_sub, INS_sve_sub, INS_sve_sub, INS_sve_sub, INS_sve_sub, INS_sve_fsub, INS_sve_fsub, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, SubtractSaturate, -1, 2, INS_sve_sqsub, INS_sve_uqsub, INS_sve_sqsub, INS_sve_uqsub, INS_sve_sqsub, INS_sve_uqsub, INS_sve_sqsub, INS_sve_uqsub, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve, TestAnyTrue, -1, 2, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, TestFirstTrue, -1, 2, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, TestLastTrue, -1, 2, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, TransposeEven, -1, 2, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant) +HARDWARE_INTRINSIC(Sve, TransposeOdd, -1, 2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant) +HARDWARE_INTRINSIC(Sve, TrigonometricMultiplyAddCoefficient, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ftmad, INS_sve_ftmad, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, TrigonometricSelectCoefficient, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ftssel, INS_sve_ftssel, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Sve, TrigonometricStartingValue, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ftsmul, INS_sve_ftsmul, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Sve, UnzipEven, -1, 2, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant) +HARDWARE_INTRINSIC(Sve, UnzipOdd, -1, 2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant) +HARDWARE_INTRINSIC(Sve, VectorTableLookup, -1, 2, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve, Xor, -1, -1, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_HasAllMaskVariant) +HARDWARE_INTRINSIC(Sve, XorAcross, -1, -1, INS_sve_eorv, INS_sve_eorv, INS_sve_eorv, INS_sve_eorv, INS_sve_eorv, INS_sve_eorv, INS_sve_eorv, INS_sve_eorv, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) +HARDWARE_INTRINSIC(Sve, ZeroExtend16, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_uxth, INS_invalid, INS_sve_uxth, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ZeroExtend32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_uxtw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ZeroExtend8, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_sve_uxtb, INS_invalid, INS_sve_uxtb, INS_invalid, INS_sve_uxtb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ZeroExtendWideningLower, -1, 1, INS_invalid, INS_sve_uunpklo, INS_invalid, INS_sve_uunpklo, INS_invalid, INS_sve_uunpklo, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Sve, ZeroExtendWideningUpper, -1, 1, INS_invalid, INS_sve_uunpkhi, INS_invalid, INS_sve_uunpkhi, INS_invalid, INS_sve_uunpkhi, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Sve, ZipHigh, -1, 2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant) +HARDWARE_INTRINSIC(Sve, ZipLow, -1, 2, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant) #define LAST_NI_Sve NI_Sve_ZipLow // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -328,209 +328,209 @@ HARDWARE_INTRINSIC(Sve, ZipLow, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // SVE2 Intrinsics #define FIRST_NI_Sve2 NI_Sve2_AbsSaturate -HARDWARE_INTRINSIC(Sve2, AbsSaturate, -1, -1, INS_sve_sqabs, INS_invalid, INS_sve_sqabs, INS_invalid, INS_sve_sqabs, INS_invalid, INS_sve_sqabs, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, AbsoluteDifferenceAdd, -1, 3, INS_sve_saba, INS_sve_uaba, INS_sve_saba, INS_sve_uaba, INS_sve_saba, INS_sve_uaba, INS_sve_saba, INS_sve_uaba, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, AbsoluteDifferenceWideningEven, -1, 2, INS_invalid, INS_invalid, INS_sve_sabdlb, INS_sve_uabdlb, INS_sve_sabdlb, INS_sve_uabdlb, INS_sve_sabdlb, INS_sve_uabdlb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, AbsoluteDifferenceWideningLowerAndAddEven, -1, 3, INS_invalid, INS_invalid, INS_sve_sabalb, INS_sve_uabalb, INS_sve_sabalb, INS_sve_uabalb, INS_sve_sabalb, INS_sve_uabalb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, AbsoluteDifferenceWideningLowerAndAddOdd, -1, 3, INS_invalid, INS_invalid, INS_sve_sabalt, INS_sve_uabalt, INS_sve_sabalt, INS_sve_uabalt, INS_sve_sabalt, INS_sve_uabalt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, AbsoluteDifferenceWideningOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_sabdlt, INS_sve_uabdlt, INS_sve_sabdlt, INS_sve_uabdlt, INS_sve_sabdlt, INS_sve_uabdlt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, AddCarryWideningEven, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_adclb, INS_invalid, INS_sve_adclb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, AddCarryWideningOdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_adclt, INS_invalid, INS_sve_adclt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, AddHighNarrowingEven, -1, 2, INS_sve_addhnb, INS_sve_addhnb, INS_sve_addhnb, INS_sve_addhnb, INS_sve_addhnb, INS_sve_addhnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, AddHighNarrowingOdd, -1, 3, INS_sve_addhnt, INS_sve_addhnt, INS_sve_addhnt, INS_sve_addhnt, INS_sve_addhnt, INS_sve_addhnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, AddPairwise, -1, -1, INS_sve_addp, INS_sve_addp, INS_sve_addp, INS_sve_addp, INS_sve_addp, INS_sve_addp, INS_sve_addp, INS_sve_addp, INS_sve_faddp, INS_sve_faddp, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) -HARDWARE_INTRINSIC(Sve2, AddPairwiseWideningAndAdd, -1, -1, INS_invalid, INS_invalid, INS_sve_sadalp, INS_sve_uadalp, INS_sve_sadalp, INS_sve_uadalp, INS_sve_sadalp, INS_sve_uadalp, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, AddRotateComplex, -1, 3, INS_sve_cadd, INS_sve_cadd, INS_sve_cadd, INS_sve_cadd, INS_sve_cadd, INS_sve_cadd, INS_sve_cadd, INS_sve_cadd, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve2, AddRoundedHighNarrowingEven, -1, 2, INS_sve_raddhnb, INS_sve_raddhnb, INS_sve_raddhnb, INS_sve_raddhnb, INS_sve_raddhnb, INS_sve_raddhnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, AddRoundedHighNarrowingOdd, -1, 3, INS_sve_raddhnt, INS_sve_raddhnt, INS_sve_raddhnt, INS_sve_raddhnt, INS_sve_raddhnt, INS_sve_raddhnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, AddSaturate, -1, -1, INS_sve_sqadd, INS_sve_uqadd, INS_sve_sqadd, INS_sve_uqadd, INS_sve_sqadd, INS_sve_uqadd, INS_sve_sqadd, INS_sve_uqadd, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve2, AddSaturateRotateComplex, -1, 3, INS_sve_sqcadd, INS_invalid, INS_sve_sqcadd, INS_invalid, INS_sve_sqcadd, INS_invalid, INS_sve_sqcadd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve2, AddWideningEven, -1, 2, INS_invalid, INS_invalid, INS_sve_saddwb, INS_sve_uaddwb, INS_sve_saddwb, INS_sve_uaddwb, INS_sve_saddwb, INS_sve_uaddwb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve2, AddWideningEvenOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_saddlbt, INS_invalid, INS_sve_saddlbt, INS_invalid, INS_sve_saddlbt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, AddWideningOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_saddwt, INS_sve_uaddwt, INS_sve_saddwt, INS_sve_uaddwt, INS_sve_saddwt, INS_sve_uaddwt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve2, BitwiseClearXor, -1, 3, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, BitwiseSelect, -1, 3, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, BitwiseSelectLeftInverted, -1, 3, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, BitwiseSelectRightInverted, -1, 3, INS_sve_bsl2n, INS_sve_bsl2n, INS_sve_bsl2n, INS_sve_bsl2n, INS_sve_bsl2n, INS_sve_bsl2n, INS_sve_bsl2n, INS_sve_bsl2n, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ConvertToDoubleOdd, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtlt, INS_sve_fcvtlt, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, ConvertToSingleEvenRoundToOdd, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtx, INS_sve_fcvtx, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, ConvertToSingleOdd, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtnt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ConvertToSingleOddRoundToOdd, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtxnt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, CountMatchingElements, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_histcnt, INS_sve_histcnt, INS_sve_histcnt, INS_sve_histcnt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve2, CountMatchingElementsIn128BitSegments, -1, 2, INS_sve_histseg, INS_sve_histseg, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskByte, -1, 2, INS_invalid, INS_sve_whilegt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskDouble, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilegt, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_whilegt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilegt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilegt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskSByte, -1, 2, INS_sve_whilegt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskSingle, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilegt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilegt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilegt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilegt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskByte, -1, 2, INS_invalid, INS_sve_whilege, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskDouble, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilege, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_whilege, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilege, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilege, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskSByte, -1, 2, INS_sve_whilege, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskSingle, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilege, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilege, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilege, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilege, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskByte, -1, 2, INS_invalid, INS_sve_whilerw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskDouble, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilerw, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_whilerw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilerw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilerw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskSByte, -1, 2, INS_sve_whilerw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskSingle, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilerw, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilerw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilerw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilerw, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskByte, -1, 2, INS_invalid, INS_sve_whilewr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskDouble, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilewr, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_whilewr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilewr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilewr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskSByte, -1, 2, INS_sve_whilewr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskSingle, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilewr, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilewr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilewr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilewr, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, DotProductRotateComplex, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_cdot, INS_invalid, INS_sve_cdot, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve2, DotProductRotateComplexBySelectedIndex, -1, 5, INS_sve_cdot, INS_invalid, INS_sve_cdot, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(Sve2, FusedAddHalving, -1, -1, INS_sve_shadd, INS_sve_uhadd, INS_sve_shadd, INS_sve_uhadd, INS_sve_shadd, INS_sve_uhadd, INS_sve_shadd, INS_sve_uhadd, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, FusedAddRoundedHalving, -1, -1, INS_sve_srhadd, INS_sve_urhadd, INS_sve_srhadd, INS_sve_urhadd, INS_sve_srhadd, INS_sve_urhadd, INS_sve_srhadd, INS_sve_urhadd, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, FusedSubtractHalving, -1, -1, INS_sve_shsub, INS_sve_uhsub, INS_sve_shsub, INS_sve_uhsub, INS_sve_shsub, INS_sve_uhsub, INS_sve_shsub, INS_sve_uhsub, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, GatherVectorByteZeroExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1b, INS_sve_ldnt1b, INS_sve_ldnt1b, INS_sve_ldnt1b, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve2, GatherVectorInt16SignExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1sh, INS_sve_ldnt1sh, INS_sve_ldnt1sh, INS_sve_ldnt1sh, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve2, GatherVectorInt16WithByteOffsetsSignExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1sh, INS_sve_ldnt1sh, INS_sve_ldnt1sh, INS_sve_ldnt1sh, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve2, GatherVectorInt32SignExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1sw, INS_sve_ldnt1sw, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve2, GatherVectorInt32WithByteOffsetsSignExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1sw, INS_sve_ldnt1sw, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve2, GatherVectorNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1w, INS_sve_ldnt1w, INS_sve_ldnt1d, INS_sve_ldnt1d, INS_sve_ldnt1w, INS_sve_ldnt1d, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve2, GatherVectorSByteSignExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1sb, INS_sve_ldnt1sb, INS_sve_ldnt1sb, INS_sve_ldnt1sb, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve2, GatherVectorUInt16WithByteOffsetsZeroExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1h, INS_sve_ldnt1h, INS_sve_ldnt1h, INS_sve_ldnt1h, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve2, GatherVectorUInt16ZeroExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1h, INS_sve_ldnt1h, INS_sve_ldnt1h, INS_sve_ldnt1h, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve2, GatherVectorUInt32WithByteOffsetsZeroExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1w, INS_sve_ldnt1w, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve2, GatherVectorUInt32ZeroExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1w, INS_sve_ldnt1w, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve2, GatherVectorWithByteOffsetsNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1w, INS_sve_ldnt1w, INS_sve_ldnt1d, INS_sve_ldnt1d, INS_sve_ldnt1w, INS_sve_ldnt1d, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve2, InterleavingXorEvenOdd, -1, 3, INS_sve_eorbt, INS_sve_eorbt, INS_sve_eorbt, INS_sve_eorbt, INS_sve_eorbt, INS_sve_eorbt, INS_sve_eorbt, INS_sve_eorbt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, InterleavingXorOddEven, -1, 3, INS_sve_eortb, INS_sve_eortb, INS_sve_eortb, INS_sve_eortb, INS_sve_eortb, INS_sve_eortb, INS_sve_eortb, INS_sve_eortb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, Log2, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_flogb, INS_invalid, INS_sve_flogb, INS_invalid, INS_sve_flogb, INS_sve_flogb, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, Match, -1, 3, INS_sve_match, INS_sve_match, INS_sve_match, INS_sve_match, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve2, MaxNumberPairwise, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmaxnmp, INS_sve_fmaxnmp, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, MaxPairwise, -1, -1, INS_sve_smaxp, INS_sve_umaxp, INS_sve_smaxp, INS_sve_umaxp, INS_sve_smaxp, INS_sve_umaxp, INS_sve_smaxp, INS_sve_umaxp, INS_sve_fmaxp, INS_sve_fmaxp, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_ReduceOperation) -HARDWARE_INTRINSIC(Sve2, MinNumberPairwise, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fminnmp, INS_sve_fminnmp, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, MinPairwise, -1, -1, INS_sve_sminp, INS_sve_uminp, INS_sve_sminp, INS_sve_uminp, INS_sve_sminp, INS_sve_uminp, INS_sve_sminp, INS_sve_uminp, INS_sve_fminp, INS_sve_fminp, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_ReduceOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyAddBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, MultiplyAddRotateComplex, -1, 4, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve2, MultiplyAddRotateComplexBySelectedScalar, -1, 5, INS_invalid, INS_invalid, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve2, MultiplyAddRoundedDoublingSaturateHighRotateComplex, -1, 4, INS_sve_sqrdcmlah, INS_invalid, INS_sve_sqrdcmlah, INS_invalid, INS_sve_sqrdcmlah, INS_invalid, INS_sve_sqrdcmlah, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve2, MultiplyAddRoundedDoublingSaturateHighRotateComplexBySelectedScalar, -1, 5, INS_invalid, INS_invalid, INS_sve_sqrdcmlah, INS_invalid, INS_sve_sqrdcmlah, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalar, -1, 3, INS_invalid, INS_invalid, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalarWideningEven, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_smullb, INS_sve_umullb, INS_sve_smullb, INS_sve_umullb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalarWideningEvenAndAdd, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_smlalb, INS_sve_umlalb, INS_sve_smlalb, INS_sve_umlalb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalarWideningEvenAndSubtract, -1, 4, INS_invalid, INS_invalid, INS_sve_smlslb, INS_sve_umlslb, INS_sve_smlslb, INS_sve_umlslb, INS_sve_smlslb, INS_sve_umlslb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalarWideningOdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_smullt, INS_sve_umullt, INS_sve_smullt, INS_sve_umullt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalarWideningOddAndAdd, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_smlalt, INS_sve_umlalt, INS_sve_smlalt, INS_sve_umlalt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalarWideningOddAndSubtract, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_smlslt, INS_sve_umlslt, INS_sve_smlslt, INS_sve_umlslt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingBySelectedScalarSaturateHigh, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdmulh, INS_invalid, INS_sve_sqdmulh, INS_invalid, INS_sve_sqdmulh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingSaturateHigh, -1, 2, INS_sve_sqdmulh, INS_invalid, INS_sve_sqdmulh, INS_invalid, INS_sve_sqdmulh, INS_invalid, INS_sve_sqdmulh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningAndAddSaturateEven, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdmlalb, INS_invalid, INS_sve_sqdmlalb, INS_invalid, INS_sve_sqdmlalb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningAndAddSaturateEvenOdd, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdmlalbt, INS_invalid, INS_sve_sqdmlalbt, INS_invalid, INS_sve_sqdmlalbt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningAndAddSaturateOdd, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdmlalt, INS_invalid, INS_sve_sqdmlalt, INS_invalid, INS_sve_sqdmlalt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningAndSubtractSaturateEven, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdmlslb, INS_invalid, INS_sve_sqdmlslb, INS_invalid, INS_sve_sqdmlslb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningAndSubtractSaturateEvenOdd, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdmlslbt, INS_invalid, INS_sve_sqdmlslbt, INS_invalid, INS_sve_sqdmlslbt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningAndSubtractSaturateOdd, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdmlslt, INS_invalid, INS_sve_sqdmlslt, INS_invalid, INS_sve_sqdmlslt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningBySelectedScalarAndAddSaturateEven, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdmlalb, INS_invalid, INS_sve_sqdmlalb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningBySelectedScalarAndAddSaturateOdd, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdmlalt, INS_invalid, INS_sve_sqdmlalt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningBySelectedScalarAndSubtractSaturateEven, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdmlslb, INS_invalid, INS_sve_sqdmlslb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningBySelectedScalarAndSubtractSaturateOdd, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdmlslt, INS_invalid, INS_sve_sqdmlslt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningSaturateEven, -1, 2, INS_invalid, INS_invalid, INS_sve_sqdmullb, INS_invalid, INS_sve_sqdmullb, INS_invalid, INS_sve_sqdmullb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningSaturateEvenBySelectedScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdmullb, INS_invalid, INS_sve_sqdmullb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningSaturateOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_sqdmullt, INS_invalid, INS_sve_sqdmullt, INS_invalid, INS_sve_sqdmullt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningSaturateOddBySelectedScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdmullt, INS_invalid, INS_sve_sqdmullt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyRoundedDoublingBySelectedScalarSaturateHigh, -1, 3, INS_invalid, INS_invalid, INS_sve_sqrdmulh, INS_invalid, INS_sve_sqrdmulh, INS_invalid, INS_sve_sqrdmulh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyRoundedDoublingSaturateAndAddHigh, -1, 3, INS_sve_sqrdmlah, INS_invalid, INS_sve_sqrdmlah, INS_invalid, INS_sve_sqrdmlah, INS_invalid, INS_sve_sqrdmlah, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, MultiplyRoundedDoublingSaturateAndSubtractHigh, -1, 3, INS_sve_sqrdmlsh, INS_invalid, INS_sve_sqrdmlsh, INS_invalid, INS_sve_sqrdmlsh, INS_invalid, INS_sve_sqrdmlsh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, MultiplyRoundedDoublingSaturateBySelectedScalarAndAddHigh, -1, 4, INS_invalid, INS_invalid, INS_sve_sqrdmlah, INS_invalid, INS_sve_sqrdmlah, INS_invalid, INS_sve_sqrdmlah, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyRoundedDoublingSaturateBySelectedScalarAndSubtractHigh, -1, 4, INS_invalid, INS_invalid, INS_sve_sqrdmlsh, INS_invalid, INS_sve_sqrdmlsh, INS_invalid, INS_sve_sqrdmlsh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyRoundedDoublingSaturateHigh, -1, 2, INS_sve_sqrdmulh, INS_invalid, INS_sve_sqrdmulh, INS_invalid, INS_sve_sqrdmulh, INS_invalid, INS_sve_sqrdmulh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, MultiplySubtractBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyWideningEven, -1, 2, INS_invalid, INS_invalid, INS_sve_smullb, INS_sve_umullb, INS_sve_smullb, INS_sve_umullb, INS_sve_smullb, INS_sve_umullb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, MultiplyWideningEvenAndAdd, -1, 3, INS_invalid, INS_invalid, INS_sve_smlalb, INS_sve_umlalb, INS_sve_smlalb, INS_sve_umlalb, INS_sve_smlalb, INS_sve_umlalb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, MultiplyWideningEvenAndSubtract, -1, 3, INS_invalid, INS_invalid, INS_sve_smlslb, INS_sve_umlslb, INS_sve_smlslb, INS_sve_umlslb, INS_sve_smlslb, INS_sve_umlslb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, MultiplyWideningOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_smullt, INS_sve_umullt, INS_sve_smullt, INS_sve_umullt, INS_sve_smullt, INS_sve_umullt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, MultiplyWideningOddAndAdd, -1, 3, INS_invalid, INS_invalid, INS_sve_smlalt, INS_sve_umlalt, INS_sve_smlalt, INS_sve_umlalt, INS_sve_smlalt, INS_sve_umlalt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, MultiplyWideningOddAndSubtract, -1, 3, INS_invalid, INS_invalid, INS_sve_smlslt, INS_sve_umlslt, INS_sve_smlslt, INS_sve_umlslt, INS_sve_smlslt, INS_sve_umlslt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, NegateSaturate, -1, -1, INS_sve_sqneg, INS_invalid, INS_sve_sqneg, INS_invalid, INS_sve_sqneg, INS_invalid, INS_sve_sqneg, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, NoMatch, -1, 3, INS_sve_nmatch, INS_sve_nmatch, INS_sve_nmatch, INS_sve_nmatch, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve2, PolynomialMultiply, -1, 2, INS_sve_pmul, INS_sve_pmul, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, PolynomialMultiplyWideningEven, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_pmullb, INS_invalid, INS_invalid, INS_invalid, INS_sve_pmullb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, PolynomialMultiplyWideningOdd, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_pmullt, INS_invalid, INS_invalid, INS_invalid, INS_sve_pmullt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, ReciprocalEstimate, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_urecpe, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, ReciprocalSqrtEstimate, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ursqrte, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, SaturatingExtractNarrowingLower, -1, 1, INS_sve_sqxtnb, INS_sve_uqxtnb, INS_sve_sqxtnb, INS_sve_uqxtnb, INS_sve_sqxtnb, INS_sve_uqxtnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, SaturatingExtractNarrowingUpper, -1, 2, INS_sve_sqxtnt, INS_sve_uqxtnt, INS_sve_sqxtnt, INS_sve_uqxtnt, INS_sve_sqxtnt, INS_sve_uqxtnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, SaturatingExtractUnsignedNarrowingLower, -1, 1, INS_invalid, INS_sve_sqxtunb, INS_invalid, INS_sve_sqxtunb, INS_invalid, INS_sve_sqxtunb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, SaturatingExtractUnsignedNarrowingUpper, -1, 2, INS_invalid, INS_sve_sqxtunt, INS_invalid, INS_sve_sqxtunt, INS_invalid, INS_sve_sqxtunt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, Scatter16BitNarrowingNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1h, INS_sve_stnt1h, INS_sve_stnt1h, INS_sve_stnt1h, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, Scatter16BitWithByteOffsetsNarrowingNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1h, INS_sve_stnt1h, INS_sve_stnt1h, INS_sve_stnt1h, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, Scatter32BitNarrowingNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1w, INS_sve_stnt1w, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, Scatter32BitWithByteOffsetsNarrowingNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1w, INS_sve_stnt1w, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, Scatter8BitNarrowingNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1b, INS_sve_stnt1b, INS_sve_stnt1b, INS_sve_stnt1b, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, Scatter8BitWithByteOffsetsNarrowingNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1b, INS_sve_stnt1b, INS_sve_stnt1b, INS_sve_stnt1b, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, ScatterNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1w, INS_sve_stnt1w, INS_sve_stnt1d, INS_sve_stnt1d, INS_sve_stnt1w, INS_sve_stnt1d, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, ScatterWithByteOffsetsNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1w, INS_sve_stnt1w, INS_sve_stnt1d, INS_sve_stnt1d, INS_sve_stnt1w, INS_sve_stnt1d, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, ShiftArithmeticRounded, -1, -1, INS_sve_srshl, INS_invalid, INS_sve_srshl, INS_invalid, INS_sve_srshl, INS_invalid, INS_sve_srshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, ShiftArithmeticRoundedSaturate, -1, -1, INS_sve_sqrshl, INS_invalid, INS_sve_sqrshl, INS_invalid, INS_sve_sqrshl, INS_invalid, INS_sve_sqrshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, ShiftArithmeticSaturate, -1, -1, INS_sve_sqshl, INS_invalid, INS_sve_sqshl, INS_invalid, INS_sve_sqshl, INS_invalid, INS_sve_sqshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, ShiftLeftAndInsert, -1, 3, INS_sve_sli, INS_sve_sli, INS_sve_sli, INS_sve_sli, INS_sve_sli, INS_sve_sli, INS_sve_sli, INS_sve_sli, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftLeftLogicalSaturate, -1, -1, INS_invalid, INS_sve_uqshl, INS_invalid, INS_sve_uqshl, INS_invalid, INS_sve_uqshl, INS_invalid, INS_sve_uqshl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftLeftLogicalSaturateUnsigned, -1, -1, INS_invalid, INS_sve_sqshlu, INS_invalid, INS_sve_sqshlu, INS_invalid, INS_sve_sqshlu, INS_invalid, INS_sve_sqshlu, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftLeftLogicalWideningEven, -1, 2, INS_invalid, INS_invalid, INS_sve_sshllb, INS_sve_ushllb, INS_sve_sshllb, INS_sve_ushllb, INS_sve_sshllb, INS_sve_ushllb, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve2, ShiftLeftLogicalWideningOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_sshllt, INS_sve_ushllt, INS_sve_sshllt, INS_sve_ushllt, INS_sve_sshllt, INS_sve_ushllt, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve2, ShiftLogicalRounded, -1, -1, INS_invalid, INS_sve_urshl, INS_invalid, INS_sve_urshl, INS_invalid, INS_sve_urshl, INS_invalid, INS_sve_urshl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftLogicalRoundedSaturate, -1, -1, INS_invalid, INS_sve_uqrshl, INS_invalid, INS_sve_uqrshl, INS_invalid, INS_sve_uqrshl, INS_invalid, INS_sve_uqrshl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftRightAndInsert, -1, 3, INS_sve_sri, INS_sve_sri, INS_sve_sri, INS_sve_sri, INS_sve_sri, INS_sve_sri, INS_sve_sri, INS_sve_sri, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticAdd, -1, 3, INS_sve_ssra, INS_invalid, INS_sve_ssra, INS_invalid, INS_sve_ssra, INS_invalid, INS_sve_ssra, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticNarrowingSaturateEven, -1, 2, INS_sve_sqshrnb, INS_invalid, INS_sve_sqshrnb, INS_invalid, INS_sve_sqshrnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticNarrowingSaturateOdd, -1, 3, INS_sve_sqshrnt, INS_invalid, INS_sve_sqshrnt, INS_invalid, INS_sve_sqshrnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticNarrowingSaturateUnsignedEven, -1, 2, INS_invalid, INS_sve_sqshrunb, INS_invalid, INS_sve_sqshrunb, INS_invalid, INS_sve_sqshrunb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticNarrowingSaturateUnsignedOdd, -1, 3, INS_invalid, INS_sve_sqshrunt, INS_invalid, INS_sve_sqshrunt, INS_invalid, INS_sve_sqshrunt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticRounded, -1, -1, INS_sve_srshr, INS_invalid, INS_sve_srshr, INS_invalid, INS_sve_srshr, INS_invalid, INS_sve_srshr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticRoundedAdd, -1, 3, INS_sve_srsra, INS_invalid, INS_sve_srsra, INS_invalid, INS_sve_srsra, INS_invalid, INS_sve_srsra, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticRoundedNarrowingSaturateEven, -1, 2, INS_sve_sqrshrnb, INS_invalid, INS_sve_sqrshrnb, INS_invalid, INS_sve_sqrshrnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticRoundedNarrowingSaturateOdd, -1, 3, INS_sve_sqrshrnt, INS_invalid, INS_sve_sqrshrnt, INS_invalid, INS_sve_sqrshrnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticRoundedNarrowingSaturateUnsignedEven, -1, 2, INS_invalid, INS_sve_sqrshrunb, INS_invalid, INS_sve_sqrshrunb, INS_invalid, INS_sve_sqrshrunb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticRoundedNarrowingSaturateUnsignedOdd, -1, 3, INS_invalid, INS_sve_sqrshrunt, INS_invalid, INS_sve_sqrshrunt, INS_invalid, INS_sve_sqrshrunt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalAdd, -1, 3, INS_invalid, INS_sve_usra, INS_invalid, INS_sve_usra, INS_invalid, INS_sve_usra, INS_invalid, INS_sve_usra, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalNarrowingEven, -1, 2, INS_sve_shrnb, INS_sve_shrnb, INS_sve_shrnb, INS_sve_shrnb, INS_sve_shrnb, INS_sve_shrnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalNarrowingOdd, -1, 3, INS_sve_shrnt, INS_sve_shrnt, INS_sve_shrnt, INS_sve_shrnt, INS_sve_shrnt, INS_sve_shrnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalNarrowingSaturateEven, -1, 2, INS_invalid, INS_sve_uqshrnb, INS_invalid, INS_sve_uqshrnb, INS_invalid, INS_sve_uqshrnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalNarrowingSaturateOdd, -1, 3, INS_invalid, INS_sve_uqshrnt, INS_invalid, INS_sve_uqshrnt, INS_invalid, INS_sve_uqshrnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalRounded, -1, -1, INS_invalid, INS_sve_urshr, INS_invalid, INS_sve_urshr, INS_invalid, INS_sve_urshr, INS_invalid, INS_sve_urshr, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalRoundedAdd, -1, 3, INS_invalid, INS_sve_ursra, INS_invalid, INS_sve_ursra, INS_invalid, INS_sve_ursra, INS_invalid, INS_sve_ursra, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalRoundedNarrowingEven, -1, 2, INS_sve_rshrnb, INS_sve_rshrnb, INS_sve_rshrnb, INS_sve_rshrnb, INS_sve_rshrnb, INS_sve_rshrnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalRoundedNarrowingOdd, -1, 3, INS_sve_rshrnt, INS_sve_rshrnt, INS_sve_rshrnt, INS_sve_rshrnt, INS_sve_rshrnt, INS_sve_rshrnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalRoundedNarrowingSaturateEven, -1, 2, INS_invalid, INS_sve_uqrshrnb, INS_invalid, INS_sve_uqrshrnb, INS_invalid, INS_sve_uqrshrnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalRoundedNarrowingSaturateOdd, -1, 3, INS_invalid, INS_sve_uqrshrnt, INS_invalid, INS_sve_uqrshrnt, INS_invalid, INS_sve_uqrshrnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, SubtractBorrowWideningEven, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sbclb, INS_invalid, INS_sve_sbclb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, SubtractBorrowWideningOdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sbclt, INS_invalid, INS_sve_sbclt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, SubtractHighNarrowingEven, -1, 2, INS_sve_subhnb, INS_sve_subhnb, INS_sve_subhnb, INS_sve_subhnb, INS_sve_subhnb, INS_sve_subhnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, SubtractHighNarrowingOdd, -1, 3, INS_sve_subhnt, INS_sve_subhnt, INS_sve_subhnt, INS_sve_subhnt, INS_sve_subhnt, INS_sve_subhnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, SubtractRoundedHighNarrowingEven, -1, 2, INS_sve_rsubhnb, INS_sve_rsubhnb, INS_sve_rsubhnb, INS_sve_rsubhnb, INS_sve_rsubhnb, INS_sve_rsubhnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, SubtractRoundedHighNarrowingOdd, -1, 3, INS_sve_rsubhnt, INS_sve_rsubhnt, INS_sve_rsubhnt, INS_sve_rsubhnt, INS_sve_rsubhnt, INS_sve_rsubhnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, SubtractSaturate, -1, -1, INS_sve_sqsub, INS_sve_uqsub, INS_sve_sqsub, INS_sve_uqsub, INS_sve_sqsub, INS_sve_uqsub, INS_sve_sqsub, INS_sve_uqsub, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, SubtractWideningEven, -1, 2, INS_invalid, INS_invalid, INS_sve_ssubwb, INS_sve_usubwb, INS_sve_ssubwb, INS_sve_usubwb, INS_sve_ssubwb, INS_sve_usubwb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve2, SubtractWideningEvenOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_ssublbt, INS_invalid, INS_sve_ssublbt, INS_invalid, INS_sve_ssublbt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, SubtractWideningOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_ssubwt, INS_sve_usubwt, INS_sve_ssubwt, INS_sve_usubwt, INS_sve_ssubwt, INS_sve_usubwt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve2, SubtractWideningOddEven, -1, 2, INS_invalid, INS_invalid, INS_sve_ssubltb, INS_invalid, INS_sve_ssubltb, INS_invalid, INS_sve_ssubltb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, VectorTableLookup, -1, 2, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve2, VectorTableLookupExtension, -1, 3, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, Xor, -1, 3, INS_sve_eor3, INS_sve_eor3, INS_sve_eor3, INS_sve_eor3, INS_sve_eor3, INS_sve_eor3, INS_sve_eor3, INS_sve_eor3, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, XorRotateRight, -1, 3, INS_sve_xar, INS_sve_xar, INS_sve_xar, INS_sve_xar, INS_sve_xar, INS_sve_xar, INS_sve_xar, INS_sve_xar, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve2, AbsSaturate, -1, -1, INS_sve_sqabs, INS_invalid, INS_sve_sqabs, INS_invalid, INS_sve_sqabs, INS_invalid, INS_sve_sqabs, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, AbsoluteDifferenceAdd, -1, 3, INS_sve_saba, INS_sve_uaba, INS_sve_saba, INS_sve_uaba, INS_sve_saba, INS_sve_uaba, INS_sve_saba, INS_sve_uaba, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, AbsoluteDifferenceWideningEven, -1, 2, INS_invalid, INS_invalid, INS_sve_sabdlb, INS_sve_uabdlb, INS_sve_sabdlb, INS_sve_uabdlb, INS_sve_sabdlb, INS_sve_uabdlb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, AbsoluteDifferenceWideningLowerAndAddEven, -1, 3, INS_invalid, INS_invalid, INS_sve_sabalb, INS_sve_uabalb, INS_sve_sabalb, INS_sve_uabalb, INS_sve_sabalb, INS_sve_uabalb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, AbsoluteDifferenceWideningLowerAndAddOdd, -1, 3, INS_invalid, INS_invalid, INS_sve_sabalt, INS_sve_uabalt, INS_sve_sabalt, INS_sve_uabalt, INS_sve_sabalt, INS_sve_uabalt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, AbsoluteDifferenceWideningOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_sabdlt, INS_sve_uabdlt, INS_sve_sabdlt, INS_sve_uabdlt, INS_sve_sabdlt, INS_sve_uabdlt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, AddCarryWideningEven, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_adclb, INS_invalid, INS_sve_adclb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, AddCarryWideningOdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_adclt, INS_invalid, INS_sve_adclt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, AddHighNarrowingEven, -1, 2, INS_sve_addhnb, INS_sve_addhnb, INS_sve_addhnb, INS_sve_addhnb, INS_sve_addhnb, INS_sve_addhnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, AddHighNarrowingOdd, -1, 3, INS_sve_addhnt, INS_sve_addhnt, INS_sve_addhnt, INS_sve_addhnt, INS_sve_addhnt, INS_sve_addhnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, AddPairwise, -1, -1, INS_sve_addp, INS_sve_addp, INS_sve_addp, INS_sve_addp, INS_sve_addp, INS_sve_addp, INS_sve_addp, INS_sve_addp, INS_sve_faddp, INS_sve_faddp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) +HARDWARE_INTRINSIC(Sve2, AddPairwiseWideningAndAdd, -1, -1, INS_invalid, INS_invalid, INS_sve_sadalp, INS_sve_uadalp, INS_sve_sadalp, INS_sve_uadalp, INS_sve_sadalp, INS_sve_uadalp, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, AddRotateComplex, -1, 3, INS_sve_cadd, INS_sve_cadd, INS_sve_cadd, INS_sve_cadd, INS_sve_cadd, INS_sve_cadd, INS_sve_cadd, INS_sve_cadd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve2, AddRoundedHighNarrowingEven, -1, 2, INS_sve_raddhnb, INS_sve_raddhnb, INS_sve_raddhnb, INS_sve_raddhnb, INS_sve_raddhnb, INS_sve_raddhnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, AddRoundedHighNarrowingOdd, -1, 3, INS_sve_raddhnt, INS_sve_raddhnt, INS_sve_raddhnt, INS_sve_raddhnt, INS_sve_raddhnt, INS_sve_raddhnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, AddSaturate, -1, -1, INS_sve_sqadd, INS_sve_uqadd, INS_sve_sqadd, INS_sve_uqadd, INS_sve_sqadd, INS_sve_uqadd, INS_sve_sqadd, INS_sve_uqadd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve2, AddSaturateRotateComplex, -1, 3, INS_sve_sqcadd, INS_invalid, INS_sve_sqcadd, INS_invalid, INS_sve_sqcadd, INS_invalid, INS_sve_sqcadd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve2, AddWideningEven, -1, 2, INS_invalid, INS_invalid, INS_sve_saddwb, INS_sve_uaddwb, INS_sve_saddwb, INS_sve_uaddwb, INS_sve_saddwb, INS_sve_uaddwb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve2, AddWideningEvenOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_saddlbt, INS_invalid, INS_sve_saddlbt, INS_invalid, INS_sve_saddlbt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, AddWideningOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_saddwt, INS_sve_uaddwt, INS_sve_saddwt, INS_sve_uaddwt, INS_sve_saddwt, INS_sve_uaddwt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve2, BitwiseClearXor, -1, 3, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, BitwiseSelect, -1, 3, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, BitwiseSelectLeftInverted, -1, 3, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, BitwiseSelectRightInverted, -1, 3, INS_sve_bsl2n, INS_sve_bsl2n, INS_sve_bsl2n, INS_sve_bsl2n, INS_sve_bsl2n, INS_sve_bsl2n, INS_sve_bsl2n, INS_sve_bsl2n, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ConvertToDoubleOdd, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtlt, INS_sve_fcvtlt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, ConvertToSingleEvenRoundToOdd, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtx, INS_sve_fcvtx, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, ConvertToSingleOdd, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtnt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ConvertToSingleOddRoundToOdd, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtxnt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, CountMatchingElements, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_histcnt, INS_sve_histcnt, INS_sve_histcnt, INS_sve_histcnt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve2, CountMatchingElementsIn128BitSegments, -1, 2, INS_sve_histseg, INS_sve_histseg, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskByte, -1, 2, INS_invalid, INS_sve_whilegt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskDouble, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilegt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_whilegt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilegt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilegt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskSByte, -1, 2, INS_sve_whilegt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskSingle, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilegt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilegt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilegt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilegt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskByte, -1, 2, INS_invalid, INS_sve_whilege, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskDouble, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilege, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_whilege, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilege, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilege, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskSByte, -1, 2, INS_sve_whilege, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskSingle, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilege, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilege, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilege, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilege, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskByte, -1, 2, INS_invalid, INS_sve_whilerw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskDouble, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilerw, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_whilerw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilerw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilerw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskSByte, -1, 2, INS_sve_whilerw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskSingle, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilerw, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilerw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilerw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilerw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskByte, -1, 2, INS_invalid, INS_sve_whilewr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskDouble, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilewr, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_whilewr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilewr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilewr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskSByte, -1, 2, INS_sve_whilewr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskSingle, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilewr, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilewr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilewr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilewr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, DotProductRotateComplex, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_cdot, INS_invalid, INS_sve_cdot, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve2, DotProductRotateComplexBySelectedIndex, -1, 5, INS_sve_cdot, INS_invalid, INS_sve_cdot, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(Sve2, FusedAddHalving, -1, -1, INS_sve_shadd, INS_sve_uhadd, INS_sve_shadd, INS_sve_uhadd, INS_sve_shadd, INS_sve_uhadd, INS_sve_shadd, INS_sve_uhadd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, FusedAddRoundedHalving, -1, -1, INS_sve_srhadd, INS_sve_urhadd, INS_sve_srhadd, INS_sve_urhadd, INS_sve_srhadd, INS_sve_urhadd, INS_sve_srhadd, INS_sve_urhadd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, FusedSubtractHalving, -1, -1, INS_sve_shsub, INS_sve_uhsub, INS_sve_shsub, INS_sve_uhsub, INS_sve_shsub, INS_sve_uhsub, INS_sve_shsub, INS_sve_uhsub, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, GatherVectorByteZeroExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1b, INS_sve_ldnt1b, INS_sve_ldnt1b, INS_sve_ldnt1b, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve2, GatherVectorInt16SignExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1sh, INS_sve_ldnt1sh, INS_sve_ldnt1sh, INS_sve_ldnt1sh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve2, GatherVectorInt16WithByteOffsetsSignExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1sh, INS_sve_ldnt1sh, INS_sve_ldnt1sh, INS_sve_ldnt1sh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve2, GatherVectorInt32SignExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1sw, INS_sve_ldnt1sw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve2, GatherVectorInt32WithByteOffsetsSignExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1sw, INS_sve_ldnt1sw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve2, GatherVectorNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1w, INS_sve_ldnt1w, INS_sve_ldnt1d, INS_sve_ldnt1d, INS_sve_ldnt1w, INS_sve_ldnt1d, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve2, GatherVectorSByteSignExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1sb, INS_sve_ldnt1sb, INS_sve_ldnt1sb, INS_sve_ldnt1sb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve2, GatherVectorUInt16WithByteOffsetsZeroExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1h, INS_sve_ldnt1h, INS_sve_ldnt1h, INS_sve_ldnt1h, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve2, GatherVectorUInt16ZeroExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1h, INS_sve_ldnt1h, INS_sve_ldnt1h, INS_sve_ldnt1h, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve2, GatherVectorUInt32WithByteOffsetsZeroExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1w, INS_sve_ldnt1w, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve2, GatherVectorUInt32ZeroExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1w, INS_sve_ldnt1w, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve2, GatherVectorWithByteOffsetsNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1w, INS_sve_ldnt1w, INS_sve_ldnt1d, INS_sve_ldnt1d, INS_sve_ldnt1w, INS_sve_ldnt1d, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve2, InterleavingXorEvenOdd, -1, 3, INS_sve_eorbt, INS_sve_eorbt, INS_sve_eorbt, INS_sve_eorbt, INS_sve_eorbt, INS_sve_eorbt, INS_sve_eorbt, INS_sve_eorbt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, InterleavingXorOddEven, -1, 3, INS_sve_eortb, INS_sve_eortb, INS_sve_eortb, INS_sve_eortb, INS_sve_eortb, INS_sve_eortb, INS_sve_eortb, INS_sve_eortb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, Log2, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_flogb, INS_invalid, INS_sve_flogb, INS_invalid, INS_sve_flogb, INS_sve_flogb, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, Match, -1, 3, INS_sve_match, INS_sve_match, INS_sve_match, INS_sve_match, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve2, MaxNumberPairwise, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmaxnmp, INS_sve_fmaxnmp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, MaxPairwise, -1, -1, INS_sve_smaxp, INS_sve_umaxp, INS_sve_smaxp, INS_sve_umaxp, INS_sve_smaxp, INS_sve_umaxp, INS_sve_smaxp, INS_sve_umaxp, INS_sve_fmaxp, INS_sve_fmaxp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_ReduceOperation) +HARDWARE_INTRINSIC(Sve2, MinNumberPairwise, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fminnmp, INS_sve_fminnmp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, MinPairwise, -1, -1, INS_sve_sminp, INS_sve_uminp, INS_sve_sminp, INS_sve_uminp, INS_sve_sminp, INS_sve_uminp, INS_sve_sminp, INS_sve_uminp, INS_sve_fminp, INS_sve_fminp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_ReduceOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyAddBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, MultiplyAddRotateComplex, -1, 4, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve2, MultiplyAddRotateComplexBySelectedScalar, -1, 5, INS_invalid, INS_invalid, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve2, MultiplyAddRoundedDoublingSaturateHighRotateComplex, -1, 4, INS_sve_sqrdcmlah, INS_invalid, INS_sve_sqrdcmlah, INS_invalid, INS_sve_sqrdcmlah, INS_invalid, INS_sve_sqrdcmlah, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve2, MultiplyAddRoundedDoublingSaturateHighRotateComplexBySelectedScalar, -1, 5, INS_invalid, INS_invalid, INS_sve_sqrdcmlah, INS_invalid, INS_sve_sqrdcmlah, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalar, -1, 3, INS_invalid, INS_invalid, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalarWideningEven, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_smullb, INS_sve_umullb, INS_sve_smullb, INS_sve_umullb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalarWideningEvenAndAdd, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_smlalb, INS_sve_umlalb, INS_sve_smlalb, INS_sve_umlalb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalarWideningEvenAndSubtract, -1, 4, INS_invalid, INS_invalid, INS_sve_smlslb, INS_sve_umlslb, INS_sve_smlslb, INS_sve_umlslb, INS_sve_smlslb, INS_sve_umlslb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalarWideningOdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_smullt, INS_sve_umullt, INS_sve_smullt, INS_sve_umullt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalarWideningOddAndAdd, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_smlalt, INS_sve_umlalt, INS_sve_smlalt, INS_sve_umlalt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalarWideningOddAndSubtract, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_smlslt, INS_sve_umlslt, INS_sve_smlslt, INS_sve_umlslt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingBySelectedScalarSaturateHigh, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdmulh, INS_invalid, INS_sve_sqdmulh, INS_invalid, INS_sve_sqdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingSaturateHigh, -1, 2, INS_sve_sqdmulh, INS_invalid, INS_sve_sqdmulh, INS_invalid, INS_sve_sqdmulh, INS_invalid, INS_sve_sqdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningAndAddSaturateEven, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdmlalb, INS_invalid, INS_sve_sqdmlalb, INS_invalid, INS_sve_sqdmlalb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningAndAddSaturateEvenOdd, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdmlalbt, INS_invalid, INS_sve_sqdmlalbt, INS_invalid, INS_sve_sqdmlalbt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningAndAddSaturateOdd, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdmlalt, INS_invalid, INS_sve_sqdmlalt, INS_invalid, INS_sve_sqdmlalt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningAndSubtractSaturateEven, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdmlslb, INS_invalid, INS_sve_sqdmlslb, INS_invalid, INS_sve_sqdmlslb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningAndSubtractSaturateEvenOdd, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdmlslbt, INS_invalid, INS_sve_sqdmlslbt, INS_invalid, INS_sve_sqdmlslbt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningAndSubtractSaturateOdd, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdmlslt, INS_invalid, INS_sve_sqdmlslt, INS_invalid, INS_sve_sqdmlslt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningBySelectedScalarAndAddSaturateEven, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdmlalb, INS_invalid, INS_sve_sqdmlalb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningBySelectedScalarAndAddSaturateOdd, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdmlalt, INS_invalid, INS_sve_sqdmlalt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningBySelectedScalarAndSubtractSaturateEven, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdmlslb, INS_invalid, INS_sve_sqdmlslb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningBySelectedScalarAndSubtractSaturateOdd, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdmlslt, INS_invalid, INS_sve_sqdmlslt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningSaturateEven, -1, 2, INS_invalid, INS_invalid, INS_sve_sqdmullb, INS_invalid, INS_sve_sqdmullb, INS_invalid, INS_sve_sqdmullb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningSaturateEvenBySelectedScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdmullb, INS_invalid, INS_sve_sqdmullb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningSaturateOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_sqdmullt, INS_invalid, INS_sve_sqdmullt, INS_invalid, INS_sve_sqdmullt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningSaturateOddBySelectedScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdmullt, INS_invalid, INS_sve_sqdmullt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyRoundedDoublingBySelectedScalarSaturateHigh, -1, 3, INS_invalid, INS_invalid, INS_sve_sqrdmulh, INS_invalid, INS_sve_sqrdmulh, INS_invalid, INS_sve_sqrdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyRoundedDoublingSaturateAndAddHigh, -1, 3, INS_sve_sqrdmlah, INS_invalid, INS_sve_sqrdmlah, INS_invalid, INS_sve_sqrdmlah, INS_invalid, INS_sve_sqrdmlah, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, MultiplyRoundedDoublingSaturateAndSubtractHigh, -1, 3, INS_sve_sqrdmlsh, INS_invalid, INS_sve_sqrdmlsh, INS_invalid, INS_sve_sqrdmlsh, INS_invalid, INS_sve_sqrdmlsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, MultiplyRoundedDoublingSaturateBySelectedScalarAndAddHigh, -1, 4, INS_invalid, INS_invalid, INS_sve_sqrdmlah, INS_invalid, INS_sve_sqrdmlah, INS_invalid, INS_sve_sqrdmlah, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyRoundedDoublingSaturateBySelectedScalarAndSubtractHigh, -1, 4, INS_invalid, INS_invalid, INS_sve_sqrdmlsh, INS_invalid, INS_sve_sqrdmlsh, INS_invalid, INS_sve_sqrdmlsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyRoundedDoublingSaturateHigh, -1, 2, INS_sve_sqrdmulh, INS_invalid, INS_sve_sqrdmulh, INS_invalid, INS_sve_sqrdmulh, INS_invalid, INS_sve_sqrdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, MultiplySubtractBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyWideningEven, -1, 2, INS_invalid, INS_invalid, INS_sve_smullb, INS_sve_umullb, INS_sve_smullb, INS_sve_umullb, INS_sve_smullb, INS_sve_umullb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, MultiplyWideningEvenAndAdd, -1, 3, INS_invalid, INS_invalid, INS_sve_smlalb, INS_sve_umlalb, INS_sve_smlalb, INS_sve_umlalb, INS_sve_smlalb, INS_sve_umlalb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, MultiplyWideningEvenAndSubtract, -1, 3, INS_invalid, INS_invalid, INS_sve_smlslb, INS_sve_umlslb, INS_sve_smlslb, INS_sve_umlslb, INS_sve_smlslb, INS_sve_umlslb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, MultiplyWideningOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_smullt, INS_sve_umullt, INS_sve_smullt, INS_sve_umullt, INS_sve_smullt, INS_sve_umullt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, MultiplyWideningOddAndAdd, -1, 3, INS_invalid, INS_invalid, INS_sve_smlalt, INS_sve_umlalt, INS_sve_smlalt, INS_sve_umlalt, INS_sve_smlalt, INS_sve_umlalt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, MultiplyWideningOddAndSubtract, -1, 3, INS_invalid, INS_invalid, INS_sve_smlslt, INS_sve_umlslt, INS_sve_smlslt, INS_sve_umlslt, INS_sve_smlslt, INS_sve_umlslt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, NegateSaturate, -1, -1, INS_sve_sqneg, INS_invalid, INS_sve_sqneg, INS_invalid, INS_sve_sqneg, INS_invalid, INS_sve_sqneg, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, NoMatch, -1, 3, INS_sve_nmatch, INS_sve_nmatch, INS_sve_nmatch, INS_sve_nmatch, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve2, PolynomialMultiply, -1, 2, INS_sve_pmul, INS_sve_pmul, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, PolynomialMultiplyWideningEven, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_pmullb, INS_invalid, INS_invalid, INS_invalid, INS_sve_pmullb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, PolynomialMultiplyWideningOdd, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_pmullt, INS_invalid, INS_invalid, INS_invalid, INS_sve_pmullt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, ReciprocalEstimate, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_urecpe, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, ReciprocalSqrtEstimate, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ursqrte, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, SaturatingExtractNarrowingLower, -1, 1, INS_sve_sqxtnb, INS_sve_uqxtnb, INS_sve_sqxtnb, INS_sve_uqxtnb, INS_sve_sqxtnb, INS_sve_uqxtnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, SaturatingExtractNarrowingUpper, -1, 2, INS_sve_sqxtnt, INS_sve_uqxtnt, INS_sve_sqxtnt, INS_sve_uqxtnt, INS_sve_sqxtnt, INS_sve_uqxtnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, SaturatingExtractUnsignedNarrowingLower, -1, 1, INS_invalid, INS_sve_sqxtunb, INS_invalid, INS_sve_sqxtunb, INS_invalid, INS_sve_sqxtunb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, SaturatingExtractUnsignedNarrowingUpper, -1, 2, INS_invalid, INS_sve_sqxtunt, INS_invalid, INS_sve_sqxtunt, INS_invalid, INS_sve_sqxtunt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, Scatter16BitNarrowingNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1h, INS_sve_stnt1h, INS_sve_stnt1h, INS_sve_stnt1h, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, Scatter16BitWithByteOffsetsNarrowingNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1h, INS_sve_stnt1h, INS_sve_stnt1h, INS_sve_stnt1h, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, Scatter32BitNarrowingNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1w, INS_sve_stnt1w, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, Scatter32BitWithByteOffsetsNarrowingNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1w, INS_sve_stnt1w, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, Scatter8BitNarrowingNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1b, INS_sve_stnt1b, INS_sve_stnt1b, INS_sve_stnt1b, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, Scatter8BitWithByteOffsetsNarrowingNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1b, INS_sve_stnt1b, INS_sve_stnt1b, INS_sve_stnt1b, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, ScatterNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1w, INS_sve_stnt1w, INS_sve_stnt1d, INS_sve_stnt1d, INS_sve_stnt1w, INS_sve_stnt1d, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, ScatterWithByteOffsetsNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1w, INS_sve_stnt1w, INS_sve_stnt1d, INS_sve_stnt1d, INS_sve_stnt1w, INS_sve_stnt1d, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, ShiftArithmeticRounded, -1, -1, INS_sve_srshl, INS_invalid, INS_sve_srshl, INS_invalid, INS_sve_srshl, INS_invalid, INS_sve_srshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, ShiftArithmeticRoundedSaturate, -1, -1, INS_sve_sqrshl, INS_invalid, INS_sve_sqrshl, INS_invalid, INS_sve_sqrshl, INS_invalid, INS_sve_sqrshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, ShiftArithmeticSaturate, -1, -1, INS_sve_sqshl, INS_invalid, INS_sve_sqshl, INS_invalid, INS_sve_sqshl, INS_invalid, INS_sve_sqshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, ShiftLeftAndInsert, -1, 3, INS_sve_sli, INS_sve_sli, INS_sve_sli, INS_sve_sli, INS_sve_sli, INS_sve_sli, INS_sve_sli, INS_sve_sli, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftLeftLogicalSaturate, -1, -1, INS_invalid, INS_sve_uqshl, INS_invalid, INS_sve_uqshl, INS_invalid, INS_sve_uqshl, INS_invalid, INS_sve_uqshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftLeftLogicalSaturateUnsigned, -1, -1, INS_invalid, INS_sve_sqshlu, INS_invalid, INS_sve_sqshlu, INS_invalid, INS_sve_sqshlu, INS_invalid, INS_sve_sqshlu, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftLeftLogicalWideningEven, -1, 2, INS_invalid, INS_invalid, INS_sve_sshllb, INS_sve_ushllb, INS_sve_sshllb, INS_sve_ushllb, INS_sve_sshllb, INS_sve_ushllb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve2, ShiftLeftLogicalWideningOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_sshllt, INS_sve_ushllt, INS_sve_sshllt, INS_sve_ushllt, INS_sve_sshllt, INS_sve_ushllt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve2, ShiftLogicalRounded, -1, -1, INS_invalid, INS_sve_urshl, INS_invalid, INS_sve_urshl, INS_invalid, INS_sve_urshl, INS_invalid, INS_sve_urshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftLogicalRoundedSaturate, -1, -1, INS_invalid, INS_sve_uqrshl, INS_invalid, INS_sve_uqrshl, INS_invalid, INS_sve_uqrshl, INS_invalid, INS_sve_uqrshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftRightAndInsert, -1, 3, INS_sve_sri, INS_sve_sri, INS_sve_sri, INS_sve_sri, INS_sve_sri, INS_sve_sri, INS_sve_sri, INS_sve_sri, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticAdd, -1, 3, INS_sve_ssra, INS_invalid, INS_sve_ssra, INS_invalid, INS_sve_ssra, INS_invalid, INS_sve_ssra, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticNarrowingSaturateEven, -1, 2, INS_sve_sqshrnb, INS_invalid, INS_sve_sqshrnb, INS_invalid, INS_sve_sqshrnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticNarrowingSaturateOdd, -1, 3, INS_sve_sqshrnt, INS_invalid, INS_sve_sqshrnt, INS_invalid, INS_sve_sqshrnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticNarrowingSaturateUnsignedEven, -1, 2, INS_invalid, INS_sve_sqshrunb, INS_invalid, INS_sve_sqshrunb, INS_invalid, INS_sve_sqshrunb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticNarrowingSaturateUnsignedOdd, -1, 3, INS_invalid, INS_sve_sqshrunt, INS_invalid, INS_sve_sqshrunt, INS_invalid, INS_sve_sqshrunt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticRounded, -1, -1, INS_sve_srshr, INS_invalid, INS_sve_srshr, INS_invalid, INS_sve_srshr, INS_invalid, INS_sve_srshr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticRoundedAdd, -1, 3, INS_sve_srsra, INS_invalid, INS_sve_srsra, INS_invalid, INS_sve_srsra, INS_invalid, INS_sve_srsra, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticRoundedNarrowingSaturateEven, -1, 2, INS_sve_sqrshrnb, INS_invalid, INS_sve_sqrshrnb, INS_invalid, INS_sve_sqrshrnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticRoundedNarrowingSaturateOdd, -1, 3, INS_sve_sqrshrnt, INS_invalid, INS_sve_sqrshrnt, INS_invalid, INS_sve_sqrshrnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticRoundedNarrowingSaturateUnsignedEven, -1, 2, INS_invalid, INS_sve_sqrshrunb, INS_invalid, INS_sve_sqrshrunb, INS_invalid, INS_sve_sqrshrunb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticRoundedNarrowingSaturateUnsignedOdd, -1, 3, INS_invalid, INS_sve_sqrshrunt, INS_invalid, INS_sve_sqrshrunt, INS_invalid, INS_sve_sqrshrunt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalAdd, -1, 3, INS_invalid, INS_sve_usra, INS_invalid, INS_sve_usra, INS_invalid, INS_sve_usra, INS_invalid, INS_sve_usra, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalNarrowingEven, -1, 2, INS_sve_shrnb, INS_sve_shrnb, INS_sve_shrnb, INS_sve_shrnb, INS_sve_shrnb, INS_sve_shrnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalNarrowingOdd, -1, 3, INS_sve_shrnt, INS_sve_shrnt, INS_sve_shrnt, INS_sve_shrnt, INS_sve_shrnt, INS_sve_shrnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalNarrowingSaturateEven, -1, 2, INS_invalid, INS_sve_uqshrnb, INS_invalid, INS_sve_uqshrnb, INS_invalid, INS_sve_uqshrnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalNarrowingSaturateOdd, -1, 3, INS_invalid, INS_sve_uqshrnt, INS_invalid, INS_sve_uqshrnt, INS_invalid, INS_sve_uqshrnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalRounded, -1, -1, INS_invalid, INS_sve_urshr, INS_invalid, INS_sve_urshr, INS_invalid, INS_sve_urshr, INS_invalid, INS_sve_urshr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalRoundedAdd, -1, 3, INS_invalid, INS_sve_ursra, INS_invalid, INS_sve_ursra, INS_invalid, INS_sve_ursra, INS_invalid, INS_sve_ursra, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalRoundedNarrowingEven, -1, 2, INS_sve_rshrnb, INS_sve_rshrnb, INS_sve_rshrnb, INS_sve_rshrnb, INS_sve_rshrnb, INS_sve_rshrnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalRoundedNarrowingOdd, -1, 3, INS_sve_rshrnt, INS_sve_rshrnt, INS_sve_rshrnt, INS_sve_rshrnt, INS_sve_rshrnt, INS_sve_rshrnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalRoundedNarrowingSaturateEven, -1, 2, INS_invalid, INS_sve_uqrshrnb, INS_invalid, INS_sve_uqrshrnb, INS_invalid, INS_sve_uqrshrnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalRoundedNarrowingSaturateOdd, -1, 3, INS_invalid, INS_sve_uqrshrnt, INS_invalid, INS_sve_uqrshrnt, INS_invalid, INS_sve_uqrshrnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, SubtractBorrowWideningEven, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sbclb, INS_invalid, INS_sve_sbclb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, SubtractBorrowWideningOdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sbclt, INS_invalid, INS_sve_sbclt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, SubtractHighNarrowingEven, -1, 2, INS_sve_subhnb, INS_sve_subhnb, INS_sve_subhnb, INS_sve_subhnb, INS_sve_subhnb, INS_sve_subhnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, SubtractHighNarrowingOdd, -1, 3, INS_sve_subhnt, INS_sve_subhnt, INS_sve_subhnt, INS_sve_subhnt, INS_sve_subhnt, INS_sve_subhnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, SubtractRoundedHighNarrowingEven, -1, 2, INS_sve_rsubhnb, INS_sve_rsubhnb, INS_sve_rsubhnb, INS_sve_rsubhnb, INS_sve_rsubhnb, INS_sve_rsubhnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, SubtractRoundedHighNarrowingOdd, -1, 3, INS_sve_rsubhnt, INS_sve_rsubhnt, INS_sve_rsubhnt, INS_sve_rsubhnt, INS_sve_rsubhnt, INS_sve_rsubhnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, SubtractSaturate, -1, -1, INS_sve_sqsub, INS_sve_uqsub, INS_sve_sqsub, INS_sve_uqsub, INS_sve_sqsub, INS_sve_uqsub, INS_sve_sqsub, INS_sve_uqsub, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, SubtractWideningEven, -1, 2, INS_invalid, INS_invalid, INS_sve_ssubwb, INS_sve_usubwb, INS_sve_ssubwb, INS_sve_usubwb, INS_sve_ssubwb, INS_sve_usubwb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve2, SubtractWideningEvenOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_ssublbt, INS_invalid, INS_sve_ssublbt, INS_invalid, INS_sve_ssublbt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, SubtractWideningOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_ssubwt, INS_sve_usubwt, INS_sve_ssubwt, INS_sve_usubwt, INS_sve_ssubwt, INS_sve_usubwt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve2, SubtractWideningOddEven, -1, 2, INS_invalid, INS_invalid, INS_sve_ssubltb, INS_invalid, INS_sve_ssubltb, INS_invalid, INS_sve_ssubltb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, VectorTableLookup, -1, 2, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve2, VectorTableLookupExtension, -1, 3, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, Xor, -1, 3, INS_sve_eor3, INS_sve_eor3, INS_sve_eor3, INS_sve_eor3, INS_sve_eor3, INS_sve_eor3, INS_sve_eor3, INS_sve_eor3, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, XorRotateRight, -1, 3, INS_sve_xar, INS_sve_xar, INS_sve_xar, INS_sve_xar, INS_sve_xar, INS_sve_xar, INS_sve_xar, INS_sve_xar, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand) #define LAST_NI_Sve2 NI_Sve2_XorRotateRight // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -539,7 +539,7 @@ HARDWARE_INTRINSIC(Sve2, XorRotateRight, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // SVE SHA3 Intrinsics #define FIRST_NI_SveSha3 NI_SveSha3_BitwiseRotateLeftBy1AndXor -HARDWARE_INTRINSIC(SveSha3, BitwiseRotateLeftBy1AndXor, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rax1, INS_sve_rax1, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(SveSha3, BitwiseRotateLeftBy1AndXor, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rax1, INS_sve_rax1, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen) #define LAST_NI_SveSha3 NI_SveSha3_BitwiseRotateLeftBy1AndXor // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -548,35 +548,35 @@ HARDWARE_INTRINSIC(SveSha3, BitwiseRotateLeftBy1AndXor, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Special intrinsics that are generated during importing or lowering #define SPECIAL_NI_Sve NI_Sve_ConditionalExtractAfterLastActiveElementScalar -HARDWARE_INTRINSIC(Sve, ConditionalExtractAfterLastActiveElementScalar, 0, 3, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, ConditionalExtractLastActiveElementScalar, 0, 3, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, ConvertMaskToVector, -1, 1, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, -1, -1, HW_Category_Helper, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve, ConvertVectorToMask, -1, 2, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, -1, -1, HW_Category_Helper, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ConditionalExtractAfterLastActiveElementScalar, 0, 3, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ConditionalExtractLastActiveElementScalar, 0, 3, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ConvertMaskToVector, -1, 1, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve, ConvertVectorToMask, -1, 2, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_LowMaskedOperation) // True mask only used inside a ConvertVectorToMask -HARDWARE_INTRINSIC(Sve, ConversionTrueMask, -1, 0, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, -1, -1, HW_Category_Helper, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, ConversionTrueMask, -1, 0, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) // Scalar variants of Saturating*By*BitElementCount. There is 8bit versions as the generic version is scalar only. -HARDWARE_INTRINSIC(Sve, SaturatingDecrementBy16BitElementCountScalar, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdech, INS_sve_uqdech, INS_sve_sqdech, INS_sve_uqdech, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SaturatingDecrementBy32BitElementCountScalar, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdecw, INS_sve_uqdecw, INS_sve_sqdecw, INS_sve_uqdecw, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SaturatingDecrementBy64BitElementCountScalar, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdecd, INS_sve_uqdecd, INS_sve_sqdecd, INS_sve_uqdecd, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy16BitElementCountScalar, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqinch, INS_sve_uqinch, INS_sve_sqinch, INS_sve_uqinch, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy32BitElementCountScalar, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqincw, INS_sve_uqincw, INS_sve_sqincw, INS_sve_uqincw, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy64BitElementCountScalar, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqincd, INS_sve_uqincd, INS_sve_sqincd, INS_sve_uqincd, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, StoreAndZipx2, -1, 3, INS_sve_st2b, INS_sve_st2b, INS_sve_st2h, INS_sve_st2h, INS_sve_st2w, INS_sve_st2w, INS_sve_st2d, INS_sve_st2d, INS_sve_st2w, INS_sve_st2d, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(Sve, StoreAndZipx3, -1, 3, INS_sve_st3b, INS_sve_st3b, INS_sve_st3h, INS_sve_st3h, INS_sve_st3w, INS_sve_st3w, INS_sve_st3d, INS_sve_st3d, INS_sve_st3w, INS_sve_st3d, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(Sve, StoreAndZipx4, -1, 3, INS_sve_st4b, INS_sve_st4b, INS_sve_st4h, INS_sve_st4h, INS_sve_st4w, INS_sve_st4w, INS_sve_st4d, INS_sve_st4d, INS_sve_st4w, INS_sve_st4d, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(Sve, SaturatingDecrementBy16BitElementCountScalar, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdech, INS_sve_uqdech, INS_sve_sqdech, INS_sve_uqdech, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, SaturatingDecrementBy32BitElementCountScalar, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdecw, INS_sve_uqdecw, INS_sve_sqdecw, INS_sve_uqdecw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, SaturatingDecrementBy64BitElementCountScalar, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdecd, INS_sve_uqdecd, INS_sve_sqdecd, INS_sve_uqdecd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy16BitElementCountScalar, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqinch, INS_sve_uqinch, INS_sve_sqinch, INS_sve_uqinch, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy32BitElementCountScalar, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqincw, INS_sve_uqincw, INS_sve_sqincw, INS_sve_uqincw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy64BitElementCountScalar, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqincd, INS_sve_uqincd, INS_sve_sqincd, INS_sve_uqincd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, StoreAndZipx2, -1, 3, INS_sve_st2b, INS_sve_st2b, INS_sve_st2h, INS_sve_st2h, INS_sve_st2w, INS_sve_st2w, INS_sve_st2d, INS_sve_st2d, INS_sve_st2w, INS_sve_st2d, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(Sve, StoreAndZipx3, -1, 3, INS_sve_st3b, INS_sve_st3b, INS_sve_st3h, INS_sve_st3h, INS_sve_st3w, INS_sve_st3w, INS_sve_st3d, INS_sve_st3d, INS_sve_st3w, INS_sve_st3d, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(Sve, StoreAndZipx4, -1, 3, INS_sve_st4b, INS_sve_st4b, INS_sve_st4h, INS_sve_st4h, INS_sve_st4w, INS_sve_st4w, INS_sve_st4d, INS_sve_st4d, INS_sve_st4w, INS_sve_st4d, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters) // Predicate variants of intrinsics, these are specialized for operating on TYP_MASK type values. -HARDWARE_INTRINSIC(Sve, And_Predicates, -1, 2, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_EmbeddedMaskedOperation|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, BitwiseClear_Predicates, -1, 2, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_EmbeddedMaskedOperation|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, Or_Predicates, -1, 2, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_EmbeddedMaskedOperation|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, Xor_Predicates, -1, 2, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_EmbeddedMaskedOperation|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, ConditionalSelect_Predicates, -1, 3, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_ExplicitMaskedOperation|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, ZipHigh_Predicates, -1, 2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, ZipLow_Predicates, -1, 2, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, UnzipEven_Predicates, -1, 2, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, UnzipOdd_Predicates, -1, 2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, TransposeEven_Predicates, -1, 2, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, TransposeOdd_Predicates, -1, 2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, ReverseElement_Predicates, -1, 1, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, And_Predicates, -1, 2, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_EmbeddedMaskedOperation|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, BitwiseClear_Predicates, -1, 2, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_EmbeddedMaskedOperation|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, Or_Predicates, -1, 2, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_EmbeddedMaskedOperation|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, Xor_Predicates, -1, 2, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_EmbeddedMaskedOperation|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, ConditionalSelect_Predicates, -1, 3, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_ExplicitMaskedOperation|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, ZipHigh_Predicates, -1, 2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, ZipLow_Predicates, -1, 2, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, UnzipEven_Predicates, -1, 2, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, UnzipOdd_Predicates, -1, 2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, TransposeEven_Predicates, -1, 2, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, TransposeOdd_Predicates, -1, 2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, ReverseElement_Predicates, -1, 1, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) #endif // FEATURE_HW_INTRINSIC From 2c036061c5a230c09765cd2fa00fdbc928d444d1 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Fri, 10 Jul 2026 00:07:48 -0700 Subject: [PATCH 07/22] Handle AVX10v1 FP16 scalar intrinsics in HWIntrinsic costing and embedded rounding The DivideScalar/SqrtScalar entries route through the special HWIntrinsic costing switch (FltCost == -1), and AddScalar/FusedMultiplyAddScalar carry HW_Flag_EmbRoundingCompatible, so all four must be handled explicitly to avoid hitting the asserting default paths when the FP16 scalar path is used. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/gentree.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 9ad468aa31c7b2..84577e56d8d968 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -4463,6 +4463,7 @@ unsigned Compiler::gtSetMultiOpOrder(GenTreeMultiOp* multiOp) case NI_AVX_Divide: case NI_AVX512_Divide: case NI_AVX512_DivideScalar: + case NI_AVX10v1_DivideScalar: { costEx = (simdBaseType == TYP_DOUBLE) ? 14 : 11; break; @@ -4513,6 +4514,7 @@ unsigned Compiler::gtSetMultiOpOrder(GenTreeMultiOp* multiOp) case NI_AVX_Sqrt: case NI_AVX512_Sqrt: case NI_AVX512_SqrtScalar: + case NI_AVX10v1_SqrtScalar: { costEx = (simdBaseType == TYP_DOUBLE) ? 16 : 12; break; @@ -30705,6 +30707,7 @@ bool GenTreeHWIntrinsic::OperIsEmbRoundingEnabled() const case NI_AVX512_FusedMultiplySubtractNegated: case NI_AVX512_FusedMultiplySubtractNegatedScalar: case NI_AVX512_FusedMultiplySubtractScalar: + case NI_AVX10v1_FusedMultiplyAddScalar: { return numArgs == 4; } @@ -30722,6 +30725,10 @@ bool GenTreeHWIntrinsic::OperIsEmbRoundingEnabled() const case NI_AVX512_X64_ConvertScalarToVector128Double: case NI_AVX512_X64_ConvertScalarToVector128Single: #endif // TARGET_AMD64 + case NI_AVX10v1_AddScalar: + case NI_AVX10v1_DivideScalar: + case NI_AVX10v1_MultiplyScalar: + case NI_AVX10v1_SubtractScalar: { return numArgs == 3; } From d2caf2f7872a2f53349f9c5c2baa57bb37b9171c Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Fri, 10 Jul 2026 00:07:56 -0700 Subject: [PATCH 08/22] Fix doubled-v display name for the FP16 scalar fmadd instructions vfmadd132sh/213sh/231sh were spelled with a leading `vv` in their display string, so JitDisasm printed e.g. `vvfmadd213sh`. These scalar forms are the first FP16 instructions actually emitted by the Half acceleration path, which surfaces the typo. Match the vfmadd*ss naming. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/instrsxarch.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/instrsxarch.h b/src/coreclr/jit/instrsxarch.h index 37524439b16311..2c4375f6d6a0c8 100644 --- a/src/coreclr/jit/instrsxarch.h +++ b/src/coreclr/jit/instrsxarch.h @@ -1041,9 +1041,9 @@ INST3(vfcmulcsh, "vfcmulcsh", IUM_WR, BAD_CODE, BAD_ INST3(vfmadd132ph, "vvfmadd132ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x98), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Multiply-Add of Packed FP16 Values INST3(vfmadd213ph, "vvfmadd213ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xA8), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Multiply-Add of Packed FP16 Values INST3(vfmadd231ph, "vvfmadd231ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xB8), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Multiply-Add of Packed FP16 Values -INST3(vfmadd132sh, "vvfmadd132sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x99), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Multiply-Add of Scalar FP16 Values -INST3(vfmadd213sh, "vvfmadd213sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xA9), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstDstSrcAVXInstruction) // Fused Multiply-Add of Scalar FP16 Values -INST3(vfmadd231sh, "vvfmadd231sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xB9), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Multiply-Add of Scalar FP16 Values +INST3(vfmadd132sh, "vfmadd132sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x99), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Multiply-Add of Scalar FP16 Values +INST3(vfmadd213sh, "vfmadd213sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xA9), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstDstSrcAVXInstruction) // Fused Multiply-Add of Scalar FP16 Values +INST3(vfmadd231sh, "vfmadd231sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xB9), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Multiply-Add of Scalar FP16 Values INST3(vfmaddcph, "vfmaddcph", IUM_RW, BAD_CODE, BAD_CODE, SSEFLTMAP(0x06, 0x56), 8C, 1C, INS_TT_FULL, Input_32Bit | KMask_Base4 | REX_W0 | Encoding_EVEX) // Complex Multiply and Accumulate Packed FP16 Values INST3(vfmaddcsh, "vfmaddcsh", IUM_RW, BAD_CODE, BAD_CODE, SSEFLTMAP(0x06, 0x57), 8C, 1C, INS_TT_TUPLE1_SCALAR, Input_32Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Complex Multiply and Accumulate Scalar FP16 Values INST3(vfmaddsub132ph, "vvfmaddsub132ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x96), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Multiply-Alternating Add/Subtract of Packed FP16 Values From 1a4337cee3fde3e662fa647e380996adf4266f16 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Fri, 10 Jul 2026 00:10:28 -0700 Subject: [PATCH 09/22] Fix swapped Ceiling/Floor rounding mode for Half RoundScaleScalar lookupHalfRoundingMode mapped Half.Ceiling to imm 1 and Half.Floor to imm 2, but the RNDSCALE rounding-control encoding is ToNegativeInfinity==1 (floor) and ToPositiveInfinity==2 (ceiling), so the two were swapped. Use the canonical FloatRoundingMode enum to make the mapping self-documenting and correct. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/importercalls.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index b79200cd99999d..34cb44de396de8 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -12522,20 +12522,20 @@ NamedIntrinsic Compiler::lookupHalfIntrinsic(NamedIntrinsic ni) // ni -- the System.Half named intrinsic // // Return Value: -// The rounding mode immediate (0=nearest, 1=+inf, 2=-inf, 3=zero). +// The rounding mode immediate (0=nearest, 1=-inf, 2=+inf, 3=zero). // int Compiler::lookupHalfRoundingMode(NamedIntrinsic ni) { switch (ni) { case NI_System_Half_Round: - return 0; // Round to nearest + return static_cast(FloatRoundingMode::ToNearestInteger); case NI_System_Half_Ceiling: - return 1; // Round towards +infinity + return static_cast(FloatRoundingMode::ToPositiveInfinity); case NI_System_Half_Floor: - return 2; // Round towards -infinity + return static_cast(FloatRoundingMode::ToNegativeInfinity); case NI_System_Half_Truncate: - return 3; // Round towards zero + return static_cast(FloatRoundingMode::ToZero); default: noway_assert(!"Should have one of the above Half intrinsics"); return -1; From 914b9b2031308b7068f17c60b137652a32ff48e4 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Fri, 10 Jul 2026 00:17:49 -0700 Subject: [PATCH 10/22] Don't accelerate Half.Min/Max with raw vminsh/vmaxsh The raw vminsh/vmaxsh instructions return src2 on unordered inputs and don't order +0/-0, which matches MaxNative/MinNative but not the IEEE maximum/minimum semantics of Half.Max/Min. Drop them from the FP16 fast path; the managed body `(Half)float.Max((float)x, (float)y)` remains correct and still F16C-accelerates the conversions while intrinsifying float.Max/Min via gtNewSimdMinMaxNode with the proper NaN/+-0 fixup. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/hwintrinsiclistxarch.h | 2 -- src/coreclr/jit/importercalls.cpp | 14 -------------- src/coreclr/jit/namedintrinsiclist.h | 2 -- .../System.Private.CoreLib/src/System/Half.cs | 2 -- 4 files changed, 20 deletions(-) diff --git a/src/coreclr/jit/hwintrinsiclistxarch.h b/src/coreclr/jit/hwintrinsiclistxarch.h index b851830b4566f2..cb755c6af754a2 100644 --- a/src/coreclr/jit/hwintrinsiclistxarch.h +++ b/src/coreclr/jit/hwintrinsiclistxarch.h @@ -684,8 +684,6 @@ HARDWARE_INTRINSIC(AVX10v1, ConvertToUInt32, HARDWARE_INTRINSIC(AVX10v1, ConvertToUInt64, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtsh2usi64, -1, 7, HW_Category_SIMDScalar, HW_Flag_EmbRoundingCompatible) HARDWARE_INTRINSIC(AVX10v1, DivideScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vdivsh, -1, -1, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) HARDWARE_INTRINSIC(AVX10v1, FusedMultiplyAddScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmadd213sh, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible|HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX10v1, MaxScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vmaxsh, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX10v1, MinScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vminsh, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) HARDWARE_INTRINSIC(AVX10v1, MultiplyScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vmulsh, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) HARDWARE_INTRINSIC(AVX10v1, ReciprocalScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrcpsh, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) HARDWARE_INTRINSIC(AVX10v1, ReciprocalSqrtScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrsqrtsh, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 34cb44de396de8..e957cd2bcd6fd4 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -4582,8 +4582,6 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, case NI_System_Half_op_Subtraction: case NI_System_Half_op_Multiply: case NI_System_Half_op_Division: - case NI_System_Half_Max: - case NI_System_Half_Min: { #if defined(TARGET_XARCH) if (compOpportunisticallyDependsOn(InstructionSet_AVX10v1)) @@ -12411,14 +12409,6 @@ NamedIntrinsic Compiler::lookupHalfNamedIntrinsic(CORINFO_METHOD_HANDLE method, { result = NI_System_Half_op_Explicit; } - else if (strcmp(methodName, "Max") == 0) - { - result = NI_System_Half_Max; - } - else if (strcmp(methodName, "Min") == 0) - { - result = NI_System_Half_Min; - } else if (strcmp(methodName, "Sqrt") == 0) { result = NI_System_Half_Sqrt; @@ -12480,10 +12470,6 @@ NamedIntrinsic Compiler::lookupHalfIntrinsic(NamedIntrinsic ni) return NI_AVX10v1_MultiplyScalar; case NI_System_Half_op_Division: return NI_AVX10v1_DivideScalar; - case NI_System_Half_Max: - return NI_AVX10v1_MaxScalar; - case NI_System_Half_Min: - return NI_AVX10v1_MinScalar; case NI_System_Half_Sqrt: return NI_AVX10v1_SqrtScalar; case NI_System_Half_ReciprocalEstimate: diff --git a/src/coreclr/jit/namedintrinsiclist.h b/src/coreclr/jit/namedintrinsiclist.h index 47b5b66590f2df..49b78e7d26c12a 100644 --- a/src/coreclr/jit/namedintrinsiclist.h +++ b/src/coreclr/jit/namedintrinsiclist.h @@ -26,8 +26,6 @@ enum NamedIntrinsic : unsigned short NI_System_SpanHelpers_Memmove, NI_System_Half_FusedMultiplyAdd, - NI_System_Half_Max, - NI_System_Half_Min, NI_System_Half_ReciprocalEstimate, NI_System_Half_ReciprocalSqrtEstimate, NI_System_Half_Round, diff --git a/src/libraries/System.Private.CoreLib/src/System/Half.cs b/src/libraries/System.Private.CoreLib/src/System/Half.cs index 7dd41648016599..a1777ad249cd57 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Half.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Half.cs @@ -1680,7 +1680,6 @@ public static Half CopySign(Half value, Half sign) } /// - [Intrinsic] public static Half Max(Half x, Half y) => (Half)float.Max((float)x, (float)y); /// @@ -1709,7 +1708,6 @@ public static Half MaxNumber(Half x, Half y) } /// - [Intrinsic] public static Half Min(Half x, Half y) => (Half)float.Min((float)x, (float)y); /// From cb53880da38b6c482950a5c626e3ee68872da039 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Fri, 10 Jul 2026 05:05:18 -0700 Subject: [PATCH 11/22] Don't redundantly check AVX10v2 alongside AVX10v1 in emitxarch asserts InstructionSet_AVX10v1 already implies InstructionSet_AVX10v2 in the JIT ISA model, so compIsaSupportedDebugOnly(InstructionSet_AVX10v1) covers the AVX10v2 case. Drop the redundant AVX10v2 checks from the EVEX map asserts. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/emitxarch.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index e6b11ff04a6458..3d9c85f6086ab7 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -3187,7 +3187,6 @@ emitter::code_t emitter::emitExtractEvexPrefix(instruction ins, code_t& code) co // 2. A map number from 0 to 7 (For AVX10.2 and above) leadingBytes = check; assert((leadingBytes == 0x0F) || ((m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX10v1) || - m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX10v2) || (m_compiler->compIsaSupportedDebugOnly(InstructionSet_APX))) && (leadingBytes >= 0x00) && (leadingBytes <= 0x07))); @@ -3215,7 +3214,6 @@ emitter::code_t emitter::emitExtractEvexPrefix(instruction ins, code_t& code) co leadingBytes = (code >> 16) & 0xFF; assert(leadingBytes == 0x0F || ((m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX10v1) || - m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX10v2) || m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX512BMM)) && leadingBytes >= 0x00 && leadingBytes <= 0x07) || (IsApxExtendedEvexInstruction(ins) && leadingBytes == 0)); @@ -3271,8 +3269,7 @@ emitter::code_t emitter::emitExtractEvexPrefix(instruction ins, code_t& code) co case 0x05: { - assert(m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX10v1) || - m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX10v2)); + assert(m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX10v1)); evexPrefix |= (0x05 << 16); break; } From bd286674026a18d3db9207054d690446950d59a5 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Fri, 10 Jul 2026 07:20:11 -0700 Subject: [PATCH 12/22] Rebind Half to ushort internally instead of adding TYP_HALF Follows the non-ABI-breaking premise from the F16C work (#127094): Half is never a GenTree value type, so no TYP_HALF exists and the ABI is unchanged (this is what caused the revert of #122649). Half is bound internally as a TYP_USHORT SIMD base type on TYP_SIMD16 nodes purely for instruction selection, with the ushort bits moved into and out of SIMD scalars at the import sites via impSimdCreateScalarHalf/impSimdToScalarHalf. Removes TYP_HALF and its per-platform HARDWARE_INTRINSIC instruction column, restoring the arm64/arm64sve/wasm/xarch tables to their original layout. Adds the FP16 conversions to/from float, double, int, uint, long, and ulong plus op_Increment/op_Decrement. Because Half binds as an integer-register base type, the affected AVX10v1 scalar entries carry an IntCost matching FltCost so the HWIntrinsic cost model no longer hits the unhandled-costing assert, and the general-purpose integer conversions produce their result with the actual machine type (TYP_INT/TYP_LONG) so the signedness is encoded solely by the selected vcvttsh2si/vcvttsh2usi instruction. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/emit.h | 14 +- src/coreclr/jit/emitxarch.cpp | 4 + src/coreclr/jit/gentree.cpp | 3 + src/coreclr/jit/hwintrinsic.cpp | 6 +- src/coreclr/jit/hwintrinsic.h | 2 +- src/coreclr/jit/hwintrinsiclist.h | 348 ++--- src/coreclr/jit/hwintrinsiclistarm64.h | 994 ++++++------ src/coreclr/jit/hwintrinsiclistarm64sve.h | 1068 ++++++------- src/coreclr/jit/hwintrinsiclistwasm.h | 128 +- src/coreclr/jit/hwintrinsiclistxarch.h | 1374 ++++++++--------- src/coreclr/jit/importercalls.cpp | 178 ++- src/coreclr/jit/namedintrinsiclist.h | 2 +- src/coreclr/jit/registeropswasm.cpp | 2 - src/coreclr/jit/typelist.h | 2 - src/coreclr/jit/valuenumfuncs.h | 2 +- .../System.Private.CoreLib/src/System/Half.cs | 12 + 16 files changed, 2131 insertions(+), 2008 deletions(-) diff --git a/src/coreclr/jit/emit.h b/src/coreclr/jit/emit.h index b4d349853bef20..d7b306be853966 100644 --- a/src/coreclr/jit/emit.h +++ b/src/coreclr/jit/emit.h @@ -2069,13 +2069,13 @@ class emitter #define PERFSCORE_THROUGHPUT_ZERO 0.0f // Only used for pseudo-instructions that don't generate code -#define PERFSCORE_THROUGHPUT_9X (1.0f / 9.0f) -#define PERFSCORE_THROUGHPUT_6X (1.0f / 6.0f) // Hextuple issue -#define PERFSCORE_THROUGHPUT_5X 0.20f // Pentuple issue -#define PERFSCORE_THROUGHPUT_4X 0.25f // Quad issue -#define PERFSCORE_THROUGHPUT_3X (1.0f / 3.0f) // Three issue -#define PERFSCORE_THROUGHPUT_2X 0.5f // Dual issue -#define PERFSCORE_THROUGHPUT_1P5X 0.67f // Dual issue +#define PERFSCORE_THROUGHPUT_9X (1.0f / 9.0f) +#define PERFSCORE_THROUGHPUT_6X (1.0f / 6.0f) // Hextuple issue +#define PERFSCORE_THROUGHPUT_5X 0.20f // Pentuple issue +#define PERFSCORE_THROUGHPUT_4X 0.25f // Quad issue +#define PERFSCORE_THROUGHPUT_3X (1.0f / 3.0f) // Three issue +#define PERFSCORE_THROUGHPUT_2X 0.5f // Dual issue +#define PERFSCORE_THROUGHPUT_1P5X 0.67f // Dual issue #define PERFSCORE_THROUGHPUT_1C 1.0f // Single Issue diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index 3d9c85f6086ab7..24414afa3c02a9 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -20838,6 +20838,10 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins case INS_vcvtusi2sh64: case INS_vcvtsh2usi32: case INS_vcvtsh2usi64: + case INS_vcvttsh2si32: + case INS_vcvttsh2si64: + case INS_vcvttsh2usi32: + case INS_vcvttsh2usi64: { insLatency = PERFSCORE_LATENCY_7C; insThroughput = PERFSCORE_THROUGHPUT_1C; diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 84577e56d8d968..b0c934b3d833cd 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -30729,6 +30729,9 @@ bool GenTreeHWIntrinsic::OperIsEmbRoundingEnabled() const case NI_AVX10v1_DivideScalar: case NI_AVX10v1_MultiplyScalar: case NI_AVX10v1_SubtractScalar: + case NI_AVX10v1_ConvertScalarToVector128Half: + case NI_AVX10v1_ConvertScalarToVector128Single: + case NI_AVX10v1_ConvertScalarToVector128Double: { return numArgs == 3; } diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index eb39ffe945ba48..26288d849e2a88 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -8,12 +8,12 @@ static const HWIntrinsicInfo hwIntrinsicInfoArray[] = { // clang-format off -#define HARDWARE_INTRINSIC(isa, name, simdSize, numArgs, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, intCost, fltCost, category, flag) \ +#define HARDWARE_INTRINSIC(isa, name, simdSize, numArgs, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, intCost, fltCost, category, flag) \ { \ /* name */ #name, \ /* flags */ static_cast(flag), \ /* id */ NI_##isa##_##name, \ - /* ins */ {t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11}, \ + /* ins */ {t1, t2, t3, t4, t5, t6, t7, t8, t9, t10}, \ /* isa */ InstructionSet_##isa, \ /* simdSize */ simdSize, \ /* numArgs */ numArgs, \ @@ -55,7 +55,7 @@ const HWIntrinsicInfo& HWIntrinsicInfo::lookup(NamedIntrinsic id) // The instruction for id and type instruction HWIntrinsicInfo::lookupIns(NamedIntrinsic id, var_types type, Compiler* comp) { - if ((type < TYP_BYTE) || (type > TYP_HALF)) + if ((type < TYP_BYTE) || (type > TYP_DOUBLE)) { assert(!"Unexpected type"); return INS_invalid; diff --git a/src/coreclr/jit/hwintrinsic.h b/src/coreclr/jit/hwintrinsic.h index 6119138f9fa743..7effb01b75f84b 100644 --- a/src/coreclr/jit/hwintrinsic.h +++ b/src/coreclr/jit/hwintrinsic.h @@ -549,7 +549,7 @@ struct HWIntrinsicInfo const char* name; // 4 or 8-bytes HWIntrinsicFlag flags; // 8-bytes NamedIntrinsic id; // 2-bytes - uint16_t ins[11]; // 11 * 2-bytes + uint16_t ins[10]; // 10 * 2-bytes uint8_t isa; // 1-byte int8_t simdSize; // 1-byte int8_t numArgs; // 1-byte diff --git a/src/coreclr/jit/hwintrinsiclist.h b/src/coreclr/jit/hwintrinsiclist.h index 7148426957c853..e9ffeba17c0411 100644 --- a/src/coreclr/jit/hwintrinsiclist.h +++ b/src/coreclr/jit/hwintrinsiclist.h @@ -33,232 +33,232 @@ // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Intrinsics for Vector64/128/256/512 and Vector #define FIRST_NI_Vector NI_Vector_Abs -HARDWARE_INTRINSIC(Vector, Abs, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AddSaturate, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AndNot, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, As, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsByte, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsDouble, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsInt16, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsInt64, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsNInt, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsNUInt, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsSByte, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsSingle, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsUInt16, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsUInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsUInt64, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsVector, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsVector128, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, Abs, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AddSaturate, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AndNot, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, As, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsByte, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsDouble, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsInt16, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsInt64, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsNInt, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsNUInt, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsSByte, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsSingle, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsUInt16, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsUInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsUInt64, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsVector, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsVector128, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) #if defined(TARGET_XARCH) -HARDWARE_INTRINSIC(Vector, AsVector128Unsafe, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movups, INS_invalid, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics|HW_Flag_NoContainment|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsVector2, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movsd_simd, INS_invalid, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsVector256, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, AsVector3, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movups, INS_invalid, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsVector128Unsafe, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movups, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics|HW_Flag_NoContainment|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsVector2, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movsd_simd, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsVector256, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsVector3, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movups, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_AvxOnlyCompatible) #elif defined(TARGET_ARM64) -HARDWARE_INTRINSIC(Vector, AsVector128Unsafe, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mov, INS_invalid, INS_invalid, -1, 1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Vector, AsVector2, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, 1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, AsVector3, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mov, INS_invalid, INS_invalid, -1, 1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Vector, AsVector128Unsafe, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mov, INS_invalid, -1, 1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Vector, AsVector2, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, 1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, AsVector3, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mov, INS_invalid, -1, 1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) #else -HARDWARE_INTRINSIC(Vector, AsVector128Unsafe, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, 1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, AsVector2, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, 1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, AsVector3, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, 1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, AsVector128Unsafe, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, 1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, AsVector2, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, 1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, AsVector3, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, 1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) #endif -HARDWARE_INTRINSIC(Vector, AsVector4, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, AsVector4, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) #if defined(TARGET_XARCH) -HARDWARE_INTRINSIC(Vector, AsVector512, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, AsVector512, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) #endif -HARDWARE_INTRINSIC(Vector, Ceiling, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, ConcatLowerLower, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, ConcatLowerUpper, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, ConcatUpperLower, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, ConcatUpperUpper, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, Ceiling, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, ConcatLowerLower, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, ConcatLowerUpper, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, ConcatUpperLower, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, ConcatUpperUpper, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) #if defined(TARGET_XARCH) -HARDWARE_INTRINSIC(Vector, ConditionalSelect, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, ConditionalSelect, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) #else -HARDWARE_INTRINSIC(Vector, ConditionalSelect, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, ConditionalSelect, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) #endif -HARDWARE_INTRINSIC(Vector, ConvertToDouble, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, ConvertToInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, ConvertToInt32Native, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, ConvertToInt64, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, ConvertToInt64Native, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, ConvertToSingle, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, ConvertToUInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, ConvertToUInt32Native, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, ConvertToUInt64, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, ConvertToUInt64Native, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, Create, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, CreateAlternatingSequence, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, CreateGeometricSequence, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, ConvertToDouble, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, ConvertToInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, ConvertToInt32Native, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, ConvertToInt64, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, ConvertToInt64Native, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, ConvertToSingle, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, ConvertToUInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, ConvertToUInt32Native, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, ConvertToUInt64, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, ConvertToUInt64Native, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, Create, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, CreateAlternatingSequence, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, CreateGeometricSequence, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) #if defined(TARGET_XARCH) -HARDWARE_INTRINSIC(Vector, CreateScalar, -1, 1, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd64, INS_movd64, INS_movss, INS_movsd_simd, INS_invalid, -1, -1, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, CreateScalarUnsafe, -1, 1, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd64, INS_movd64, INS_movss, INS_movsd_simd, INS_invalid, -1, -1, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, CreateScalar, -1, 1, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd64, INS_movd64, INS_movss, INS_movsd_simd, -1, -1, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, CreateScalarUnsafe, -1, 1, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd64, INS_movd64, INS_movss, INS_movsd_simd, -1, -1, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics|HW_Flag_AvxOnlyCompatible) #elif defined(TARGET_ARM64) -HARDWARE_INTRINSIC(Vector, CreateScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) -HARDWARE_INTRINSIC(Vector, CreateScalarUnsafe, -1, 1, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_fmov, INS_fmov, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment) +HARDWARE_INTRINSIC(Vector, CreateScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) +HARDWARE_INTRINSIC(Vector, CreateScalarUnsafe, -1, 1, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_fmov, INS_fmov, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment) #else -HARDWARE_INTRINSIC(Vector, CreateScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) -HARDWARE_INTRINSIC(Vector, CreateScalarUnsafe, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Vector, CreateScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) +HARDWARE_INTRINSIC(Vector, CreateScalarUnsafe, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) #endif -HARDWARE_INTRINSIC(Vector, CreateSequence, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, Dot, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, Equals, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, EqualsAny, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, ExtractMostSignificantBits, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, Floor, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, FusedMultiplyAdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, CreateSequence, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, Dot, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, Equals, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, EqualsAny, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, ExtractMostSignificantBits, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, Floor, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, FusedMultiplyAdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) #if defined(TARGET_XARCH) -HARDWARE_INTRINSIC(Vector, GetElement, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_extractps, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsScalarT|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, GetElement, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_extractps, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsScalarT|HW_Flag_AvxOnlyCompatible) #elif defined(TARGET_ARM64) -HARDWARE_INTRINSIC(Vector, GetElement, -1, 2, INS_smov, INS_umov, INS_smov, INS_umov, INS_smov, INS_umov, INS_umov, INS_umov, INS_dup, INS_dup, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsScalarT|HW_Flag_SupportsContainment) +HARDWARE_INTRINSIC(Vector, GetElement, -1, 2, INS_smov, INS_umov, INS_smov, INS_umov, INS_smov, INS_umov, INS_umov, INS_umov, INS_dup, INS_dup, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsScalarT|HW_Flag_SupportsContainment) #else -HARDWARE_INTRINSIC(Vector, GetElement, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsScalarT) +HARDWARE_INTRINSIC(Vector, GetElement, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsScalarT) #endif #if defined(TARGET_XARCH) -HARDWARE_INTRINSIC(Vector, GetLower, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_movdqu32, INS_movdqu32, INS_movups, INS_movupd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, GetLower128, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_vmovdqu64, INS_vmovdqu64, INS_movups, INS_movupd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, GetUpper, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, GetLower, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_movdqu32, INS_movdqu32, INS_movups, INS_movupd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, GetLower128, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_vmovdqu64, INS_vmovdqu64, INS_movups, INS_movupd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, GetUpper, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) #elif defined(TARGET_ARM64) -HARDWARE_INTRINSIC(Vector, GetLower, -1, 1, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Vector, GetUpper, -1, 1, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Vector, GetLower, -1, 1, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Vector, GetUpper, -1, 1, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) #endif -HARDWARE_INTRINSIC(Vector, GreaterThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, GreaterThanAll, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, GreaterThanAny, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, GreaterThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, GreaterThanOrEqualAll, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, GreaterThanOrEqualAny, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, GreaterThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, GreaterThanAll, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, GreaterThanAny, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, GreaterThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, GreaterThanOrEqualAll, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, GreaterThanOrEqualAny, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, IsEvenInteger, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, IsFinite, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, IsInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, IsInteger, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, IsNaN, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, IsNegative, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, IsNegativeInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, IsNormal, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, IsOddInteger, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, IsPositive, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, IsPositiveInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, IsSubnormal, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, IsZero, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, LessThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, LessThanAll, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, LessThanAny, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, LessThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, LessThanOrEqualAll, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, LessThanOrEqualAny, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, LoadAligned, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, LoadAlignedNonTemporal, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, LoadUnsafe, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, Max, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, MaxMagnitude, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, MaxMagnitudeNumber, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, MaxNative, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, MaxNumber, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, Min, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, MinMagnitude, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, MinMagnitudeNumber, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, MinNative, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, MinNumber, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, MultiplyAddEstimate, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, Narrow, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, IsEvenInteger, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, IsFinite, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, IsInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, IsInteger, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, IsNaN, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, IsNegative, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, IsNegativeInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, IsNormal, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, IsOddInteger, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, IsPositive, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, IsPositiveInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, IsSubnormal, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, IsZero, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, LessThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, LessThanAll, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, LessThanAny, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, LessThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, LessThanOrEqualAll, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, LessThanOrEqualAny, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, LoadAligned, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, LoadAlignedNonTemporal, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, LoadUnsafe, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, Max, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, MaxMagnitude, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, MaxMagnitudeNumber, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, MaxNative, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, MaxNumber, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, Min, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, MinMagnitude, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, MinMagnitudeNumber, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, MinNative, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, MinNumber, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, MultiplyAddEstimate, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, Narrow, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) #if defined(TARGET_ARM64) -HARDWARE_INTRINSIC(Vector, NarrowWithSaturation, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, NarrowWithSaturation, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) #else -HARDWARE_INTRINSIC(Vector, NarrowWithSaturation, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, NarrowWithSaturation, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) #endif -HARDWARE_INTRINSIC(Vector, Reverse, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, Round, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, ShiftLeft, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, Shuffle, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(Vector, ShuffleNative, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(Vector, ShuffleNativeFallback, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(Vector, Sqrt, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, StoreAligned, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, StoreAlignedNonTemporal, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, StoreUnsafe, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, SubtractSaturate, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, Sum, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, Reverse, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, Round, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, ShiftLeft, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, Shuffle, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(Vector, ShuffleNative, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(Vector, ShuffleNativeFallback, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(Vector, Sqrt, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, StoreAligned, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, StoreAlignedNonTemporal, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, StoreUnsafe, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, SubtractSaturate, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, Sum, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) #if defined(TARGET_XARCH) -HARDWARE_INTRINSIC(Vector, ToScalar, -1, 1, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd64, INS_movd64, INS_movss, INS_movsd_simd, INS_invalid, -1, -1, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsScalarT|HW_Flag_SpecialImport|HW_Flag_NoRMWSemantics|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, ToVector256, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_movdqu32, INS_movdqu32, INS_movups, INS_movupd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_NoRMWSemantics|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, ToVector256Unsafe, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_movdqu32, INS_movdqu32, INS_movups, INS_movupd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_NoRMWSemantics|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, ToVector512, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_vmovdqu64, INS_vmovdqu64, INS_movups, INS_movupd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(Vector, ToVector512Unsafe, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_vmovdqu64, INS_vmovdqu64, INS_movups, INS_movupd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(Vector, ToScalar, -1, 1, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd64, INS_movd64, INS_movss, INS_movsd_simd, -1, -1, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsScalarT|HW_Flag_SpecialImport|HW_Flag_NoRMWSemantics|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, ToVector256, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_movdqu32, INS_movdqu32, INS_movups, INS_movupd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_NoRMWSemantics|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, ToVector256Unsafe, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_movdqu32, INS_movdqu32, INS_movups, INS_movupd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_NoRMWSemantics|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, ToVector512, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_vmovdqu64, INS_vmovdqu64, INS_movups, INS_movupd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(Vector, ToVector512Unsafe, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_vmovdqu64, INS_vmovdqu64, INS_movups, INS_movupd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NormalizeSmallTypeToInt) #elif defined(TARGET_ARM64) -HARDWARE_INTRINSIC(Vector, ToScalar, -1, 1, INS_smov, INS_umov, INS_smov, INS_umov, INS_smov, INS_umov, INS_umov, INS_umov, INS_dup, INS_dup, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsScalarT|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(Vector, ToVector128, -1, 1, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Vector, ToVector128Unsafe, -1, 1, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, ToScalar, -1, 1, INS_smov, INS_umov, INS_smov, INS_umov, INS_smov, INS_umov, INS_umov, INS_umov, INS_dup, INS_dup, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsScalarT|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Vector, ToVector128, -1, 1, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector, ToVector128Unsafe, -1, 1, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg) #else -HARDWARE_INTRINSIC(Vector, ToScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsScalarT) +HARDWARE_INTRINSIC(Vector, ToScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsScalarT) #endif -HARDWARE_INTRINSIC(Vector, Truncate, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, UnzipEven, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, UnzipOdd, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, WidenLower, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, WidenUpper, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, WithElement, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoContainment|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, Truncate, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, UnzipEven, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, UnzipOdd, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, WidenLower, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, WidenUpper, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, WithElement, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoContainment|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) #if defined(TARGET_XARCH) || defined(TARGET_ARM64) -HARDWARE_INTRINSIC(Vector, WithLower, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, WithUpper, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, WithLower, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, WithUpper, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) #endif -HARDWARE_INTRINSIC(Vector, ZipLower, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, ZipUpper, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, get_AllBitsSet, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, get_E, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, get_Epsilon, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, get_Indices, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, get_NaN, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, get_NegativeInfinity, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, get_NegativeOne, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, get_NegativeZero, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, get_One, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, get_Pi, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, get_PositiveInfinity, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, get_SignSequence, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, get_Tau, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, get_Zero, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, op_Addition, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, op_BitwiseAnd, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, op_BitwiseOr, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, ZipLower, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, ZipUpper, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, get_AllBitsSet, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, get_E, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, get_Epsilon, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, get_Indices, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, get_NaN, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, get_NegativeInfinity, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, get_NegativeOne, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, get_NegativeZero, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, get_One, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, get_Pi, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, get_PositiveInfinity, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, get_SignSequence, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, get_Tau, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, get_Zero, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, op_Addition, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, op_BitwiseAnd, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, op_BitwiseOr, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) #if defined(TARGET_XARCH) -HARDWARE_INTRINSIC(Vector, op_Division, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialSideEffect_Other|HW_Flag_SpecialImport|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, op_Division, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialSideEffect_Other|HW_Flag_SpecialImport|HW_Flag_AvxOnlyCompatible) #else -HARDWARE_INTRINSIC(Vector, op_Division, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, op_Division, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) #endif -HARDWARE_INTRINSIC(Vector, op_Equality, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsBoolean|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, op_ExclusiveOr, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, op_Inequality, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsBoolean|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, op_LeftShift, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, op_Multiply, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, op_OnesComplement, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, op_RightShift, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(Vector, op_Subtraction, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, op_UnaryNegation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, op_UnaryPlus, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) -HARDWARE_INTRINSIC(Vector, op_UnsignedRightShift, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, op_Equality, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsBoolean|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, op_ExclusiveOr, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, op_Inequality, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsBoolean|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, op_LeftShift, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, op_Multiply, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, op_OnesComplement, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, op_RightShift, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector, op_Subtraction, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, op_UnaryNegation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, op_UnaryPlus, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector, op_UnsignedRightShift, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) #define LAST_NI_Vector NI_Vector_op_UnsignedRightShift #if defined(TARGET_XARCH) diff --git a/src/coreclr/jit/hwintrinsiclistarm64.h b/src/coreclr/jit/hwintrinsiclistarm64.h index 35bc5966184561..c04f9bf9a8be30 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64.h +++ b/src/coreclr/jit/hwintrinsiclistarm64.h @@ -16,276 +16,276 @@ // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // AdvSimd Intrinsics #define FIRST_NI_AdvSimd NI_AdvSimd_Abs -HARDWARE_INTRINSIC(AdvSimd, Abs, -1, 1, INS_abs, INS_invalid, INS_abs, INS_invalid, INS_abs, INS_invalid, INS_invalid, INS_invalid, INS_fabs, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, AbsSaturate, -1, 1, INS_sqabs, INS_invalid, INS_sqabs, INS_invalid, INS_sqabs, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, AbsScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fabs, INS_fabs, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, AbsoluteCompareGreaterThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facgt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, AbsoluteCompareGreaterThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facge, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, AbsoluteCompareLessThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facgt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, AbsoluteCompareLessThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facge, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, AbsoluteDifference, -1, 2, INS_sabd, INS_uabd, INS_sabd, INS_uabd, INS_sabd, INS_uabd, INS_invalid, INS_invalid, INS_fabd, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, AbsoluteDifferenceAdd, -1, 3, INS_saba, INS_uaba, INS_saba, INS_uaba, INS_saba, INS_uaba, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, AbsoluteDifferenceWideningLower, 8, 2, INS_sabdl, INS_uabdl, INS_sabdl, INS_uabdl, INS_sabdl, INS_uabdl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, AbsoluteDifferenceWideningLowerAndAdd, 8, 3, INS_sabal, INS_uabal, INS_sabal, INS_uabal, INS_sabal, INS_uabal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, AbsoluteDifferenceWideningUpper, 16, 2, INS_sabdl2, INS_uabdl2, INS_sabdl2, INS_uabdl2, INS_sabdl2, INS_uabdl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, AbsoluteDifferenceWideningUpperAndAdd, 16, 3, INS_sabal2, INS_uabal2, INS_sabal2, INS_uabal2, INS_sabal2, INS_uabal2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, Add, -1, 2, INS_add, INS_add, INS_add, INS_add, INS_add, INS_add, INS_add, INS_add, INS_fadd, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, AddHighNarrowingLower, 8, 2, INS_addhn, INS_addhn, INS_addhn, INS_addhn, INS_addhn, INS_addhn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, AddHighNarrowingUpper, 16, 3, INS_addhn2, INS_addhn2, INS_addhn2, INS_addhn2, INS_addhn2, INS_addhn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, AddPairwise, 8, 2, INS_addp, INS_addp, INS_addp, INS_addp, INS_addp, INS_addp, INS_invalid, INS_invalid, INS_faddp, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, AddPairwiseWidening, -1, 1, INS_saddlp, INS_uaddlp, INS_saddlp, INS_uaddlp, INS_saddlp, INS_uaddlp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, AddPairwiseWideningAndAdd, -1, 2, INS_sadalp, INS_uadalp, INS_sadalp, INS_uadalp, INS_sadalp, INS_uadalp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, AddPairwiseWideningAndAddScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sadalp, INS_uadalp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, AddPairwiseWideningScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_saddlp, INS_uaddlp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, AddRoundedHighNarrowingLower, 8, 2, INS_raddhn, INS_raddhn, INS_raddhn, INS_raddhn, INS_raddhn, INS_raddhn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, AddRoundedHighNarrowingUpper, 16, 3, INS_raddhn2, INS_raddhn2, INS_raddhn2, INS_raddhn2, INS_raddhn2, INS_raddhn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, AddSaturate, -1, 2, INS_sqadd, INS_uqadd, INS_sqadd, INS_uqadd, INS_sqadd, INS_uqadd, INS_sqadd, INS_uqadd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, AddSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqadd, INS_uqadd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, AddScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_add, INS_add, INS_fadd, INS_fadd, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, AddWideningLower, 8, 2, INS_saddl, INS_uaddl, INS_saddl, INS_uaddl, INS_saddl, INS_uaddl, INS_saddw, INS_uaddw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, AddWideningUpper, 16, 2, INS_saddl2, INS_uaddl2, INS_saddl2, INS_uaddl2, INS_saddl2, INS_uaddl2, INS_saddw2, INS_uaddw2, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, And, -1, 2, INS_and, INS_and, INS_and, INS_and, INS_and, INS_and, INS_and, INS_and, INS_and, INS_and, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, BitwiseClear, -1, 2, INS_bic, INS_bic, INS_bic, INS_bic, INS_bic, INS_bic, INS_bic, INS_bic, INS_bic, INS_bic, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AdvSimd, BitwiseSelect, -1, 3, INS_bsl, INS_bsl, INS_bsl, INS_bsl, INS_bsl, INS_bsl, INS_bsl, INS_bsl, INS_bsl, INS_bsl, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, Ceiling, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintp, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, CeilingScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintp, INS_frintp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, CompareEqual, -1, 2, INS_cmeq, INS_cmeq, INS_cmeq, INS_cmeq, INS_cmeq, INS_cmeq, INS_invalid, INS_invalid, INS_fcmeq, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AdvSimd, CompareGreaterThan, -1, 2, INS_cmgt, INS_cmhi, INS_cmgt, INS_cmhi, INS_cmgt, INS_cmhi, INS_invalid, INS_invalid, INS_fcmgt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AdvSimd, CompareGreaterThanOrEqual, -1, 2, INS_cmge, INS_cmhs, INS_cmge, INS_cmhs, INS_cmge, INS_cmhs, INS_invalid, INS_invalid, INS_fcmge, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AdvSimd, CompareLessThan, -1, 2, INS_cmgt, INS_cmhi, INS_cmgt, INS_cmhi, INS_cmgt, INS_cmhi, INS_invalid, INS_invalid, INS_fcmgt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AdvSimd, CompareLessThanOrEqual, -1, 2, INS_cmge, INS_cmhs, INS_cmge, INS_cmhs, INS_cmge, INS_cmhs, INS_invalid, INS_invalid, INS_fcmge, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AdvSimd, CompareTest, -1, 2, INS_cmtst, INS_cmtst, INS_cmtst, INS_cmtst, INS_cmtst, INS_cmtst, INS_invalid, INS_invalid, INS_cmtst, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundAwayFromZero, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtas, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundAwayFromZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtas, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToEven, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtns, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToEvenScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtns, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToNegativeInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtms, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToNegativeInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtms, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToPositiveInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtps, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToPositiveInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtps, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToZero, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzs, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzs, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ConvertToSingle, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_scvtf, INS_ucvtf, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, ConvertToSingleScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_scvtf, INS_ucvtf, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundAwayFromZero, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtau, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundAwayFromZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtau, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToEven, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtnu, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToEvenScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtnu, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToNegativeInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtmu, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToNegativeInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtmu, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToPositiveInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtpu, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToPositiveInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtpu, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToZero, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzu, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzu, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, DivideScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fdiv, INS_fdiv, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, DuplicateSelectedScalarToVector128, -1, 2, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_invalid, INS_invalid, INS_dup, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, DuplicateSelectedScalarToVector64, -1, 2, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_invalid, INS_invalid, INS_dup, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, DuplicateToVector128, 16, 1, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_invalid, INS_invalid, INS_dup, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment) -HARDWARE_INTRINSIC(AdvSimd, DuplicateToVector64, 8, 1, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_invalid, INS_invalid, INS_dup, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment) -HARDWARE_INTRINSIC(AdvSimd, Extract, -1, 2, INS_smov, INS_umov, INS_smov, INS_umov, INS_smov, INS_umov, INS_umov, INS_umov, INS_dup, INS_dup, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsScalarT) -HARDWARE_INTRINSIC(AdvSimd, ExtractNarrowingLower, 8, 1, INS_xtn, INS_xtn, INS_xtn, INS_xtn, INS_xtn, INS_xtn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, ExtractNarrowingSaturateLower, 8, 1, INS_sqxtn, INS_uqxtn, INS_sqxtn, INS_uqxtn, INS_sqxtn, INS_uqxtn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, ExtractNarrowingSaturateUnsignedLower, 8, 1, INS_invalid, INS_sqxtun, INS_invalid, INS_sqxtun, INS_invalid, INS_sqxtun, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, ExtractNarrowingSaturateUnsignedUpper, 16, 2, INS_invalid, INS_sqxtun2, INS_invalid, INS_sqxtun2, INS_invalid, INS_sqxtun2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ExtractNarrowingSaturateUpper, 16, 2, INS_sqxtn2, INS_uqxtn2, INS_sqxtn2, INS_uqxtn2, INS_sqxtn2, INS_uqxtn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ExtractNarrowingUpper, 16, 2, INS_xtn2, INS_xtn2, INS_xtn2, INS_xtn2, INS_xtn2, INS_xtn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ExtractVector128, 16, 3, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, ExtractVector64, 8, 3, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_invalid, INS_invalid, INS_ext, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, Floor, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintm, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, FloorScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintm, INS_frintm, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, FusedAddHalving, -1, 2, INS_shadd, INS_uhadd, INS_shadd, INS_uhadd, INS_shadd, INS_uhadd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, FusedAddRoundedHalving, -1, 2, INS_srhadd, INS_urhadd, INS_srhadd, INS_urhadd, INS_srhadd, INS_urhadd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, FusedMultiplyAdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmla, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, FusedMultiplyAddNegatedScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fnmadd, INS_fnmadd, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, FusedMultiplyAddScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmadd, INS_fmadd, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, FusedMultiplySubtract, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmls, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, FusedMultiplySubtractNegatedScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fnmsub, INS_fnmsub, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, FusedMultiplySubtractScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmsub, INS_fmsub, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, FusedSubtractHalving, -1, 2, INS_shsub, INS_uhsub, INS_shsub, INS_uhsub, INS_shsub, INS_uhsub, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, Insert, -1, 3, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment) -HARDWARE_INTRINSIC(AdvSimd, InsertScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ins, INS_ins, INS_invalid, INS_ins, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, LeadingSignCount, -1, 1, INS_cls, INS_invalid, INS_cls, INS_invalid, INS_cls, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, LeadingZeroCount, -1, 1, INS_clz, INS_clz, INS_clz, INS_clz, INS_clz, INS_clz, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, Load2xVector64, 8, 1, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_invalid, INS_invalid, INS_ld1_2regs, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, Load2xVector64AndUnzip, 8, 1, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_invalid, INS_invalid, INS_ld2, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, Load3xVector64, 8, 1, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_invalid, INS_invalid, INS_ld1_3regs, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, Load3xVector64AndUnzip, 8, 1, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_invalid, INS_invalid, INS_ld3, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, Load4xVector64, 8, 1, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_invalid, INS_invalid, INS_ld1_4regs, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, Load4xVector64AndUnzip, 8, 1, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_invalid, INS_invalid, INS_ld4, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, LoadAndInsertScalar, -1, 3, INS_ld1, INS_ld1, INS_ld1, INS_ld1, INS_ld1, INS_ld1, INS_ld1, INS_ld1, INS_ld1, INS_ld1, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, LoadAndInsertScalarVector64x2, 8, 3, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_invalid, INS_invalid, INS_ld2, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, LoadAndInsertScalarVector64x3, 8, 3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_invalid, INS_invalid, INS_ld3, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, LoadAndInsertScalarVector64x4, 8, 3, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_invalid, INS_invalid, INS_ld4, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, LoadAndReplicateToVector128, 16, 1, INS_ld1r, INS_ld1r, INS_ld1r, INS_ld1r, INS_ld1r, INS_ld1r, INS_invalid, INS_invalid, INS_ld1r, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, LoadAndReplicateToVector64, 8, 1, INS_ld1r, INS_ld1r, INS_ld1r, INS_ld1r, INS_ld1r, INS_ld1r, INS_invalid, INS_invalid, INS_ld1r, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, LoadAndReplicateToVector64x2, 8, 1, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_invalid, INS_invalid, INS_ld2r, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, LoadAndReplicateToVector64x3, 8, 1, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_invalid, INS_invalid, INS_ld3r, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, LoadAndReplicateToVector64x4, 8, 1, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_invalid, INS_invalid, INS_ld4r, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, LoadVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AdvSimd, LoadVector64, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AdvSimd, Max, -1, 2, INS_smax, INS_umax, INS_smax, INS_umax, INS_smax, INS_umax, INS_invalid, INS_invalid, INS_fmax, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, MaxNumber, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmaxnm, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, MaxNumberScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmaxnm, INS_fmaxnm, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, MaxPairwise, 8, 2, INS_smaxp, INS_umaxp, INS_smaxp, INS_umaxp, INS_smaxp, INS_umaxp, INS_invalid, INS_invalid, INS_fmaxp, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, Min, -1, 2, INS_smin, INS_umin, INS_smin, INS_umin, INS_smin, INS_umin, INS_invalid, INS_invalid, INS_fmin, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, MinNumber, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fminnm, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, MinNumberScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fminnm, INS_fminnm, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, MinPairwise, 8, 2, INS_sminp, INS_uminp, INS_sminp, INS_uminp, INS_sminp, INS_uminp, INS_invalid, INS_invalid, INS_fminp, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, Multiply, -1, 2, INS_mul, INS_mul, INS_mul, INS_mul, INS_mul, INS_mul, INS_invalid, INS_invalid, INS_fmul, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, MultiplyAdd, -1, 3, INS_mla, INS_mla, INS_mla, INS_mla, INS_mla, INS_mla, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyAddByScalar, -1, 3, INS_invalid, INS_invalid, INS_mla, INS_mla, INS_mla, INS_mla, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyAddBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_mla, INS_mla, INS_mla, INS_mla, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyByScalar, -1, 2, INS_invalid, INS_invalid, INS_mul, INS_mul, INS_mul, INS_mul, INS_invalid, INS_invalid, INS_fmul, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, MultiplyBySelectedScalar, -1, 3, INS_invalid, INS_invalid, INS_mul, INS_mul, INS_mul, INS_mul, INS_invalid, INS_invalid, INS_fmul, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, MultiplyBySelectedScalarWideningLower, 8, 3, INS_invalid, INS_invalid, INS_smull, INS_umull, INS_smull, INS_umull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, MultiplyBySelectedScalarWideningLowerAndAdd, 8, 4, INS_invalid, INS_invalid, INS_smlal, INS_umlal, INS_smlal, INS_umlal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyBySelectedScalarWideningLowerAndSubtract, 8, 4, INS_invalid, INS_invalid, INS_smlsl, INS_umlsl, INS_smlsl, INS_umlsl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyBySelectedScalarWideningUpper, 16, 3, INS_invalid, INS_invalid, INS_smull2, INS_umull2, INS_smull2, INS_umull2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, MultiplyBySelectedScalarWideningUpperAndAdd, 16, 4, INS_invalid, INS_invalid, INS_smlal2, INS_umlal2, INS_smlal2, INS_umlal2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyBySelectedScalarWideningUpperAndSubtract, 16, 4, INS_invalid, INS_invalid, INS_smlsl2, INS_umlsl2, INS_smlsl2, INS_umlsl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingByScalarSaturateHigh, -1, 2, INS_invalid, INS_invalid, INS_sqdmulh, INS_invalid, INS_sqdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingBySelectedScalarSaturateHigh, -1, 3, INS_invalid, INS_invalid, INS_sqdmulh, INS_invalid, INS_sqdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingSaturateHigh, -1, 2, INS_invalid, INS_invalid, INS_sqdmulh, INS_invalid, INS_sqdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningLowerAndAddSaturate, 8, 3, INS_invalid, INS_invalid, INS_sqdmlal, INS_invalid, INS_sqdmlal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningLowerAndSubtractSaturate, 8, 3, INS_invalid, INS_invalid, INS_sqdmlsl, INS_invalid, INS_sqdmlsl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningLowerByScalarAndAddSaturate, 8, 3, INS_invalid, INS_invalid, INS_sqdmlal, INS_invalid, INS_sqdmlal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningLowerByScalarAndSubtractSaturate, 8, 3, INS_invalid, INS_invalid, INS_sqdmlsl, INS_invalid, INS_sqdmlsl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningLowerBySelectedScalarAndAddSaturate, 8, 4, INS_invalid, INS_invalid, INS_sqdmlal, INS_invalid, INS_sqdmlal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningLowerBySelectedScalarAndSubtractSaturate, 8, 4, INS_invalid, INS_invalid, INS_sqdmlsl, INS_invalid, INS_sqdmlsl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningSaturateLower, 8, 2, INS_invalid, INS_invalid, INS_sqdmull, INS_invalid, INS_sqdmull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningSaturateLowerByScalar, 8, 2, INS_invalid, INS_invalid, INS_sqdmull, INS_invalid, INS_sqdmull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningSaturateLowerBySelectedScalar, 8, 3, INS_invalid, INS_invalid, INS_sqdmull, INS_invalid, INS_sqdmull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningSaturateUpper, 16, 2, INS_invalid, INS_invalid, INS_sqdmull2, INS_invalid, INS_sqdmull2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningSaturateUpperByScalar, 16, 2, INS_invalid, INS_invalid, INS_sqdmull2, INS_invalid, INS_sqdmull2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningSaturateUpperBySelectedScalar, 16, 3, INS_invalid, INS_invalid, INS_sqdmull2, INS_invalid, INS_sqdmull2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningUpperAndAddSaturate, 16, 3, INS_invalid, INS_invalid, INS_sqdmlal2, INS_invalid, INS_sqdmlal2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningUpperAndSubtractSaturate, 16, 3, INS_invalid, INS_invalid, INS_sqdmlsl2, INS_invalid, INS_sqdmlsl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningUpperByScalarAndAddSaturate, 16, 3, INS_invalid, INS_invalid, INS_sqdmlal2, INS_invalid, INS_sqdmlal2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningUpperByScalarAndSubtractSaturate, 16, 3, INS_invalid, INS_invalid, INS_sqdmlsl2, INS_invalid, INS_sqdmlsl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningUpperBySelectedScalarAndAddSaturate, 16, 4, INS_invalid, INS_invalid, INS_sqdmlal2, INS_invalid, INS_sqdmlal2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningUpperBySelectedScalarAndSubtractSaturate, 16, 4, INS_invalid, INS_invalid, INS_sqdmlsl2, INS_invalid, INS_sqdmlsl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyRoundedDoublingByScalarSaturateHigh, -1, 2, INS_invalid, INS_invalid, INS_sqrdmulh, INS_invalid, INS_sqrdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, MultiplyRoundedDoublingBySelectedScalarSaturateHigh, -1, 3, INS_invalid, INS_invalid, INS_sqrdmulh, INS_invalid, INS_sqrdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, MultiplyRoundedDoublingSaturateHigh, -1, 2, INS_invalid, INS_invalid, INS_sqrdmulh, INS_invalid, INS_sqrdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, MultiplyScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmul, INS_fmul, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, MultiplyScalarBySelectedScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmul, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, MultiplySubtract, -1, 3, INS_mls, INS_mls, INS_mls, INS_mls, INS_mls, INS_mls, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplySubtractByScalar, -1, 3, INS_invalid, INS_invalid, INS_mls, INS_mls, INS_mls, INS_mls, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplySubtractBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_mls, INS_mls, INS_mls, INS_mls, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyWideningLower, 8, 2, INS_smull, INS_umull, INS_smull, INS_umull, INS_smull, INS_umull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, MultiplyWideningLowerAndAdd, 8, 3, INS_smlal, INS_umlal, INS_smlal, INS_umlal, INS_smlal, INS_umlal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyWideningLowerAndSubtract, 8, 3, INS_smlsl, INS_umlsl, INS_smlsl, INS_umlsl, INS_smlsl, INS_umlsl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyWideningUpper, 16, 2, INS_smull2, INS_umull2, INS_smull2, INS_umull2, INS_smull2, INS_umull2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, MultiplyWideningUpperAndAdd, 16, 3, INS_smlal2, INS_umlal2, INS_smlal2, INS_umlal2, INS_smlal2, INS_umlal2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, MultiplyWideningUpperAndSubtract, 16, 3, INS_smlsl2, INS_umlsl2, INS_smlsl2, INS_umlsl2, INS_smlsl2, INS_umlsl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, Negate, -1, 1, INS_neg, INS_invalid, INS_neg, INS_invalid, INS_neg, INS_invalid, INS_invalid, INS_invalid, INS_fneg, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, NegateSaturate, -1, 1, INS_sqneg, INS_invalid, INS_sqneg, INS_invalid, INS_sqneg, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, NegateScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fneg, INS_fneg, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, Not, -1, 1, INS_mvn, INS_mvn, INS_mvn, INS_mvn, INS_mvn, INS_mvn, INS_mvn, INS_mvn, INS_mvn, INS_mvn, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, Or, -1, 2, INS_orr, INS_orr, INS_orr, INS_orr, INS_orr, INS_orr, INS_orr, INS_orr, INS_orr, INS_orr, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, OrNot, -1, 2, INS_orn, INS_orn, INS_orn, INS_orn, INS_orn, INS_orn, INS_orn, INS_orn, INS_orn, INS_orn, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AdvSimd, PolynomialMultiply, -1, 2, INS_pmul, INS_pmul, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, PolynomialMultiplyWideningLower, 8, 2, INS_pmull, INS_pmull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, PolynomialMultiplyWideningUpper, 16, 2, INS_pmull2, INS_pmull2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, PopCount, -1, 1, INS_cnt, INS_cnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, ReciprocalEstimate, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_urecpe, INS_invalid, INS_invalid, INS_frecpe, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, ReciprocalSquareRootEstimate, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ursqrte, INS_invalid, INS_invalid, INS_frsqrte, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, ReciprocalSquareRootStep, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frsqrts, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, ReciprocalStep, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frecps, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, ReverseElement16, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rev32, INS_rev32, INS_rev64, INS_rev64, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, ReverseElement32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rev64, INS_rev64, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, ReverseElement8, -1, 1, INS_invalid, INS_invalid, INS_rev16, INS_rev16, INS_rev32, INS_rev32, INS_rev64, INS_rev64, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, RoundAwayFromZero, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frinta, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, RoundAwayFromZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frinta, INS_frinta, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, RoundToNearest, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintn, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, RoundToNearestScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintn, INS_frintn, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, RoundToNegativeInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintm, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, RoundToNegativeInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintm, INS_frintm, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, RoundToPositiveInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintp, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, RoundToPositiveInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintp, INS_frintp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, RoundToZero, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintz, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, RoundToZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintz, INS_frintz, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftArithmetic, -1, 2, INS_sshl, INS_invalid, INS_sshl, INS_invalid, INS_sshl, INS_invalid, INS_sshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, ShiftArithmeticRounded, -1, 2, INS_srshl, INS_invalid, INS_srshl, INS_invalid, INS_srshl, INS_invalid, INS_srshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, ShiftArithmeticRoundedSaturate, -1, 2, INS_sqrshl, INS_invalid, INS_sqrshl, INS_invalid, INS_sqrshl, INS_invalid, INS_sqrshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, ShiftArithmeticRoundedSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqrshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftArithmeticRoundedScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_srshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftArithmeticSaturate, -1, 2, INS_sqshl, INS_invalid, INS_sqshl, INS_invalid, INS_sqshl, INS_invalid, INS_sqshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, ShiftArithmeticSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftArithmeticScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftLeftAndInsert, -1, 3, INS_sli, INS_sli, INS_sli, INS_sli, INS_sli, INS_sli, INS_sli, INS_sli, INS_sli, INS_sli, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ShiftLeftAndInsertScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sli, INS_sli, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogical, -1, 2, INS_shl, INS_shl, INS_shl, INS_shl, INS_shl, INS_shl, INS_shl, INS_shl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_NoJmpTableIMM) -HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalSaturate, -1, 2, INS_sqshl, INS_uqshl, INS_sqshl, INS_uqshl, INS_sqshl, INS_uqshl, INS_sqshl, INS_uqshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqshl, INS_uqshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalSaturateUnsigned, -1, 2, INS_sqshlu, INS_invalid, INS_sqshlu, INS_invalid, INS_sqshlu, INS_invalid, INS_sqshlu, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalSaturateUnsignedScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqshlu, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_shl, INS_shl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar|HW_Flag_NoJmpTableIMM) -HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalWideningLower, 8, 2, INS_sshll, INS_ushll, INS_sshll, INS_ushll, INS_sshll, INS_ushll, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalWideningUpper, 16, 2, INS_sshll2, INS_ushll2, INS_sshll2, INS_ushll2, INS_sshll2, INS_ushll2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, ShiftLogical, -1, 2, INS_ushl, INS_ushl, INS_ushl, INS_ushl, INS_ushl, INS_ushl, INS_ushl, INS_ushl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, ShiftLogicalRounded, -1, 2, INS_urshl, INS_urshl, INS_urshl, INS_urshl, INS_urshl, INS_urshl, INS_urshl, INS_urshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, ShiftLogicalRoundedSaturate, -1, 2, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, ShiftLogicalRoundedSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_uqrshl, INS_uqrshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftLogicalRoundedScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_urshl, INS_urshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftLogicalSaturate, -1, 2, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, ShiftLogicalSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_uqshl, INS_uqshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftLogicalScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ushl, INS_ushl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightAndInsert, -1, 3, INS_sri, INS_sri, INS_sri, INS_sri, INS_sri, INS_sri, INS_sri, INS_sri, INS_sri, INS_sri, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightAndInsertScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sri, INS_sri, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmetic, -1, 2, INS_sshr, INS_invalid, INS_sshr, INS_invalid, INS_sshr, INS_invalid, INS_sshr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_NoJmpTableIMM) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticAdd, -1, 3, INS_ssra, INS_invalid, INS_ssra, INS_invalid, INS_ssra, INS_invalid, INS_ssra, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticAddScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ssra, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticNarrowingSaturateLower, 8, 2, INS_sqshrn, INS_invalid, INS_sqshrn, INS_invalid, INS_sqshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticNarrowingSaturateUnsignedLower, 8, 2, INS_invalid, INS_sqshrun, INS_invalid, INS_sqshrun, INS_invalid, INS_sqshrun, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticNarrowingSaturateUnsignedUpper, 16, 3, INS_invalid, INS_sqshrun2, INS_invalid, INS_sqshrun2, INS_invalid, INS_sqshrun2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticNarrowingSaturateUpper, 16, 3, INS_sqshrn2, INS_invalid, INS_sqshrn2, INS_invalid, INS_sqshrn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRounded, -1, 2, INS_srshr, INS_invalid, INS_srshr, INS_invalid, INS_srshr, INS_invalid, INS_srshr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedAdd, -1, 3, INS_srsra, INS_invalid, INS_srsra, INS_invalid, INS_srsra, INS_invalid, INS_srsra, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedAddScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_srsra, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedNarrowingSaturateLower, 8, 2, INS_sqrshrn, INS_invalid, INS_sqrshrn, INS_invalid, INS_sqrshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedNarrowingSaturateUnsignedLower, 8, 2, INS_invalid, INS_sqrshrun, INS_invalid, INS_sqrshrun, INS_invalid, INS_sqrshrun, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedNarrowingSaturateUnsignedUpper, 16, 3, INS_invalid, INS_sqrshrun2, INS_invalid, INS_sqrshrun2, INS_invalid, INS_sqrshrun2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedNarrowingSaturateUpper, 16, 3, INS_sqrshrn2, INS_invalid, INS_sqrshrn2, INS_invalid, INS_sqrshrn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_srshr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sshr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar|HW_Flag_NoJmpTableIMM) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogical, -1, 2, INS_ushr, INS_ushr, INS_ushr, INS_ushr, INS_ushr, INS_ushr, INS_ushr, INS_ushr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_NoJmpTableIMM) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalAdd, -1, 3, INS_usra, INS_usra, INS_usra, INS_usra, INS_usra, INS_usra, INS_usra, INS_usra, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalAddScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_usra, INS_usra, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalNarrowingLower, 8, 2, INS_shrn, INS_shrn, INS_shrn, INS_shrn, INS_shrn, INS_shrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalNarrowingSaturateLower, 8, 2, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalNarrowingSaturateUpper, 16, 3, INS_uqshrn2, INS_uqshrn2, INS_uqshrn2, INS_uqshrn2, INS_uqshrn2, INS_uqshrn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalNarrowingUpper, 16, 3, INS_shrn2, INS_shrn2, INS_shrn2, INS_shrn2, INS_shrn2, INS_shrn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRounded, -1, 2, INS_urshr, INS_urshr, INS_urshr, INS_urshr, INS_urshr, INS_urshr, INS_urshr, INS_urshr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedAdd, -1, 3, INS_ursra, INS_ursra, INS_ursra, INS_ursra, INS_ursra, INS_ursra, INS_ursra, INS_ursra, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedAddScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ursra, INS_ursra, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedNarrowingLower, 8, 2, INS_rshrn, INS_rshrn, INS_rshrn, INS_rshrn, INS_rshrn, INS_rshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedNarrowingSaturateLower, 8, 2, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedNarrowingSaturateUpper, 16, 3, INS_uqrshrn2, INS_uqrshrn2, INS_uqrshrn2, INS_uqrshrn2, INS_uqrshrn2, INS_uqrshrn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedNarrowingUpper, 16, 3, INS_rshrn2, INS_rshrn2, INS_rshrn2, INS_rshrn2, INS_rshrn2, INS_rshrn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_urshr, INS_urshr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ushr, INS_ushr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar|HW_Flag_NoJmpTableIMM) -HARDWARE_INTRINSIC(AdvSimd, SignExtendWideningLower, 8, 1, INS_sxtl, INS_invalid, INS_sxtl, INS_invalid, INS_sxtl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, SignExtendWideningUpper, 16, 1, INS_sxtl2, INS_invalid, INS_sxtl2, INS_invalid, INS_sxtl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, SqrtScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fsqrt, INS_fsqrt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, Store, -1, 2, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_invalid, INS_invalid, INS_st1_2regs, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_BaseTypeFromValueTupleArg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, StoreSelectedScalar, -1, 3, INS_st1, INS_st1, INS_st1, INS_st1, INS_st1, INS_st1, INS_st1, INS_st1, INS_st1, INS_st1, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_BaseTypeFromValueTupleArg|HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, StoreVectorAndZip, 8, 2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_BaseTypeFromValueTupleArg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, Subtract, -1, 2, INS_sub, INS_sub, INS_sub, INS_sub, INS_sub, INS_sub, INS_sub, INS_sub, INS_fsub, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, SubtractHighNarrowingLower, 8, 2, INS_subhn, INS_subhn, INS_subhn, INS_subhn, INS_subhn, INS_subhn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, SubtractHighNarrowingUpper, 16, 3, INS_subhn2, INS_subhn2, INS_subhn2, INS_subhn2, INS_subhn2, INS_subhn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, SubtractRoundedHighNarrowingLower, 8, 2, INS_rsubhn, INS_rsubhn, INS_rsubhn, INS_rsubhn, INS_rsubhn, INS_rsubhn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, SubtractRoundedHighNarrowingUpper, 16, 3, INS_rsubhn2, INS_rsubhn2, INS_rsubhn2, INS_rsubhn2, INS_rsubhn2, INS_rsubhn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd, SubtractSaturate, -1, 2, INS_sqsub, INS_uqsub, INS_sqsub, INS_uqsub, INS_sqsub, INS_uqsub, INS_sqsub, INS_uqsub, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd, SubtractSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqsub, INS_uqsub, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, SubtractScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sub, INS_sub, INS_fsub, INS_fsub, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd, SubtractWideningLower, 8, 2, INS_ssubl, INS_usubl, INS_ssubl, INS_usubl, INS_ssubl, INS_usubl, INS_ssubw, INS_usubw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, SubtractWideningUpper, 16, 2, INS_ssubl2, INS_usubl2, INS_ssubl2, INS_usubl2, INS_ssubl2, INS_usubl2, INS_ssubw2, INS_usubw2, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd, VectorTableLookup, 8, 2, INS_tbl, INS_tbl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, VectorTableLookupExtension, 8, 3, INS_tbx, INS_tbx, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd, Xor, -1, 2, INS_eor, INS_eor, INS_eor, INS_eor, INS_eor, INS_eor, INS_eor, INS_eor, INS_eor, INS_eor, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd, ZeroExtendWideningLower, 8, 1, INS_uxtl, INS_uxtl, INS_uxtl, INS_uxtl, INS_uxtl, INS_uxtl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd, ZeroExtendWideningUpper, 16, 1, INS_uxtl2, INS_uxtl2, INS_uxtl2, INS_uxtl2, INS_uxtl2, INS_uxtl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, Abs, -1, 1, INS_abs, INS_invalid, INS_abs, INS_invalid, INS_abs, INS_invalid, INS_invalid, INS_invalid, INS_fabs, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, AbsSaturate, -1, 1, INS_sqabs, INS_invalid, INS_sqabs, INS_invalid, INS_sqabs, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, AbsScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fabs, INS_fabs, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, AbsoluteCompareGreaterThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facgt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, AbsoluteCompareGreaterThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facge, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, AbsoluteCompareLessThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facgt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, AbsoluteCompareLessThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facge, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, AbsoluteDifference, -1, 2, INS_sabd, INS_uabd, INS_sabd, INS_uabd, INS_sabd, INS_uabd, INS_invalid, INS_invalid, INS_fabd, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, AbsoluteDifferenceAdd, -1, 3, INS_saba, INS_uaba, INS_saba, INS_uaba, INS_saba, INS_uaba, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, AbsoluteDifferenceWideningLower, 8, 2, INS_sabdl, INS_uabdl, INS_sabdl, INS_uabdl, INS_sabdl, INS_uabdl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, AbsoluteDifferenceWideningLowerAndAdd, 8, 3, INS_sabal, INS_uabal, INS_sabal, INS_uabal, INS_sabal, INS_uabal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, AbsoluteDifferenceWideningUpper, 16, 2, INS_sabdl2, INS_uabdl2, INS_sabdl2, INS_uabdl2, INS_sabdl2, INS_uabdl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, AbsoluteDifferenceWideningUpperAndAdd, 16, 3, INS_sabal2, INS_uabal2, INS_sabal2, INS_uabal2, INS_sabal2, INS_uabal2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, Add, -1, 2, INS_add, INS_add, INS_add, INS_add, INS_add, INS_add, INS_add, INS_add, INS_fadd, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, AddHighNarrowingLower, 8, 2, INS_addhn, INS_addhn, INS_addhn, INS_addhn, INS_addhn, INS_addhn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, AddHighNarrowingUpper, 16, 3, INS_addhn2, INS_addhn2, INS_addhn2, INS_addhn2, INS_addhn2, INS_addhn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, AddPairwise, 8, 2, INS_addp, INS_addp, INS_addp, INS_addp, INS_addp, INS_addp, INS_invalid, INS_invalid, INS_faddp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, AddPairwiseWidening, -1, 1, INS_saddlp, INS_uaddlp, INS_saddlp, INS_uaddlp, INS_saddlp, INS_uaddlp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, AddPairwiseWideningAndAdd, -1, 2, INS_sadalp, INS_uadalp, INS_sadalp, INS_uadalp, INS_sadalp, INS_uadalp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, AddPairwiseWideningAndAddScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sadalp, INS_uadalp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, AddPairwiseWideningScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_saddlp, INS_uaddlp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, AddRoundedHighNarrowingLower, 8, 2, INS_raddhn, INS_raddhn, INS_raddhn, INS_raddhn, INS_raddhn, INS_raddhn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, AddRoundedHighNarrowingUpper, 16, 3, INS_raddhn2, INS_raddhn2, INS_raddhn2, INS_raddhn2, INS_raddhn2, INS_raddhn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, AddSaturate, -1, 2, INS_sqadd, INS_uqadd, INS_sqadd, INS_uqadd, INS_sqadd, INS_uqadd, INS_sqadd, INS_uqadd, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, AddSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqadd, INS_uqadd, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, AddScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_add, INS_add, INS_fadd, INS_fadd, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, AddWideningLower, 8, 2, INS_saddl, INS_uaddl, INS_saddl, INS_uaddl, INS_saddl, INS_uaddl, INS_saddw, INS_uaddw, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, AddWideningUpper, 16, 2, INS_saddl2, INS_uaddl2, INS_saddl2, INS_uaddl2, INS_saddl2, INS_uaddl2, INS_saddw2, INS_uaddw2, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, And, -1, 2, INS_and, INS_and, INS_and, INS_and, INS_and, INS_and, INS_and, INS_and, INS_and, INS_and, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, BitwiseClear, -1, 2, INS_bic, INS_bic, INS_bic, INS_bic, INS_bic, INS_bic, INS_bic, INS_bic, INS_bic, INS_bic, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AdvSimd, BitwiseSelect, -1, 3, INS_bsl, INS_bsl, INS_bsl, INS_bsl, INS_bsl, INS_bsl, INS_bsl, INS_bsl, INS_bsl, INS_bsl, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, Ceiling, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, CeilingScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintp, INS_frintp, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, CompareEqual, -1, 2, INS_cmeq, INS_cmeq, INS_cmeq, INS_cmeq, INS_cmeq, INS_cmeq, INS_invalid, INS_invalid, INS_fcmeq, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AdvSimd, CompareGreaterThan, -1, 2, INS_cmgt, INS_cmhi, INS_cmgt, INS_cmhi, INS_cmgt, INS_cmhi, INS_invalid, INS_invalid, INS_fcmgt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AdvSimd, CompareGreaterThanOrEqual, -1, 2, INS_cmge, INS_cmhs, INS_cmge, INS_cmhs, INS_cmge, INS_cmhs, INS_invalid, INS_invalid, INS_fcmge, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AdvSimd, CompareLessThan, -1, 2, INS_cmgt, INS_cmhi, INS_cmgt, INS_cmhi, INS_cmgt, INS_cmhi, INS_invalid, INS_invalid, INS_fcmgt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AdvSimd, CompareLessThanOrEqual, -1, 2, INS_cmge, INS_cmhs, INS_cmge, INS_cmhs, INS_cmge, INS_cmhs, INS_invalid, INS_invalid, INS_fcmge, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AdvSimd, CompareTest, -1, 2, INS_cmtst, INS_cmtst, INS_cmtst, INS_cmtst, INS_cmtst, INS_cmtst, INS_invalid, INS_invalid, INS_cmtst, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundAwayFromZero, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtas, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundAwayFromZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtas, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToEven, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtns, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToEvenScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtns, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToNegativeInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtms, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToNegativeInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtms, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToPositiveInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtps, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToPositiveInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtps, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToZero, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzs, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, ConvertToInt32RoundToZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzs, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ConvertToSingle, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_scvtf, INS_ucvtf, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, ConvertToSingleScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_scvtf, INS_ucvtf, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundAwayFromZero, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtau, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundAwayFromZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtau, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToEven, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtnu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToEvenScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtnu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToNegativeInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtmu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToNegativeInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtmu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToPositiveInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtpu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToPositiveInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtpu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToZero, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, ConvertToUInt32RoundToZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, DivideScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fdiv, INS_fdiv, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, DuplicateSelectedScalarToVector128, -1, 2, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_invalid, INS_invalid, INS_dup, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, DuplicateSelectedScalarToVector64, -1, 2, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_invalid, INS_invalid, INS_dup, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, DuplicateToVector128, 16, 1, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_invalid, INS_invalid, INS_dup, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment) +HARDWARE_INTRINSIC(AdvSimd, DuplicateToVector64, 8, 1, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_dup, INS_invalid, INS_invalid, INS_dup, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment) +HARDWARE_INTRINSIC(AdvSimd, Extract, -1, 2, INS_smov, INS_umov, INS_smov, INS_umov, INS_smov, INS_umov, INS_umov, INS_umov, INS_dup, INS_dup, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsScalarT) +HARDWARE_INTRINSIC(AdvSimd, ExtractNarrowingLower, 8, 1, INS_xtn, INS_xtn, INS_xtn, INS_xtn, INS_xtn, INS_xtn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, ExtractNarrowingSaturateLower, 8, 1, INS_sqxtn, INS_uqxtn, INS_sqxtn, INS_uqxtn, INS_sqxtn, INS_uqxtn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, ExtractNarrowingSaturateUnsignedLower, 8, 1, INS_invalid, INS_sqxtun, INS_invalid, INS_sqxtun, INS_invalid, INS_sqxtun, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, ExtractNarrowingSaturateUnsignedUpper, 16, 2, INS_invalid, INS_sqxtun2, INS_invalid, INS_sqxtun2, INS_invalid, INS_sqxtun2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ExtractNarrowingSaturateUpper, 16, 2, INS_sqxtn2, INS_uqxtn2, INS_sqxtn2, INS_uqxtn2, INS_sqxtn2, INS_uqxtn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ExtractNarrowingUpper, 16, 2, INS_xtn2, INS_xtn2, INS_xtn2, INS_xtn2, INS_xtn2, INS_xtn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ExtractVector128, 16, 3, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, -1, -1, HW_Category_SIMD, HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, ExtractVector64, 8, 3, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_ext, INS_invalid, INS_invalid, INS_ext, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, Floor, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintm, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, FloorScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintm, INS_frintm, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, FusedAddHalving, -1, 2, INS_shadd, INS_uhadd, INS_shadd, INS_uhadd, INS_shadd, INS_uhadd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, FusedAddRoundedHalving, -1, 2, INS_srhadd, INS_urhadd, INS_srhadd, INS_urhadd, INS_srhadd, INS_urhadd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, FusedMultiplyAdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmla, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, FusedMultiplyAddNegatedScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fnmadd, INS_fnmadd, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, FusedMultiplyAddScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmadd, INS_fmadd, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, FusedMultiplySubtract, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmls, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, FusedMultiplySubtractNegatedScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fnmsub, INS_fnmsub, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, FusedMultiplySubtractScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmsub, INS_fmsub, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, FusedSubtractHalving, -1, 2, INS_shsub, INS_uhsub, INS_shsub, INS_uhsub, INS_shsub, INS_uhsub, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, Insert, -1, 3, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, -1, -1, HW_Category_SIMD, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment) +HARDWARE_INTRINSIC(AdvSimd, InsertScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ins, INS_ins, INS_invalid, INS_ins, -1, -1, HW_Category_SIMD, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, LeadingSignCount, -1, 1, INS_cls, INS_invalid, INS_cls, INS_invalid, INS_cls, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, LeadingZeroCount, -1, 1, INS_clz, INS_clz, INS_clz, INS_clz, INS_clz, INS_clz, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, Load2xVector64, 8, 1, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_invalid, INS_invalid, INS_ld1_2regs, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, Load2xVector64AndUnzip, 8, 1, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_invalid, INS_invalid, INS_ld2, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, Load3xVector64, 8, 1, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_invalid, INS_invalid, INS_ld1_3regs, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, Load3xVector64AndUnzip, 8, 1, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_invalid, INS_invalid, INS_ld3, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, Load4xVector64, 8, 1, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_invalid, INS_invalid, INS_ld1_4regs, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, Load4xVector64AndUnzip, 8, 1, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_invalid, INS_invalid, INS_ld4, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, LoadAndInsertScalar, -1, 3, INS_ld1, INS_ld1, INS_ld1, INS_ld1, INS_ld1, INS_ld1, INS_ld1, INS_ld1, INS_ld1, INS_ld1, -1, -1, HW_Category_MemoryLoad, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, LoadAndInsertScalarVector64x2, 8, 3, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_invalid, INS_invalid, INS_ld2, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, LoadAndInsertScalarVector64x3, 8, 3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_invalid, INS_invalid, INS_ld3, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, LoadAndInsertScalarVector64x4, 8, 3, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_invalid, INS_invalid, INS_ld4, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, LoadAndReplicateToVector128, 16, 1, INS_ld1r, INS_ld1r, INS_ld1r, INS_ld1r, INS_ld1r, INS_ld1r, INS_invalid, INS_invalid, INS_ld1r, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, LoadAndReplicateToVector64, 8, 1, INS_ld1r, INS_ld1r, INS_ld1r, INS_ld1r, INS_ld1r, INS_ld1r, INS_invalid, INS_invalid, INS_ld1r, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, LoadAndReplicateToVector64x2, 8, 1, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_invalid, INS_invalid, INS_ld2r, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, LoadAndReplicateToVector64x3, 8, 1, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_invalid, INS_invalid, INS_ld3r, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, LoadAndReplicateToVector64x4, 8, 1, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_invalid, INS_invalid, INS_ld4r, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, LoadVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AdvSimd, LoadVector64, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AdvSimd, Max, -1, 2, INS_smax, INS_umax, INS_smax, INS_umax, INS_smax, INS_umax, INS_invalid, INS_invalid, INS_fmax, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, MaxNumber, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmaxnm, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, MaxNumberScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmaxnm, INS_fmaxnm, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, MaxPairwise, 8, 2, INS_smaxp, INS_umaxp, INS_smaxp, INS_umaxp, INS_smaxp, INS_umaxp, INS_invalid, INS_invalid, INS_fmaxp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, Min, -1, 2, INS_smin, INS_umin, INS_smin, INS_umin, INS_smin, INS_umin, INS_invalid, INS_invalid, INS_fmin, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, MinNumber, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fminnm, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, MinNumberScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fminnm, INS_fminnm, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, MinPairwise, 8, 2, INS_sminp, INS_uminp, INS_sminp, INS_uminp, INS_sminp, INS_uminp, INS_invalid, INS_invalid, INS_fminp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, Multiply, -1, 2, INS_mul, INS_mul, INS_mul, INS_mul, INS_mul, INS_mul, INS_invalid, INS_invalid, INS_fmul, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, MultiplyAdd, -1, 3, INS_mla, INS_mla, INS_mla, INS_mla, INS_mla, INS_mla, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyAddByScalar, -1, 3, INS_invalid, INS_invalid, INS_mla, INS_mla, INS_mla, INS_mla, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyAddBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_mla, INS_mla, INS_mla, INS_mla, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyByScalar, -1, 2, INS_invalid, INS_invalid, INS_mul, INS_mul, INS_mul, INS_mul, INS_invalid, INS_invalid, INS_fmul, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, MultiplyBySelectedScalar, -1, 3, INS_invalid, INS_invalid, INS_mul, INS_mul, INS_mul, INS_mul, INS_invalid, INS_invalid, INS_fmul, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, MultiplyBySelectedScalarWideningLower, 8, 3, INS_invalid, INS_invalid, INS_smull, INS_umull, INS_smull, INS_umull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, MultiplyBySelectedScalarWideningLowerAndAdd, 8, 4, INS_invalid, INS_invalid, INS_smlal, INS_umlal, INS_smlal, INS_umlal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyBySelectedScalarWideningLowerAndSubtract, 8, 4, INS_invalid, INS_invalid, INS_smlsl, INS_umlsl, INS_smlsl, INS_umlsl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyBySelectedScalarWideningUpper, 16, 3, INS_invalid, INS_invalid, INS_smull2, INS_umull2, INS_smull2, INS_umull2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, MultiplyBySelectedScalarWideningUpperAndAdd, 16, 4, INS_invalid, INS_invalid, INS_smlal2, INS_umlal2, INS_smlal2, INS_umlal2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyBySelectedScalarWideningUpperAndSubtract, 16, 4, INS_invalid, INS_invalid, INS_smlsl2, INS_umlsl2, INS_smlsl2, INS_umlsl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingByScalarSaturateHigh, -1, 2, INS_invalid, INS_invalid, INS_sqdmulh, INS_invalid, INS_sqdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingBySelectedScalarSaturateHigh, -1, 3, INS_invalid, INS_invalid, INS_sqdmulh, INS_invalid, INS_sqdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingSaturateHigh, -1, 2, INS_invalid, INS_invalid, INS_sqdmulh, INS_invalid, INS_sqdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningLowerAndAddSaturate, 8, 3, INS_invalid, INS_invalid, INS_sqdmlal, INS_invalid, INS_sqdmlal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningLowerAndSubtractSaturate, 8, 3, INS_invalid, INS_invalid, INS_sqdmlsl, INS_invalid, INS_sqdmlsl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningLowerByScalarAndAddSaturate, 8, 3, INS_invalid, INS_invalid, INS_sqdmlal, INS_invalid, INS_sqdmlal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningLowerByScalarAndSubtractSaturate, 8, 3, INS_invalid, INS_invalid, INS_sqdmlsl, INS_invalid, INS_sqdmlsl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningLowerBySelectedScalarAndAddSaturate, 8, 4, INS_invalid, INS_invalid, INS_sqdmlal, INS_invalid, INS_sqdmlal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningLowerBySelectedScalarAndSubtractSaturate, 8, 4, INS_invalid, INS_invalid, INS_sqdmlsl, INS_invalid, INS_sqdmlsl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningSaturateLower, 8, 2, INS_invalid, INS_invalid, INS_sqdmull, INS_invalid, INS_sqdmull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningSaturateLowerByScalar, 8, 2, INS_invalid, INS_invalid, INS_sqdmull, INS_invalid, INS_sqdmull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningSaturateLowerBySelectedScalar, 8, 3, INS_invalid, INS_invalid, INS_sqdmull, INS_invalid, INS_sqdmull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningSaturateUpper, 16, 2, INS_invalid, INS_invalid, INS_sqdmull2, INS_invalid, INS_sqdmull2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningSaturateUpperByScalar, 16, 2, INS_invalid, INS_invalid, INS_sqdmull2, INS_invalid, INS_sqdmull2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningSaturateUpperBySelectedScalar, 16, 3, INS_invalid, INS_invalid, INS_sqdmull2, INS_invalid, INS_sqdmull2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningUpperAndAddSaturate, 16, 3, INS_invalid, INS_invalid, INS_sqdmlal2, INS_invalid, INS_sqdmlal2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningUpperAndSubtractSaturate, 16, 3, INS_invalid, INS_invalid, INS_sqdmlsl2, INS_invalid, INS_sqdmlsl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningUpperByScalarAndAddSaturate, 16, 3, INS_invalid, INS_invalid, INS_sqdmlal2, INS_invalid, INS_sqdmlal2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningUpperByScalarAndSubtractSaturate, 16, 3, INS_invalid, INS_invalid, INS_sqdmlsl2, INS_invalid, INS_sqdmlsl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningUpperBySelectedScalarAndAddSaturate, 16, 4, INS_invalid, INS_invalid, INS_sqdmlal2, INS_invalid, INS_sqdmlal2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyDoublingWideningUpperBySelectedScalarAndSubtractSaturate, 16, 4, INS_invalid, INS_invalid, INS_sqdmlsl2, INS_invalid, INS_sqdmlsl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyRoundedDoublingByScalarSaturateHigh, -1, 2, INS_invalid, INS_invalid, INS_sqrdmulh, INS_invalid, INS_sqrdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, MultiplyRoundedDoublingBySelectedScalarSaturateHigh, -1, 3, INS_invalid, INS_invalid, INS_sqrdmulh, INS_invalid, INS_sqrdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, MultiplyRoundedDoublingSaturateHigh, -1, 2, INS_invalid, INS_invalid, INS_sqrdmulh, INS_invalid, INS_sqrdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, MultiplyScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmul, INS_fmul, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, MultiplyScalarBySelectedScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmul, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, MultiplySubtract, -1, 3, INS_mls, INS_mls, INS_mls, INS_mls, INS_mls, INS_mls, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplySubtractByScalar, -1, 3, INS_invalid, INS_invalid, INS_mls, INS_mls, INS_mls, INS_mls, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplySubtractBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_mls, INS_mls, INS_mls, INS_mls, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyWideningLower, 8, 2, INS_smull, INS_umull, INS_smull, INS_umull, INS_smull, INS_umull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, MultiplyWideningLowerAndAdd, 8, 3, INS_smlal, INS_umlal, INS_smlal, INS_umlal, INS_smlal, INS_umlal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyWideningLowerAndSubtract, 8, 3, INS_smlsl, INS_umlsl, INS_smlsl, INS_umlsl, INS_smlsl, INS_umlsl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyWideningUpper, 16, 2, INS_smull2, INS_umull2, INS_smull2, INS_umull2, INS_smull2, INS_umull2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, MultiplyWideningUpperAndAdd, 16, 3, INS_smlal2, INS_umlal2, INS_smlal2, INS_umlal2, INS_smlal2, INS_umlal2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, MultiplyWideningUpperAndSubtract, 16, 3, INS_smlsl2, INS_umlsl2, INS_smlsl2, INS_umlsl2, INS_smlsl2, INS_umlsl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, Negate, -1, 1, INS_neg, INS_invalid, INS_neg, INS_invalid, INS_neg, INS_invalid, INS_invalid, INS_invalid, INS_fneg, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, NegateSaturate, -1, 1, INS_sqneg, INS_invalid, INS_sqneg, INS_invalid, INS_sqneg, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, NegateScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fneg, INS_fneg, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, Not, -1, 1, INS_mvn, INS_mvn, INS_mvn, INS_mvn, INS_mvn, INS_mvn, INS_mvn, INS_mvn, INS_mvn, INS_mvn, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, Or, -1, 2, INS_orr, INS_orr, INS_orr, INS_orr, INS_orr, INS_orr, INS_orr, INS_orr, INS_orr, INS_orr, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, OrNot, -1, 2, INS_orn, INS_orn, INS_orn, INS_orn, INS_orn, INS_orn, INS_orn, INS_orn, INS_orn, INS_orn, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AdvSimd, PolynomialMultiply, -1, 2, INS_pmul, INS_pmul, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, PolynomialMultiplyWideningLower, 8, 2, INS_pmull, INS_pmull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, PolynomialMultiplyWideningUpper, 16, 2, INS_pmull2, INS_pmull2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, PopCount, -1, 1, INS_cnt, INS_cnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, ReciprocalEstimate, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_urecpe, INS_invalid, INS_invalid, INS_frecpe, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, ReciprocalSquareRootEstimate, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ursqrte, INS_invalid, INS_invalid, INS_frsqrte, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, ReciprocalSquareRootStep, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frsqrts, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, ReciprocalStep, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frecps, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, ReverseElement16, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rev32, INS_rev32, INS_rev64, INS_rev64, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, ReverseElement32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rev64, INS_rev64, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, ReverseElement8, -1, 1, INS_invalid, INS_invalid, INS_rev16, INS_rev16, INS_rev32, INS_rev32, INS_rev64, INS_rev64, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, RoundAwayFromZero, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frinta, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, RoundAwayFromZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frinta, INS_frinta, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, RoundToNearest, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintn, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, RoundToNearestScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintn, INS_frintn, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, RoundToNegativeInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintm, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, RoundToNegativeInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintm, INS_frintm, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, RoundToPositiveInfinity, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, RoundToPositiveInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintp, INS_frintp, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, RoundToZero, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintz, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, RoundToZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintz, INS_frintz, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftArithmetic, -1, 2, INS_sshl, INS_invalid, INS_sshl, INS_invalid, INS_sshl, INS_invalid, INS_sshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, ShiftArithmeticRounded, -1, 2, INS_srshl, INS_invalid, INS_srshl, INS_invalid, INS_srshl, INS_invalid, INS_srshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, ShiftArithmeticRoundedSaturate, -1, 2, INS_sqrshl, INS_invalid, INS_sqrshl, INS_invalid, INS_sqrshl, INS_invalid, INS_sqrshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, ShiftArithmeticRoundedSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqrshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftArithmeticRoundedScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_srshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftArithmeticSaturate, -1, 2, INS_sqshl, INS_invalid, INS_sqshl, INS_invalid, INS_sqshl, INS_invalid, INS_sqshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, ShiftArithmeticSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftArithmeticScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftLeftAndInsert, -1, 3, INS_sli, INS_sli, INS_sli, INS_sli, INS_sli, INS_sli, INS_sli, INS_sli, INS_sli, INS_sli, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ShiftLeftAndInsertScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sli, INS_sli, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogical, -1, 2, INS_shl, INS_shl, INS_shl, INS_shl, INS_shl, INS_shl, INS_shl, INS_shl, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_NoJmpTableIMM) +HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalSaturate, -1, 2, INS_sqshl, INS_uqshl, INS_sqshl, INS_uqshl, INS_sqshl, INS_uqshl, INS_sqshl, INS_uqshl, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqshl, INS_uqshl, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalSaturateUnsigned, -1, 2, INS_sqshlu, INS_invalid, INS_sqshlu, INS_invalid, INS_sqshlu, INS_invalid, INS_sqshlu, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalSaturateUnsignedScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqshlu, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_shl, INS_shl, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar|HW_Flag_NoJmpTableIMM) +HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalWideningLower, 8, 2, INS_sshll, INS_ushll, INS_sshll, INS_ushll, INS_sshll, INS_ushll, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalWideningUpper, 16, 2, INS_sshll2, INS_ushll2, INS_sshll2, INS_ushll2, INS_sshll2, INS_ushll2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, ShiftLogical, -1, 2, INS_ushl, INS_ushl, INS_ushl, INS_ushl, INS_ushl, INS_ushl, INS_ushl, INS_ushl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, ShiftLogicalRounded, -1, 2, INS_urshl, INS_urshl, INS_urshl, INS_urshl, INS_urshl, INS_urshl, INS_urshl, INS_urshl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, ShiftLogicalRoundedSaturate, -1, 2, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, ShiftLogicalRoundedSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_uqrshl, INS_uqrshl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftLogicalRoundedScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_urshl, INS_urshl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftLogicalSaturate, -1, 2, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, ShiftLogicalSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_uqshl, INS_uqshl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftLogicalScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ushl, INS_ushl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightAndInsert, -1, 3, INS_sri, INS_sri, INS_sri, INS_sri, INS_sri, INS_sri, INS_sri, INS_sri, INS_sri, INS_sri, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightAndInsertScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sri, INS_sri, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmetic, -1, 2, INS_sshr, INS_invalid, INS_sshr, INS_invalid, INS_sshr, INS_invalid, INS_sshr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_NoJmpTableIMM) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticAdd, -1, 3, INS_ssra, INS_invalid, INS_ssra, INS_invalid, INS_ssra, INS_invalid, INS_ssra, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticAddScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ssra, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticNarrowingSaturateLower, 8, 2, INS_sqshrn, INS_invalid, INS_sqshrn, INS_invalid, INS_sqshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticNarrowingSaturateUnsignedLower, 8, 2, INS_invalid, INS_sqshrun, INS_invalid, INS_sqshrun, INS_invalid, INS_sqshrun, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticNarrowingSaturateUnsignedUpper, 16, 3, INS_invalid, INS_sqshrun2, INS_invalid, INS_sqshrun2, INS_invalid, INS_sqshrun2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticNarrowingSaturateUpper, 16, 3, INS_sqshrn2, INS_invalid, INS_sqshrn2, INS_invalid, INS_sqshrn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRounded, -1, 2, INS_srshr, INS_invalid, INS_srshr, INS_invalid, INS_srshr, INS_invalid, INS_srshr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedAdd, -1, 3, INS_srsra, INS_invalid, INS_srsra, INS_invalid, INS_srsra, INS_invalid, INS_srsra, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedAddScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_srsra, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedNarrowingSaturateLower, 8, 2, INS_sqrshrn, INS_invalid, INS_sqrshrn, INS_invalid, INS_sqrshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedNarrowingSaturateUnsignedLower, 8, 2, INS_invalid, INS_sqrshrun, INS_invalid, INS_sqrshrun, INS_invalid, INS_sqrshrun, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedNarrowingSaturateUnsignedUpper, 16, 3, INS_invalid, INS_sqrshrun2, INS_invalid, INS_sqrshrun2, INS_invalid, INS_sqrshrun2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedNarrowingSaturateUpper, 16, 3, INS_sqrshrn2, INS_invalid, INS_sqrshrn2, INS_invalid, INS_sqrshrn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_srshr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sshr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar|HW_Flag_NoJmpTableIMM) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogical, -1, 2, INS_ushr, INS_ushr, INS_ushr, INS_ushr, INS_ushr, INS_ushr, INS_ushr, INS_ushr, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_NoJmpTableIMM) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalAdd, -1, 3, INS_usra, INS_usra, INS_usra, INS_usra, INS_usra, INS_usra, INS_usra, INS_usra, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalAddScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_usra, INS_usra, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalNarrowingLower, 8, 2, INS_shrn, INS_shrn, INS_shrn, INS_shrn, INS_shrn, INS_shrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalNarrowingSaturateLower, 8, 2, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalNarrowingSaturateUpper, 16, 3, INS_uqshrn2, INS_uqshrn2, INS_uqshrn2, INS_uqshrn2, INS_uqshrn2, INS_uqshrn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalNarrowingUpper, 16, 3, INS_shrn2, INS_shrn2, INS_shrn2, INS_shrn2, INS_shrn2, INS_shrn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRounded, -1, 2, INS_urshr, INS_urshr, INS_urshr, INS_urshr, INS_urshr, INS_urshr, INS_urshr, INS_urshr, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedAdd, -1, 3, INS_ursra, INS_ursra, INS_ursra, INS_ursra, INS_ursra, INS_ursra, INS_ursra, INS_ursra, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedAddScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ursra, INS_ursra, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedNarrowingLower, 8, 2, INS_rshrn, INS_rshrn, INS_rshrn, INS_rshrn, INS_rshrn, INS_rshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedNarrowingSaturateLower, 8, 2, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedNarrowingSaturateUpper, 16, 3, INS_uqrshrn2, INS_uqrshrn2, INS_uqrshrn2, INS_uqrshrn2, INS_uqrshrn2, INS_uqrshrn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedNarrowingUpper, 16, 3, INS_rshrn2, INS_rshrn2, INS_rshrn2, INS_rshrn2, INS_rshrn2, INS_rshrn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_urshr, INS_urshr, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ushr, INS_ushr, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar|HW_Flag_NoJmpTableIMM) +HARDWARE_INTRINSIC(AdvSimd, SignExtendWideningLower, 8, 1, INS_sxtl, INS_invalid, INS_sxtl, INS_invalid, INS_sxtl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, SignExtendWideningUpper, 16, 1, INS_sxtl2, INS_invalid, INS_sxtl2, INS_invalid, INS_sxtl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, SqrtScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fsqrt, INS_fsqrt, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, Store, -1, 2, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_invalid, INS_invalid, INS_st1_2regs, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_BaseTypeFromValueTupleArg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, StoreSelectedScalar, -1, 3, INS_st1, INS_st1, INS_st1, INS_st1, INS_st1, INS_st1, INS_st1, INS_st1, INS_st1, INS_st1, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_BaseTypeFromValueTupleArg|HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, StoreVectorAndZip, 8, 2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_BaseTypeFromValueTupleArg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, Subtract, -1, 2, INS_sub, INS_sub, INS_sub, INS_sub, INS_sub, INS_sub, INS_sub, INS_sub, INS_fsub, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, SubtractHighNarrowingLower, 8, 2, INS_subhn, INS_subhn, INS_subhn, INS_subhn, INS_subhn, INS_subhn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, SubtractHighNarrowingUpper, 16, 3, INS_subhn2, INS_subhn2, INS_subhn2, INS_subhn2, INS_subhn2, INS_subhn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, SubtractRoundedHighNarrowingLower, 8, 2, INS_rsubhn, INS_rsubhn, INS_rsubhn, INS_rsubhn, INS_rsubhn, INS_rsubhn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, SubtractRoundedHighNarrowingUpper, 16, 3, INS_rsubhn2, INS_rsubhn2, INS_rsubhn2, INS_rsubhn2, INS_rsubhn2, INS_rsubhn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd, SubtractSaturate, -1, 2, INS_sqsub, INS_uqsub, INS_sqsub, INS_uqsub, INS_sqsub, INS_uqsub, INS_sqsub, INS_uqsub, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd, SubtractSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqsub, INS_uqsub, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, SubtractScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sub, INS_sub, INS_fsub, INS_fsub, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd, SubtractWideningLower, 8, 2, INS_ssubl, INS_usubl, INS_ssubl, INS_usubl, INS_ssubl, INS_usubl, INS_ssubw, INS_usubw, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, SubtractWideningUpper, 16, 2, INS_ssubl2, INS_usubl2, INS_ssubl2, INS_usubl2, INS_ssubl2, INS_usubl2, INS_ssubw2, INS_usubw2, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd, VectorTableLookup, 8, 2, INS_tbl, INS_tbl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, VectorTableLookupExtension, 8, 3, INS_tbx, INS_tbx, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd, Xor, -1, 2, INS_eor, INS_eor, INS_eor, INS_eor, INS_eor, INS_eor, INS_eor, INS_eor, INS_eor, INS_eor, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd, ZeroExtendWideningLower, 8, 1, INS_uxtl, INS_uxtl, INS_uxtl, INS_uxtl, INS_uxtl, INS_uxtl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd, ZeroExtendWideningUpper, 16, 1, INS_uxtl2, INS_uxtl2, INS_uxtl2, INS_uxtl2, INS_uxtl2, INS_uxtl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) #define LAST_NI_AdvSimd NI_AdvSimd_ZeroExtendWideningUpper // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -294,189 +294,189 @@ HARDWARE_INTRINSIC(AdvSimd, ZeroExtendWideningUpper, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // AdvSimd 64-bit only Intrinsics #define FIRST_NI_AdvSimd_Arm64 NI_AdvSimd_Arm64_Abs -HARDWARE_INTRINSIC(AdvSimd_Arm64, Abs, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_abs, INS_invalid, INS_invalid, INS_fabs, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsSaturate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqabs, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsSaturateScalar, 8, 1, INS_sqabs, INS_invalid, INS_sqabs, INS_invalid, INS_sqabs, INS_invalid, INS_sqabs, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_abs, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facgt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facge, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareGreaterThanOrEqualScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facge, INS_facge, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareGreaterThanScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facgt, INS_facgt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facgt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facge, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareLessThanOrEqualScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facge, INS_facge, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareLessThanScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facgt, INS_facgt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteDifference, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fabd, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteDifferenceScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fabd, INS_fabd, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Add, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fadd, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AddAcross, -1, 1, INS_addv, INS_addv, INS_addv, INS_addv, INS_addv, INS_addv, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AddAcrossWidening, -1, 1, INS_saddlv, INS_uaddlv, INS_saddlv, INS_uaddlv, INS_saddlv, INS_uaddlv, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AddPairwise, 16, 2, INS_addp, INS_addp, INS_addp, INS_addp, INS_addp, INS_addp, INS_addp, INS_addp, INS_faddp, INS_faddp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AddPairwiseScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_addp, INS_addp, INS_faddp, INS_faddp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AddSaturate, -1, 2, INS_suqadd, INS_usqadd, INS_suqadd, INS_usqadd, INS_suqadd, INS_usqadd, INS_suqadd, INS_usqadd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd_Arm64, AddSaturateScalar, 8, 2, INS_sqadd, INS_uqadd, INS_sqadd, INS_uqadd, INS_sqadd, INS_uqadd, INS_suqadd, INS_usqadd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Ceiling, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmeq, INS_cmeq, INS_invalid, INS_fcmeq, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareEqualScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmeq, INS_cmeq, INS_fcmeq, INS_fcmeq, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmgt, INS_cmhi, INS_invalid, INS_fcmgt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmge, INS_cmhs, INS_invalid, INS_fcmge, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareGreaterThanOrEqualScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmge, INS_cmhs, INS_fcmge, INS_fcmge, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareGreaterThanScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmgt, INS_cmhi, INS_fcmgt, INS_fcmgt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmgt, INS_cmhi, INS_invalid, INS_fcmgt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmge, INS_cmhs, INS_invalid, INS_fcmge, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareLessThanOrEqualScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmge, INS_cmhs, INS_fcmge, INS_fcmge, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareLessThanScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmgt, INS_cmhi, INS_fcmgt, INS_fcmgt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareTest, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmtst, INS_cmtst, INS_invalid, INS_cmtst, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareTestScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmtst, INS_cmtst, INS_invalid, INS_cmtst, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToDouble, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_scvtf, INS_ucvtf, INS_fcvtl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToDoubleScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_scvtf, INS_ucvtf, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToDoubleUpper, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtl2, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundAwayFromZero, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtas, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundAwayFromZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtas, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToEven, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtns, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToEvenScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtns, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToNegativeInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtms, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToNegativeInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtms, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToPositiveInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtps, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToPositiveInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtps, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToZero, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzs, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzs, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToSingleLower, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtn, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToSingleRoundToOddLower, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtxn, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToSingleRoundToOddUpper, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtxn2, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToSingleUpper, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtn2, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundAwayFromZero, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtau, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundAwayFromZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtau, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToEven, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtnu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToEvenScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtnu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToNegativeInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtmu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToNegativeInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtmu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToPositiveInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtpu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToPositiveInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtpu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToZero, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Divide, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fdiv, INS_fdiv, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, DuplicateSelectedScalarToVector128, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_dup, INS_dup, INS_invalid, INS_dup, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd_Arm64, DuplicateToVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_dup, INS_dup, INS_invalid, INS_dup, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment) -HARDWARE_INTRINSIC(AdvSimd_Arm64, DuplicateToVector64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mov, INS_mov, INS_invalid, INS_fmov, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ExtractNarrowingSaturateScalar, 8, 1, INS_sqxtn, INS_uqxtn, INS_sqxtn, INS_uqxtn, INS_sqxtn, INS_uqxtn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ExtractNarrowingSaturateUnsignedScalar, 8, 1, INS_invalid, INS_sqxtun, INS_invalid, INS_sqxtun, INS_invalid, INS_sqxtun, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Floor, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintm, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplyAdd, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmla, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplyAddByScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmla, INS_fmla, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplyAddBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmla, INS_fmla, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplyAddScalarBySelectedScalar, 8, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmla, INS_fmla, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplySubtract, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmls, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplySubtractByScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmls, INS_fmls, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplySubtractBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmls, INS_fmls, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplySubtractScalarBySelectedScalar, 8, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmls, INS_fmls, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, InsertSelectedScalar, -1, 4, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_NoJmpTableIMM|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Load2xVector128, 16, 1, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Load2xVector128AndUnzip, 16, 1, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Load3xVector128, 16, 1, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Load3xVector128AndUnzip, 16, 1, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Load4xVector128, 16, 1, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Load4xVector128AndUnzip, 16, 1, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndInsertScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoCodeGen) -HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndInsertScalarVector128x2, 16, 3, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndInsertScalarVector128x3, 16, 3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndInsertScalarVector128x4, 16, 3, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndReplicateToVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ld1r, INS_ld1r, INS_invalid, INS_ld1r, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndReplicateToVector128x2, 16, 1, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndReplicateToVector128x3, 16, 1, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndReplicateToVector128x4, 16, 1, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadPairScalarVector64, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ldp, INS_ldp, INS_invalid, INS_invalid, INS_ldp, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_MultiReg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadPairScalarVector64NonTemporal, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ldnp, INS_ldnp, INS_invalid, INS_invalid, INS_ldnp, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_MultiReg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadPairVector128, 16, 1, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_MultiReg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadPairVector128NonTemporal, 16, 1, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_MultiReg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadPairVector64, 8, 1, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_MultiReg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadPairVector64NonTemporal, 8, 1, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_MultiReg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Max, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmax, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxAcross, -1, 1, INS_smaxv, INS_umaxv, INS_smaxv, INS_umaxv, INS_smaxv, INS_umaxv, INS_invalid, INS_invalid, INS_fmaxv, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxNumber, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmaxnm, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxNumberAcross, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmaxnmv, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxNumberPairwise, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmaxnmp, INS_fmaxnmp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxNumberPairwiseScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmaxnmp, INS_fmaxnmp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxPairwise, 16, 2, INS_smaxp, INS_umaxp, INS_smaxp, INS_umaxp, INS_smaxp, INS_umaxp, INS_invalid, INS_invalid, INS_fmaxp, INS_fmaxp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxPairwiseScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmaxp, INS_fmaxp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmax, INS_fmax, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Min, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmin, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MinAcross, -1, 1, INS_sminv, INS_uminv, INS_sminv, INS_uminv, INS_sminv, INS_uminv, INS_invalid, INS_invalid, INS_fminv, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MinNumber, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fminnm, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MinNumberAcross, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fminnmv, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MinNumberPairwise, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fminnmp, INS_fminnmp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MinNumberPairwiseScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fminnmp, INS_fminnmp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MinPairwise, 16, 2, INS_sminp, INS_uminp, INS_sminp, INS_uminp, INS_sminp, INS_uminp, INS_invalid, INS_invalid, INS_fminp, INS_fminp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MinPairwiseScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fminp, INS_fminp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MinScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmin, INS_fmin, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Multiply, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmul, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyByScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmul, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyBySelectedScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmul, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingSaturateHighScalar, 8, 2, INS_invalid, INS_invalid, INS_sqdmulh, INS_invalid, INS_sqdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingScalarBySelectedScalarSaturateHigh, 8, 3, INS_invalid, INS_invalid, INS_sqdmulh, INS_invalid, INS_sqdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingWideningAndAddSaturateScalar, 8, 3, INS_invalid, INS_invalid, INS_sqdmlal, INS_invalid, INS_sqdmlal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingWideningAndSubtractSaturateScalar, 8, 3, INS_invalid, INS_invalid, INS_sqdmlsl, INS_invalid, INS_sqdmlsl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingWideningSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_sqdmull, INS_invalid, INS_sqdmull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingWideningSaturateScalarBySelectedScalar, 8, 3, INS_invalid, INS_invalid, INS_sqdmull, INS_invalid, INS_sqdmull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingWideningScalarBySelectedScalarAndAddSaturate, 8, 4, INS_invalid, INS_invalid, INS_sqdmlal, INS_invalid, INS_sqdmlal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingWideningScalarBySelectedScalarAndSubtractSaturate, 8, 4, INS_invalid, INS_invalid, INS_sqdmlsl, INS_invalid, INS_sqdmlsl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyExtended, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmulx, INS_fmulx, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyExtendedByScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmulx, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyExtendedBySelectedScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmulx, INS_fmulx, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyExtendedScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmulx, INS_fmulx, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyExtendedScalarBySelectedScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmulx, INS_fmulx, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyRoundedDoublingSaturateHighScalar, 8, 2, INS_invalid, INS_invalid, INS_sqrdmulh, INS_invalid, INS_sqrdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyRoundedDoublingScalarBySelectedScalarSaturateHigh, 8, 3, INS_invalid, INS_invalid, INS_sqrdmulh, INS_invalid, INS_sqrdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyScalarBySelectedScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmul, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Negate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_neg, INS_invalid, INS_invalid, INS_fneg, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, NegateSaturate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqneg, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, NegateSaturateScalar, 8, 1, INS_sqneg, INS_invalid, INS_sqneg, INS_invalid, INS_sqneg, INS_invalid, INS_sqneg, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, NegateScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_neg, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalEstimate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frecpe, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalEstimateScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frecpe, INS_frecpe, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalExponentScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frecpx, INS_frecpx, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalSquareRootEstimate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frsqrte, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalSquareRootEstimateScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frsqrte, INS_frsqrte, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalSquareRootStep, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frsqrts, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalSquareRootStepScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frsqrts, INS_frsqrts, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalStep, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frecps, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalStepScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frecps, INS_frecps, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ReverseElementBits, -1, 1, INS_rbit, INS_rbit, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, RoundAwayFromZero, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frinta, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, RoundToNearest, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintn, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, RoundToNegativeInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintm, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, RoundToPositiveInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, RoundToZero, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintz, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftArithmeticRoundedSaturateScalar, 8, 2, INS_sqrshl, INS_invalid, INS_sqrshl, INS_invalid, INS_sqrshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftArithmeticSaturateScalar, 8, 2, INS_sqshl, INS_invalid, INS_sqshl, INS_invalid, INS_sqshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftLeftLogicalSaturateScalar, 8, 2, INS_sqshl, INS_uqshl, INS_sqshl, INS_uqshl, INS_sqshl, INS_uqshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftLeftLogicalSaturateUnsignedScalar, 8, 2, INS_sqshlu, INS_invalid, INS_sqshlu, INS_invalid, INS_sqshlu, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftLogicalRoundedSaturateScalar, 8, 2, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftLogicalSaturateScalar, 8, 2, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftRightArithmeticNarrowingSaturateScalar, 8, 2, INS_sqshrn, INS_invalid, INS_sqshrn, INS_invalid, INS_sqshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftRightArithmeticNarrowingSaturateUnsignedScalar, 8, 2, INS_invalid, INS_sqshrun, INS_invalid, INS_sqshrun, INS_invalid, INS_sqshrun, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftRightArithmeticRoundedNarrowingSaturateScalar, 8, 2, INS_sqrshrn, INS_invalid, INS_sqrshrn, INS_invalid, INS_sqrshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftRightArithmeticRoundedNarrowingSaturateUnsignedScalar, 8, 2, INS_invalid, INS_sqrshrun, INS_invalid, INS_sqrshrun, INS_invalid, INS_sqrshrun, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftRightLogicalNarrowingSaturateScalar, 8, 2, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftRightLogicalRoundedNarrowingSaturateScalar, 8, 2, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Sqrt, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fsqrt, INS_fsqrt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Store, 16, 2, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_BaseTypeFromValueTupleArg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, StorePair, -1, 3, INS_stp, INS_stp, INS_stp, INS_stp, INS_stp, INS_stp, INS_stp, INS_stp, INS_stp, INS_stp, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd_Arm64, StorePairNonTemporal, -1, 3, INS_stnp, INS_stnp, INS_stnp, INS_stnp, INS_stnp, INS_stnp, INS_stnp, INS_stnp, INS_stnp, INS_stp, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd_Arm64, StorePairScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_stp, INS_stp, INS_invalid, INS_invalid, INS_stp, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd_Arm64, StorePairScalarNonTemporal, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_stnp, INS_stnp, INS_invalid, INS_invalid, INS_stnp, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AdvSimd_Arm64, StoreSelectedScalar, 16, 3, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_BaseTypeFromValueTupleArg|HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, StoreVectorAndZip, 16, 2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_BaseTypeFromValueTupleArg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, Subtract, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fsub, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, SubtractSaturateScalar, 8, 2, INS_sqsub, INS_uqsub, INS_sqsub, INS_uqsub, INS_sqsub, INS_uqsub, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(AdvSimd_Arm64, TransposeEven, -1, 2, INS_trn1, INS_trn1, INS_trn1, INS_trn1, INS_trn1, INS_trn1, INS_trn1, INS_trn1, INS_trn1, INS_trn1, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, TransposeOdd, -1, 2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, UnzipEven, -1, 2, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, UnzipOdd, -1, 2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, VectorTableLookup, 16, 2, INS_tbl, INS_tbl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, VectorTableLookupExtension, 16, 3, INS_tbx, INS_tbx, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ZipHigh, -1, 2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AdvSimd_Arm64, ZipLow, -1, 2, INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Abs, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_abs, INS_invalid, INS_invalid, INS_fabs, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsSaturate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqabs, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsSaturateScalar, 8, 1, INS_sqabs, INS_invalid, INS_sqabs, INS_invalid, INS_sqabs, INS_invalid, INS_sqabs, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_abs, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facgt, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facge, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareGreaterThanOrEqualScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facge, INS_facge, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareGreaterThanScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facgt, INS_facgt, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facgt, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facge, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareLessThanOrEqualScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facge, INS_facge, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteCompareLessThanScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_facgt, INS_facgt, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteDifference, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fabd, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AbsoluteDifferenceScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fabd, INS_fabd, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Add, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fadd, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AddAcross, -1, 1, INS_addv, INS_addv, INS_addv, INS_addv, INS_addv, INS_addv, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AddAcrossWidening, -1, 1, INS_saddlv, INS_uaddlv, INS_saddlv, INS_uaddlv, INS_saddlv, INS_uaddlv, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AddPairwise, 16, 2, INS_addp, INS_addp, INS_addp, INS_addp, INS_addp, INS_addp, INS_addp, INS_addp, INS_faddp, INS_faddp, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AddPairwiseScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_addp, INS_addp, INS_faddp, INS_faddp, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AddSaturate, -1, 2, INS_suqadd, INS_usqadd, INS_suqadd, INS_usqadd, INS_suqadd, INS_usqadd, INS_suqadd, INS_usqadd, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd_Arm64, AddSaturateScalar, 8, 2, INS_sqadd, INS_uqadd, INS_sqadd, INS_uqadd, INS_sqadd, INS_uqadd, INS_suqadd, INS_usqadd, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Ceiling, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintp, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmeq, INS_cmeq, INS_invalid, INS_fcmeq, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareEqualScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmeq, INS_cmeq, INS_fcmeq, INS_fcmeq, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmgt, INS_cmhi, INS_invalid, INS_fcmgt, -1, -1, HW_Category_SIMD, HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmge, INS_cmhs, INS_invalid, INS_fcmge, -1, -1, HW_Category_SIMD, HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareGreaterThanOrEqualScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmge, INS_cmhs, INS_fcmge, INS_fcmge, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareGreaterThanScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmgt, INS_cmhi, INS_fcmgt, INS_fcmgt, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmgt, INS_cmhi, INS_invalid, INS_fcmgt, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmge, INS_cmhs, INS_invalid, INS_fcmge, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareLessThanOrEqualScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmge, INS_cmhs, INS_fcmge, INS_fcmge, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareLessThanScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmgt, INS_cmhi, INS_fcmgt, INS_fcmgt, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareTest, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmtst, INS_cmtst, INS_invalid, INS_cmtst, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd_Arm64, CompareTestScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmtst, INS_cmtst, INS_invalid, INS_cmtst, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToDouble, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_scvtf, INS_ucvtf, INS_fcvtl, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToDoubleScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_scvtf, INS_ucvtf, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToDoubleUpper, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtl2, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundAwayFromZero, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtas, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundAwayFromZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtas, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToEven, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtns, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToEvenScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtns, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToNegativeInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtms, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToNegativeInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtms, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToPositiveInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtps, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToPositiveInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtps, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToZero, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzs, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToInt64RoundToZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzs, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToSingleLower, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtn, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToSingleRoundToOddLower, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtxn, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToSingleRoundToOddUpper, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtxn2, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToSingleUpper, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtn2, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundAwayFromZero, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtau, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundAwayFromZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtau, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToEven, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtnu, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToEvenScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtnu, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToNegativeInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtmu, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToNegativeInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtmu, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToPositiveInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtpu, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToPositiveInfinityScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtpu, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToZero, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzu, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ConvertToUInt64RoundToZeroScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzu, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Divide, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fdiv, INS_fdiv, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, DuplicateSelectedScalarToVector128, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_dup, INS_dup, INS_invalid, INS_dup, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd_Arm64, DuplicateToVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_dup, INS_dup, INS_invalid, INS_dup, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment) +HARDWARE_INTRINSIC(AdvSimd_Arm64, DuplicateToVector64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mov, INS_mov, INS_invalid, INS_fmov, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ExtractNarrowingSaturateScalar, 8, 1, INS_sqxtn, INS_uqxtn, INS_sqxtn, INS_uqxtn, INS_sqxtn, INS_uqxtn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ExtractNarrowingSaturateUnsignedScalar, 8, 1, INS_invalid, INS_sqxtun, INS_invalid, INS_sqxtun, INS_invalid, INS_sqxtun, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Floor, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintm, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplyAdd, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmla, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplyAddByScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmla, INS_fmla, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplyAddBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmla, INS_fmla, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplyAddScalarBySelectedScalar, 8, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmla, INS_fmla, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplySubtract, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmls, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplySubtractByScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmls, INS_fmls, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplySubtractBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmls, INS_fmls, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(AdvSimd_Arm64, FusedMultiplySubtractScalarBySelectedScalar, 8, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmls, INS_fmls, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, InsertSelectedScalar, -1, 4, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, -1, -1, HW_Category_SIMD, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_NoJmpTableIMM|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Load2xVector128, 16, 1, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, INS_ld1_2regs, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Load2xVector128AndUnzip, 16, 1, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Load3xVector128, 16, 1, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, INS_ld1_3regs, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Load3xVector128AndUnzip, 16, 1, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Load4xVector128, 16, 1, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, INS_ld1_4regs, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Load4xVector128AndUnzip, 16, 1, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndInsertScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoCodeGen) +HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndInsertScalarVector128x2, 16, 3, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, INS_ld2, -1, -1, HW_Category_MemoryLoad, HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndInsertScalarVector128x3, 16, 3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, INS_ld3, -1, -1, HW_Category_MemoryLoad, HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndInsertScalarVector128x4, 16, 3, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, INS_ld4, -1, -1, HW_Category_MemoryLoad, HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndReplicateToVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ld1r, INS_ld1r, INS_invalid, INS_ld1r, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndReplicateToVector128x2, 16, 1, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, INS_ld2r, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndReplicateToVector128x3, 16, 1, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, INS_ld3r, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadAndReplicateToVector128x4, 16, 1, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, INS_ld4r, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_MultiReg|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadPairScalarVector64, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ldp, INS_ldp, INS_invalid, INS_invalid, INS_ldp, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_MultiReg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadPairScalarVector64NonTemporal, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ldnp, INS_ldnp, INS_invalid, INS_invalid, INS_ldnp, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_MultiReg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadPairVector128, 16, 1, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_MultiReg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadPairVector128NonTemporal, 16, 1, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_MultiReg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadPairVector64, 8, 1, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, INS_ldp, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_MultiReg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AdvSimd_Arm64, LoadPairVector64NonTemporal, 8, 1, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, INS_ldnp, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg|HW_Flag_MultiReg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Max, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmax, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxAcross, -1, 1, INS_smaxv, INS_umaxv, INS_smaxv, INS_umaxv, INS_smaxv, INS_umaxv, INS_invalid, INS_invalid, INS_fmaxv, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxNumber, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmaxnm, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxNumberAcross, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmaxnmv, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxNumberPairwise, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmaxnmp, INS_fmaxnmp, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxNumberPairwiseScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmaxnmp, INS_fmaxnmp, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxPairwise, 16, 2, INS_smaxp, INS_umaxp, INS_smaxp, INS_umaxp, INS_smaxp, INS_umaxp, INS_invalid, INS_invalid, INS_fmaxp, INS_fmaxp, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxPairwiseScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmaxp, INS_fmaxp, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MaxScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmax, INS_fmax, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Min, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmin, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MinAcross, -1, 1, INS_sminv, INS_uminv, INS_sminv, INS_uminv, INS_sminv, INS_uminv, INS_invalid, INS_invalid, INS_fminv, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MinNumber, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fminnm, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MinNumberAcross, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fminnmv, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MinNumberPairwise, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fminnmp, INS_fminnmp, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MinNumberPairwiseScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fminnmp, INS_fminnmp, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MinPairwise, 16, 2, INS_sminp, INS_uminp, INS_sminp, INS_uminp, INS_sminp, INS_uminp, INS_invalid, INS_invalid, INS_fminp, INS_fminp, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MinPairwiseScalar, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fminp, INS_fminp, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MinScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmin, INS_fmin, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Multiply, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmul, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyByScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmul, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyBySelectedScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmul, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingSaturateHighScalar, 8, 2, INS_invalid, INS_invalid, INS_sqdmulh, INS_invalid, INS_sqdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingScalarBySelectedScalarSaturateHigh, 8, 3, INS_invalid, INS_invalid, INS_sqdmulh, INS_invalid, INS_sqdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingWideningAndAddSaturateScalar, 8, 3, INS_invalid, INS_invalid, INS_sqdmlal, INS_invalid, INS_sqdmlal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingWideningAndSubtractSaturateScalar, 8, 3, INS_invalid, INS_invalid, INS_sqdmlsl, INS_invalid, INS_sqdmlsl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingWideningSaturateScalar, 8, 2, INS_invalid, INS_invalid, INS_sqdmull, INS_invalid, INS_sqdmull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingWideningSaturateScalarBySelectedScalar, 8, 3, INS_invalid, INS_invalid, INS_sqdmull, INS_invalid, INS_sqdmull, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingWideningScalarBySelectedScalarAndAddSaturate, 8, 4, INS_invalid, INS_invalid, INS_sqdmlal, INS_invalid, INS_sqdmlal, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyDoublingWideningScalarBySelectedScalarAndSubtractSaturate, 8, 4, INS_invalid, INS_invalid, INS_sqdmlsl, INS_invalid, INS_sqdmlsl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyExtended, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmulx, INS_fmulx, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyExtendedByScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmulx, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyExtendedBySelectedScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmulx, INS_fmulx, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyExtendedScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmulx, INS_fmulx, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyExtendedScalarBySelectedScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmulx, INS_fmulx, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyRoundedDoublingSaturateHighScalar, 8, 2, INS_invalid, INS_invalid, INS_sqrdmulh, INS_invalid, INS_sqrdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyRoundedDoublingScalarBySelectedScalarSaturateHigh, 8, 3, INS_invalid, INS_invalid, INS_sqrdmulh, INS_invalid, INS_sqrdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, MultiplyScalarBySelectedScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fmul, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Negate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_neg, INS_invalid, INS_invalid, INS_fneg, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, NegateSaturate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqneg, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, NegateSaturateScalar, 8, 1, INS_sqneg, INS_invalid, INS_sqneg, INS_invalid, INS_sqneg, INS_invalid, INS_sqneg, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, NegateScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_neg, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalEstimate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frecpe, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalEstimateScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frecpe, INS_frecpe, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalExponentScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frecpx, INS_frecpx, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalSquareRootEstimate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frsqrte, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalSquareRootEstimateScalar, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frsqrte, INS_frsqrte, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalSquareRootStep, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frsqrts, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalSquareRootStepScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frsqrts, INS_frsqrts, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalStep, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frecps, -1, -1, HW_Category_SIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ReciprocalStepScalar, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frecps, INS_frecps, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ReverseElementBits, -1, 1, INS_rbit, INS_rbit, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, RoundAwayFromZero, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frinta, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, RoundToNearest, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintn, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, RoundToNegativeInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintm, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, RoundToPositiveInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintp, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, RoundToZero, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_frintz, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftArithmeticRoundedSaturateScalar, 8, 2, INS_sqrshl, INS_invalid, INS_sqrshl, INS_invalid, INS_sqrshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftArithmeticSaturateScalar, 8, 2, INS_sqshl, INS_invalid, INS_sqshl, INS_invalid, INS_sqshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftLeftLogicalSaturateScalar, 8, 2, INS_sqshl, INS_uqshl, INS_sqshl, INS_uqshl, INS_sqshl, INS_uqshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftLeftLogicalSaturateUnsignedScalar, 8, 2, INS_sqshlu, INS_invalid, INS_sqshlu, INS_invalid, INS_sqshlu, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftLogicalRoundedSaturateScalar, 8, 2, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_uqrshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftLogicalSaturateScalar, 8, 2, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_uqshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftRightArithmeticNarrowingSaturateScalar, 8, 2, INS_sqshrn, INS_invalid, INS_sqshrn, INS_invalid, INS_sqshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftRightArithmeticNarrowingSaturateUnsignedScalar, 8, 2, INS_invalid, INS_sqshrun, INS_invalid, INS_sqshrun, INS_invalid, INS_sqshrun, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftRightArithmeticRoundedNarrowingSaturateScalar, 8, 2, INS_sqrshrn, INS_invalid, INS_sqrshrn, INS_invalid, INS_sqrshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftRightArithmeticRoundedNarrowingSaturateUnsignedScalar, 8, 2, INS_invalid, INS_sqrshrun, INS_invalid, INS_sqrshrun, INS_invalid, INS_sqrshrun, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftRightLogicalNarrowingSaturateScalar, 8, 2, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_uqshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ShiftRightLogicalRoundedNarrowingSaturateScalar, 8, 2, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_uqrshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Sqrt, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fsqrt, INS_fsqrt, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Store, 16, 2, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, INS_st1_2regs, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_BaseTypeFromValueTupleArg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, StorePair, -1, 3, INS_stp, INS_stp, INS_stp, INS_stp, INS_stp, INS_stp, INS_stp, INS_stp, INS_stp, INS_stp, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd_Arm64, StorePairNonTemporal, -1, 3, INS_stnp, INS_stnp, INS_stnp, INS_stnp, INS_stnp, INS_stnp, INS_stnp, INS_stnp, INS_stnp, INS_stp, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd_Arm64, StorePairScalar, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_stp, INS_stp, INS_invalid, INS_invalid, INS_stp, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd_Arm64, StorePairScalarNonTemporal, 8, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_stnp, INS_stnp, INS_invalid, INS_invalid, INS_stnp, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AdvSimd_Arm64, StoreSelectedScalar, 16, 3, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_BaseTypeFromValueTupleArg|HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, StoreVectorAndZip, 16, 2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, INS_st2, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_BaseTypeFromValueTupleArg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, Subtract, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fsub, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, SubtractSaturateScalar, 8, 2, INS_sqsub, INS_uqsub, INS_sqsub, INS_uqsub, INS_sqsub, INS_uqsub, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(AdvSimd_Arm64, TransposeEven, -1, 2, INS_trn1, INS_trn1, INS_trn1, INS_trn1, INS_trn1, INS_trn1, INS_trn1, INS_trn1, INS_trn1, INS_trn1, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, TransposeOdd, -1, 2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, UnzipEven, -1, 2, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, UnzipOdd, -1, 2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, VectorTableLookup, 16, 2, INS_tbl, INS_tbl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, VectorTableLookupExtension, 16, 3, INS_tbx, INS_tbx, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ZipHigh, -1, 2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AdvSimd_Arm64, ZipLow, -1, 2, INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_zip1, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) #define LAST_NI_AdvSimd_Arm64 NI_AdvSimd_Arm64_ZipLow // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -485,12 +485,12 @@ HARDWARE_INTRINSIC(AdvSimd_Arm64, ZipLow, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // AES Intrinsics #define FIRST_NI_Aes NI_Aes_Decrypt -HARDWARE_INTRINSIC(Aes, Decrypt, 16, 2, INS_invalid, INS_aesd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Aes, Encrypt, 16, 2, INS_invalid, INS_aese, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Aes, InverseMixColumns, 16, 1, INS_invalid, INS_aesimc, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(Aes, MixColumns, 16, 1, INS_invalid, INS_aesmc, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(Aes, PolynomialMultiplyWideningLower, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pmull, INS_pmull, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(Aes, PolynomialMultiplyWideningUpper, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pmull2, INS_pmull2, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(Aes, Decrypt, 16, 2, INS_invalid, INS_aesd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Aes, Encrypt, 16, 2, INS_invalid, INS_aese, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Aes, InverseMixColumns, 16, 1, INS_invalid, INS_aesimc, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(Aes, MixColumns, 16, 1, INS_invalid, INS_aesmc, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(Aes, PolynomialMultiplyWideningLower, 8, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pmull, INS_pmull, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(Aes, PolynomialMultiplyWideningUpper, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pmull2, INS_pmull2, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) #define LAST_NI_Aes NI_Aes_PolynomialMultiplyWideningUpper // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -499,9 +499,9 @@ HARDWARE_INTRINSIC(Aes, PolynomialMultiplyWideningUpper, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Base Intrinsics #define FIRST_NI_ArmBase NI_ArmBase_LeadingZeroCount -HARDWARE_INTRINSIC(ArmBase, LeadingZeroCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_clz, INS_clz, INS_clz, INS_clz, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(ArmBase, ReverseElementBits, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rbit, INS_rbit, INS_rbit, INS_rbit, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(ArmBase, Yield, 0, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(ArmBase, LeadingZeroCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_clz, INS_clz, INS_clz, INS_clz, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(ArmBase, ReverseElementBits, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rbit, INS_rbit, INS_rbit, INS_rbit, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(ArmBase, Yield, 0, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_SpecialSideEffect_Other) #define LAST_NI_ArmBase NI_ArmBase_Yield // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -510,13 +510,13 @@ HARDWARE_INTRINSIC(ArmBase, Yield, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Base 64-bit only Intrinsics #define FIRST_NI_ArmBase_Arm64 NI_ArmBase_Arm64_LeadingSignCount -HARDWARE_INTRINSIC(ArmBase_Arm64, LeadingSignCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cls, INS_invalid, INS_cls, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(ArmBase_Arm64, LeadingZeroCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_clz, INS_clz, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(ArmBase_Arm64, MultiplyHigh, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_smulh, INS_umulh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(ArmBase_Arm64, MultiplyLongAdd, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_smaddl, INS_umaddl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(ArmBase_Arm64, MultiplyLongNeg, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_smnegl, INS_umnegl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(ArmBase_Arm64, MultiplyLongSub, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_smsubl, INS_umsubl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(ArmBase_Arm64, ReverseElementBits, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rbit, INS_rbit, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(ArmBase_Arm64, LeadingSignCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cls, INS_invalid, INS_cls, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(ArmBase_Arm64, LeadingZeroCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_clz, INS_clz, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(ArmBase_Arm64, MultiplyHigh, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_smulh, INS_umulh, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(ArmBase_Arm64, MultiplyLongAdd, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_smaddl, INS_umaddl, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(ArmBase_Arm64, MultiplyLongNeg, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_smnegl, INS_umnegl, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(ArmBase_Arm64, MultiplyLongSub, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_smsubl, INS_umsubl, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(ArmBase_Arm64, ReverseElementBits, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rbit, INS_rbit, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) #define LAST_NI_ArmBase_Arm64 NI_ArmBase_Arm64_ReverseElementBits // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -525,8 +525,8 @@ HARDWARE_INTRINSIC(ArmBase_Arm64, ReverseElementBits, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // CRC32 Intrinsics #define FIRST_NI_Crc32 NI_Crc32_ComputeCrc32 -HARDWARE_INTRINSIC(Crc32, ComputeCrc32, 0, 2, INS_invalid, INS_crc32b, INS_invalid, INS_crc32h, INS_invalid, INS_crc32w, INS_invalid, INS_crc32x, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Crc32, ComputeCrc32C, 0, 2, INS_invalid, INS_crc32cb, INS_invalid, INS_crc32ch, INS_invalid, INS_crc32cw, INS_invalid, INS_crc32cx, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Crc32, ComputeCrc32, 0, 2, INS_invalid, INS_crc32b, INS_invalid, INS_crc32h, INS_invalid, INS_crc32w, INS_invalid, INS_crc32x, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Crc32, ComputeCrc32C, 0, 2, INS_invalid, INS_crc32cb, INS_invalid, INS_crc32ch, INS_invalid, INS_crc32cw, INS_invalid, INS_crc32cx, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen) #define LAST_NI_Crc32 NI_Crc32_ComputeCrc32C // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -535,8 +535,8 @@ HARDWARE_INTRINSIC(Crc32, ComputeCrc32C, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // CRC32 64-bit only Intrinsics #define FIRST_NI_Crc32_Arm64 NI_Crc32_Arm64_ComputeCrc32 -HARDWARE_INTRINSIC(Crc32_Arm64, ComputeCrc32, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_crc32x, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Crc32_Arm64, ComputeCrc32C, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_crc32cx, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Crc32_Arm64, ComputeCrc32, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_crc32x, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Crc32_Arm64, ComputeCrc32C, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_crc32cx, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen) #define LAST_NI_Crc32_Arm64 NI_Crc32_Arm64_ComputeCrc32C // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -545,8 +545,8 @@ HARDWARE_INTRINSIC(Crc32_Arm64, ComputeCrc32C, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // DP Intrinsics #define FIRST_NI_Dp NI_Dp_DotProduct -HARDWARE_INTRINSIC(Dp, DotProduct, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sdot, INS_udot, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Dp, DotProductBySelectedQuadruplet, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sdot, INS_udot, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Dp, DotProduct, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sdot, INS_udot, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Dp, DotProductBySelectedQuadruplet, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sdot, INS_udot, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) #define LAST_NI_Dp NI_Dp_DotProductBySelectedQuadruplet // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -555,10 +555,10 @@ HARDWARE_INTRINSIC(Dp, DotProductBySelectedQuadruplet, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // RDM Intrinsics #define FIRST_NI_Rdm NI_Rdm_MultiplyRoundedDoublingAndAddSaturateHigh -HARDWARE_INTRINSIC(Rdm, MultiplyRoundedDoublingAndAddSaturateHigh, -1, 3, INS_invalid, INS_invalid, INS_sqrdmlah, INS_invalid, INS_sqrdmlah, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Rdm, MultiplyRoundedDoublingAndSubtractSaturateHigh, -1, 3, INS_invalid, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Rdm, MultiplyRoundedDoublingBySelectedScalarAndAddSaturateHigh, -1, 4, INS_invalid, INS_invalid, INS_sqrdmlah, INS_invalid, INS_sqrdmlah, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Rdm, MultiplyRoundedDoublingBySelectedScalarAndSubtractSaturateHigh, -1, 4, INS_invalid, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Rdm, MultiplyRoundedDoublingAndAddSaturateHigh, -1, 3, INS_invalid, INS_invalid, INS_sqrdmlah, INS_invalid, INS_sqrdmlah, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Rdm, MultiplyRoundedDoublingAndSubtractSaturateHigh, -1, 3, INS_invalid, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Rdm, MultiplyRoundedDoublingBySelectedScalarAndAddSaturateHigh, -1, 4, INS_invalid, INS_invalid, INS_sqrdmlah, INS_invalid, INS_sqrdmlah, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Rdm, MultiplyRoundedDoublingBySelectedScalarAndSubtractSaturateHigh, -1, 4, INS_invalid, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) #define LAST_NI_Rdm NI_Rdm_MultiplyRoundedDoublingBySelectedScalarAndSubtractSaturateHigh // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -567,10 +567,10 @@ HARDWARE_INTRINSIC(Rdm, MultiplyRoundedDoublingBySelectedScalarAndSu // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // RDM 64-bit only Intrinsics #define FIRST_NI_Rdm_Arm64 NI_Rdm_Arm64_MultiplyRoundedDoublingAndAddSaturateHighScalar -HARDWARE_INTRINSIC(Rdm_Arm64, MultiplyRoundedDoublingAndAddSaturateHighScalar, 8, 3, INS_invalid, INS_invalid, INS_sqrdmlah, INS_invalid, INS_sqrdmlah, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(Rdm_Arm64, MultiplyRoundedDoublingAndSubtractSaturateHighScalar, 8, 3, INS_invalid, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(Rdm_Arm64, MultiplyRoundedDoublingScalarBySelectedScalarAndAddSaturateHigh, 8, 4, INS_invalid, INS_invalid, INS_sqrdmlah, INS_invalid, INS_sqrdmlah, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(Rdm_Arm64, MultiplyRoundedDoublingScalarBySelectedScalarAndSubtractSaturateHigh, 8, 4, INS_invalid, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Rdm_Arm64, MultiplyRoundedDoublingAndAddSaturateHighScalar, 8, 3, INS_invalid, INS_invalid, INS_sqrdmlah, INS_invalid, INS_sqrdmlah, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Rdm_Arm64, MultiplyRoundedDoublingAndSubtractSaturateHighScalar, 8, 3, INS_invalid, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Rdm_Arm64, MultiplyRoundedDoublingScalarBySelectedScalarAndAddSaturateHigh, 8, 4, INS_invalid, INS_invalid, INS_sqrdmlah, INS_invalid, INS_sqrdmlah, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Rdm_Arm64, MultiplyRoundedDoublingScalarBySelectedScalarAndSubtractSaturateHigh, 8, 4, INS_invalid, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_sqrdmlsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar) #define LAST_NI_Rdm_Arm64 NI_Rdm_Arm64_MultiplyRoundedDoublingScalarBySelectedScalarAndSubtractSaturateHigh // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -579,12 +579,12 @@ HARDWARE_INTRINSIC(Rdm_Arm64, MultiplyRoundedDoublingScalarBySelectedScala // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // SHA1 Intrinsics #define FIRST_NI_Sha1 NI_Sha1_FixedRotate -HARDWARE_INTRINSIC(Sha1, FixedRotate, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) -HARDWARE_INTRINSIC(Sha1, HashUpdateChoose, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha1c, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sha1, HashUpdateMajority, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha1m, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sha1, HashUpdateParity, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha1p, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sha1, ScheduleUpdate0, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha1su0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sha1, ScheduleUpdate1, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha1su1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sha1, FixedRotate, 8, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Sha1, HashUpdateChoose, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha1c, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sha1, HashUpdateMajority, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha1m, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sha1, HashUpdateParity, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha1p, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sha1, ScheduleUpdate0, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha1su0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sha1, ScheduleUpdate1, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha1su1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) #define LAST_NI_Sha1 NI_Sha1_ScheduleUpdate1 // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -593,10 +593,10 @@ HARDWARE_INTRINSIC(Sha1, ScheduleUpdate1, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // SHA256 Intrinsics #define FIRST_NI_Sha256 NI_Sha256_HashUpdate1 -HARDWARE_INTRINSIC(Sha256, HashUpdate1, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha256h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sha256, HashUpdate2, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha256h2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sha256, ScheduleUpdate0, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha256su0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sha256, ScheduleUpdate1, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha256su1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sha256, HashUpdate1, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha256h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sha256, HashUpdate2, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha256h2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sha256, ScheduleUpdate0, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha256su0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sha256, ScheduleUpdate1, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha256su1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) #define LAST_NI_Sha256 NI_Sha256_ScheduleUpdate1 // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -605,10 +605,10 @@ HARDWARE_INTRINSIC(Sha256, ScheduleUpdate1, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // SHA3 Intrinsics #define FIRST_NI_Sha3 NI_Sha3_BitwiseClearXor -HARDWARE_INTRINSIC(Sha3, BitwiseClearXor, 16, 3, INS_bcax, INS_bcax, INS_bcax, INS_bcax, INS_bcax, INS_bcax, INS_bcax, INS_bcax, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sha3, BitwiseRotateLeftBy1AndXor, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rax1, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(Sha3, Xor, 16, 3, INS_eor3, INS_eor3, INS_eor3, INS_eor3, INS_eor3, INS_eor3, INS_eor3, INS_eor3, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sha3, XorRotateRight, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_xar, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sha3, BitwiseClearXor, 16, 3, INS_bcax, INS_bcax, INS_bcax, INS_bcax, INS_bcax, INS_bcax, INS_bcax, INS_bcax, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sha3, BitwiseRotateLeftBy1AndXor, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rax1, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(Sha3, Xor, 16, 3, INS_eor3, INS_eor3, INS_eor3, INS_eor3, INS_eor3, INS_eor3, INS_eor3, INS_eor3, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sha3, XorRotateRight, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_xar, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasImmediateOperand) #define LAST_NI_Sha3 NI_Sha3_XorRotateRight diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index 6c15735e515834..81ccf47a89f8bf 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -16,310 +16,310 @@ // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // SVE Intrinsics #define FIRST_NI_Sve NI_Sve_Abs -HARDWARE_INTRINSIC(Sve, Abs, -1, -1, INS_sve_abs, INS_invalid, INS_sve_abs, INS_invalid, INS_sve_abs, INS_invalid, INS_sve_abs, INS_invalid, INS_sve_fabs, INS_sve_fabs, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, AbsoluteCompareGreaterThan, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_facgt, INS_sve_facgt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, AbsoluteCompareGreaterThanOrEqual, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_facge, INS_sve_facge, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, AbsoluteCompareLessThan, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_faclt, INS_sve_faclt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, AbsoluteCompareLessThanOrEqual, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_facle, INS_sve_facle, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, AbsoluteDifference, -1, -1, INS_sve_sabd, INS_sve_uabd, INS_sve_sabd, INS_sve_uabd, INS_sve_sabd, INS_sve_uabd, INS_sve_sabd, INS_sve_uabd, INS_sve_fabd, INS_sve_fabd, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, Add, -1, -1, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_fadd, INS_sve_fadd, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, AddAcross, -1, 1, INS_sve_saddv, INS_sve_uaddv, INS_sve_saddv, INS_sve_uaddv, INS_sve_saddv, INS_sve_uaddv, INS_sve_uaddv, INS_sve_uaddv, INS_sve_faddv, INS_sve_faddv, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) -HARDWARE_INTRINSIC(Sve, AddRotateComplex, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcadd, INS_sve_fcadd, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve, AddSaturate, -1, 2, INS_sve_sqadd, INS_sve_uqadd, INS_sve_sqadd, INS_sve_uqadd, INS_sve_sqadd, INS_sve_uqadd, INS_sve_sqadd, INS_sve_uqadd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve, AddSequentialAcross, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fadda, INS_sve_fadda, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_ReduceOperation) -HARDWARE_INTRINSIC(Sve, And, -1, -1, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_HasAllMaskVariant) -HARDWARE_INTRINSIC(Sve, AndAcross, -1, -1, INS_sve_andv, INS_sve_andv, INS_sve_andv, INS_sve_andv, INS_sve_andv, INS_sve_andv, INS_sve_andv, INS_sve_andv, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) -HARDWARE_INTRINSIC(Sve, BitwiseClear, -1, -1, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_HasAllMaskVariant) -HARDWARE_INTRINSIC(Sve, BooleanNot, -1, -1, INS_sve_cnot, INS_sve_cnot, INS_sve_cnot, INS_sve_cnot, INS_sve_cnot, INS_sve_cnot, INS_sve_cnot, INS_sve_cnot, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, Compact, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_compact, INS_sve_compact, INS_sve_compact, INS_sve_compact, INS_sve_compact, INS_sve_compact, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, CompareEqual, -1, -1, INS_sve_cmpeq, INS_sve_cmpeq, INS_sve_cmpeq, INS_sve_cmpeq, INS_sve_cmpeq, INS_sve_cmpeq, INS_sve_cmpeq, INS_sve_cmpeq, INS_sve_fcmeq, INS_sve_fcmeq, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, CompareGreaterThan, -1, -1, INS_sve_cmpgt, INS_sve_cmphi, INS_sve_cmpgt, INS_sve_cmphi, INS_sve_cmpgt, INS_sve_cmphi, INS_sve_cmpgt, INS_sve_cmphi, INS_sve_fcmgt, INS_sve_fcmgt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, CompareGreaterThanOrEqual, -1, -1, INS_sve_cmpge, INS_sve_cmphs, INS_sve_cmpge, INS_sve_cmphs, INS_sve_cmpge, INS_sve_cmphs, INS_sve_cmpge, INS_sve_cmphs, INS_sve_fcmge, INS_sve_fcmge, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, CompareLessThan, -1, -1, INS_sve_cmplt, INS_sve_cmplo, INS_sve_cmplt, INS_sve_cmplo, INS_sve_cmplt, INS_sve_cmplo, INS_sve_cmplt, INS_sve_cmplo, INS_sve_fcmlt, INS_sve_fcmlt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, CompareLessThanOrEqual, -1, -1, INS_sve_cmple, INS_sve_cmpls, INS_sve_cmple, INS_sve_cmpls, INS_sve_cmple, INS_sve_cmpls, INS_sve_cmple, INS_sve_cmpls, INS_sve_fcmle, INS_sve_fcmle, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, CompareNotEqualTo, -1, -1, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_fcmne, INS_sve_fcmne, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, CompareUnordered, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcmuo, INS_sve_fcmuo, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, Compute16BitAddresses, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_adr, INS_invalid, INS_sve_adr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, Compute32BitAddresses, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_adr, INS_invalid, INS_sve_adr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, Compute64BitAddresses, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_adr, INS_invalid, INS_sve_adr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, Compute8BitAddresses, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_adr, INS_invalid, INS_sve_adr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, ConditionalExtractAfterLastActiveElement, -1, 3, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, ConditionalExtractAfterLastActiveElementAndReplicate, -1, 3, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, ConditionalExtractLastActiveElement, -1, 3, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, ConditionalExtractLastActiveElementAndReplicate, -1, 3, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, ConditionalSelect, -1, 3, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_SupportsContainment|HW_Flag_HasAllMaskVariant) -HARDWARE_INTRINSIC(Sve, ConvertToDouble, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_scvtf, INS_sve_ucvtf, INS_sve_scvtf, INS_sve_ucvtf, INS_sve_fcvt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, ConvertToInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtzs, INS_sve_fcvtzs, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, ConvertToInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtzs, INS_sve_fcvtzs, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, ConvertToSingle, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_scvtf, INS_sve_ucvtf, INS_sve_scvtf, INS_sve_ucvtf, INS_invalid, INS_sve_fcvt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, ConvertToUInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtzu, INS_sve_fcvtzu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, ConvertToUInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtzu, INS_sve_fcvtzu, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, Count16BitElements, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_cnth, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(Sve, Count32BitElements, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_cntw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(Sve, Count64BitElements, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_cntd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(Sve, Count8BitElements, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_cntb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(Sve, CreateBreakAfterMask, -1, 2, INS_sve_brka, INS_sve_brka, INS_sve_brka, INS_sve_brka, INS_sve_brka, INS_sve_brka, INS_sve_brka, INS_sve_brka, INS_sve_brka, INS_sve_brka, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, CreateBreakAfterPropagateMask, -1, 3, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, CreateBreakBeforeMask, -1, 2, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, CreateBreakBeforePropagateMask, -1, 3, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, CreateBreakPropagateMask, -1, -1, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_HasRMWSemantics|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, CreateFalseMaskByte, -1, 0, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateFalseMaskDouble, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateFalseMaskInt16, -1, 0, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateFalseMaskInt32, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateFalseMaskInt64, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateFalseMaskSByte, -1, 0, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateFalseMaskSingle, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateFalseMaskUInt16, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateFalseMaskUInt32, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateFalseMaskUInt64, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateMaskForFirstActiveElement, -1, 2, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, CreateMaskForNextActiveElement, -1, 2, INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, CreateTrueMaskByte, -1, 1, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateTrueMaskDouble, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateTrueMaskInt16, -1, 1, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateTrueMaskInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateTrueMaskInt64, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateTrueMaskSByte, -1, 1, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateTrueMaskSingle, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateTrueMaskUInt16, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateTrueMaskUInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateTrueMaskUInt64, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskByte, -1, 2, INS_invalid, INS_sve_whilelt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskDouble, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilelt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_whilelt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilelt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilelt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskSByte, -1, 2, INS_sve_whilelt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskSingle, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilelt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilelt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilelt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilelt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskByte, -1, 2, INS_invalid, INS_sve_whilele, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskDouble, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilele, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_whilele, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilele, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilele, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskSByte, -1, 2, INS_sve_whilele, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskSingle, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilele, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilele, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilele, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilele, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, Divide, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sdiv, INS_sve_udiv, INS_sve_sdiv, INS_sve_udiv, INS_sve_fdiv, INS_sve_fdiv, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, DotProduct, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sdot, INS_sve_udot, INS_sve_sdot, INS_sve_udot, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, DotProductBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sdot, INS_sve_udot, INS_sve_sdot, INS_sve_udot, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve, DuplicateSelectedScalarToVector, -1, 2, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve, ExtractAfterLastActiveElement, -1, 2, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, ExtractAfterLastActiveElementScalar, 0, 2, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, ExtractLastActiveElement, -1, 2, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, ExtractLastActiveElementScalar, 0, 2, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, ExtractVector, -1, 3, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, FloatingPointExponentialAccelerator, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fexpa, INS_invalid, INS_sve_fexpa, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Sve, FusedMultiplyAdd, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmla, INS_sve_fmla, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, FusedMultiplyAddBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmla, INS_sve_fmla, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_FmaIntrinsic|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve, FusedMultiplyAddNegated, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fnmla, INS_sve_fnmla, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, FusedMultiplySubtract, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmls, INS_sve_fmls, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, FusedMultiplySubtractBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmls, INS_sve_fmls, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_FmaIntrinsic|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve, FusedMultiplySubtractNegated, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fnmls, INS_sve_fnmls, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, GatherPrefetch16Bit, -1, -1, INS_invalid, INS_invalid, INS_sve_prfh, INS_sve_prfh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherPrefetch32Bit, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_prfw, INS_sve_prfw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherPrefetch64Bit, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_prfd, INS_sve_prfd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherPrefetch8Bit, -1, -1, INS_sve_prfb, INS_sve_prfb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherVector, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1d, INS_sve_ld1d, INS_sve_ld1w, INS_sve_ld1d, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, GatherVectorByteZeroExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_sve_ld1b, INS_sve_ld1b, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, GatherVectorByteZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherVectorFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_sve_ldff1d, INS_sve_ldff1d, INS_sve_ldff1w, INS_sve_ldff1d, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherVectorInt16SignExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_sve_ld1sh, INS_sve_ld1sh, INS_sve_ld1sh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, GatherVectorInt16SignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherVectorInt16WithByteOffsetsSignExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_sve_ld1sh, INS_sve_ld1sh, INS_sve_ld1sh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, GatherVectorInt16WithByteOffsetsSignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherVectorInt32SignExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sw, INS_sve_ld1sw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, GatherVectorInt32SignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sw, INS_sve_ldff1sw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherVectorInt32WithByteOffsetsSignExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sw, INS_sve_ld1sw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, GatherVectorInt32WithByteOffsetsSignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sw, INS_sve_ldff1sw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherVectorSByteSignExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_sve_ld1sb, INS_sve_ld1sb, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, GatherVectorSByteSignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherVectorUInt16WithByteOffsetsZeroExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_sve_ld1h, INS_sve_ld1h, INS_sve_ld1h, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, GatherVectorUInt16WithByteOffsetsZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherVectorUInt16ZeroExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_sve_ld1h, INS_sve_ld1h, INS_sve_ld1h, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, GatherVectorUInt16ZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherVectorUInt32WithByteOffsetsZeroExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, GatherVectorUInt32WithByteOffsetsZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherVectorUInt32ZeroExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, GatherVectorUInt32ZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherVectorWithByteOffsetFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_sve_ldff1d, INS_sve_ldff1d, INS_sve_ldff1w, INS_sve_ldff1d, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GatherVectorWithByteOffsets, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1d, INS_sve_ld1d, INS_sve_ld1w, INS_sve_ld1d, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, GetActiveElementCount, -1, 2, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation) -HARDWARE_INTRINSIC(Sve, GetFfrByte, -1, 0, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GetFfrDouble, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GetFfrInt16, -1, 0, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GetFfrInt32, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GetFfrInt64, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GetFfrSByte, -1, 0, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GetFfrSingle, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GetFfrUInt16, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GetFfrUInt32, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, GetFfrUInt64, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, InsertIntoShiftedVector, -1, 2, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, LeadingSignCount, -1, -1, INS_sve_cls, INS_invalid, INS_sve_cls, INS_invalid, INS_sve_cls, INS_invalid, INS_sve_cls, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, LeadingZeroCount, -1, -1, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, Load2xVectorAndUnzip, -1, 2, INS_sve_ld2b, INS_sve_ld2b, INS_sve_ld2h, INS_sve_ld2h, INS_sve_ld2w, INS_sve_ld2w, INS_sve_ld2d, INS_sve_ld2d, INS_sve_ld2w, INS_sve_ld2d, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Sve, Load3xVectorAndUnzip, -1, 2, INS_sve_ld3b, INS_sve_ld3b, INS_sve_ld3h, INS_sve_ld3h, INS_sve_ld3w, INS_sve_ld3w, INS_sve_ld3d, INS_sve_ld3d, INS_sve_ld3w, INS_sve_ld3d, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Sve, Load4xVectorAndUnzip, -1, 2, INS_sve_ld4b, INS_sve_ld4b, INS_sve_ld4h, INS_sve_ld4h, INS_sve_ld4w, INS_sve_ld4w, INS_sve_ld4d, INS_sve_ld4d, INS_sve_ld4w, INS_sve_ld4d, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Sve, LoadVector, -1, 2, INS_sve_ld1b, INS_sve_ld1b, INS_sve_ld1h, INS_sve_ld1h, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1d, INS_sve_ld1d, INS_sve_ld1w, INS_sve_ld1d, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVector128AndReplicateToVector, -1, 2, INS_sve_ld1rqb, INS_sve_ld1rqb, INS_sve_ld1rqh, INS_sve_ld1rqh, INS_sve_ld1rqw, INS_sve_ld1rqw, INS_sve_ld1rqd, INS_sve_ld1rqd, INS_sve_ld1rqw, INS_sve_ld1rqd, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToInt16, -1, -1, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToUInt16, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToUInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToUInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorFirstFaulting, -1, -1, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1w, INS_sve_ldff1w, INS_sve_ldff1d, INS_sve_ldff1d, INS_sve_ldff1w, INS_sve_ldff1d, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToUInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToUInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorInt32NonFaultingSignExtendToInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorInt32NonFaultingSignExtendToUInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorInt32SignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sw, INS_sve_ldff1sw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorInt32SignExtendToInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorInt32SignExtendToUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorNonFaulting, -1, -1, INS_sve_ldnf1b, INS_sve_ldnf1b, INS_sve_ldnf1h, INS_sve_ldnf1h, INS_sve_ldnf1w, INS_sve_ldnf1w, INS_sve_ldnf1d, INS_sve_ldnf1d, INS_sve_ldnf1w, INS_sve_ldnf1d, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorNonTemporal, -1, 2, INS_sve_ldnt1b, INS_sve_ldnt1b, INS_sve_ldnt1h, INS_sve_ldnt1h, INS_sve_ldnt1w, INS_sve_ldnt1w, INS_sve_ldnt1d, INS_sve_ldnt1d, INS_sve_ldnt1w, INS_sve_ldnt1d, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToInt16, -1, -1, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToUInt16, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToUInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToUInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToUInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToUInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt32NonFaultingZeroExtendToInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1w, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt32NonFaultingZeroExtendToUInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1w, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve, Max, -1, -1, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_fmax, INS_sve_fmax, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, MaxAcross, -1, -1, INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_fmaxv, INS_sve_fmaxv, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) -HARDWARE_INTRINSIC(Sve, MaxNumber, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmaxnm, INS_sve_fmaxnm, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, MaxNumberAcross, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmaxnmv, INS_sve_fmaxnmv, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) -HARDWARE_INTRINSIC(Sve, Min, -1, -1, INS_sve_smin, INS_sve_umin, INS_sve_smin, INS_sve_umin, INS_sve_smin, INS_sve_umin, INS_sve_smin, INS_sve_umin, INS_sve_fmin, INS_sve_fmin, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, MinAcross, -1, -1, INS_sve_sminv, INS_sve_uminv, INS_sve_sminv, INS_sve_uminv, INS_sve_sminv, INS_sve_uminv, INS_sve_sminv, INS_sve_uminv, INS_sve_fminv, INS_sve_fminv, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) -HARDWARE_INTRINSIC(Sve, MinNumber, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fminnm, INS_sve_fminnm, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, MinNumberAcross, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fminnmv, INS_sve_fminnmv, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) -HARDWARE_INTRINSIC(Sve, Multiply, -1, 2, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_fmul, INS_sve_fmul, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, MultiplyAdd, -1, -1, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, MultiplyAddRotateComplex, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcmla, INS_sve_fcmla, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve, MultiplyAddRotateComplexBySelectedScalar, -1, 5, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcmla, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve, MultiplyBySelectedScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmul, INS_sve_fmul, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve, MultiplyExtended, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmulx, INS_sve_fmulx, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, MultiplySubtract, -1, -1, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, Negate, -1, -1, INS_sve_neg, INS_invalid, INS_sve_neg, INS_invalid, INS_sve_neg, INS_invalid, INS_sve_neg, INS_invalid, INS_sve_fneg, INS_sve_fneg, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, Not, -1, -1, INS_sve_not, INS_sve_not, INS_sve_not, INS_sve_not, INS_sve_not, INS_sve_not, INS_sve_not, INS_sve_not, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation) -HARDWARE_INTRINSIC(Sve, Or, -1, -1, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_HasAllMaskVariant) -HARDWARE_INTRINSIC(Sve, OrAcross, -1, -1, INS_sve_orv, INS_sve_orv, INS_sve_orv, INS_sve_orv, INS_sve_orv, INS_sve_orv, INS_sve_orv, INS_sve_orv, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) -HARDWARE_INTRINSIC(Sve, PopCount, -1, -1, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, Prefetch16Bit, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_sve_prfh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, Prefetch32Bit, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_prfw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, Prefetch64Bit, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_prfd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, Prefetch8Bit, -1, 3, INS_invalid, INS_sve_prfb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(Sve, ReciprocalEstimate, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frecpe, INS_sve_frecpe, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve, ReciprocalExponent, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frecpx, INS_sve_frecpx, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, ReciprocalSqrtEstimate, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frsqrte, INS_sve_frsqrte, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve, ReciprocalSqrtStep, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frsqrts, INS_sve_frsqrts, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve, ReciprocalStep, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frecps, INS_sve_frecps, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve, ReverseBits, -1, -1, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, ReverseElement, -1, 1, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant) -HARDWARE_INTRINSIC(Sve, ReverseElement16, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_revh, INS_sve_revh, INS_sve_revh, INS_sve_revh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, ReverseElement32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_revw, INS_sve_revw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, ReverseElement8, -1, -1, INS_invalid, INS_invalid, INS_sve_revb, INS_sve_revb, INS_sve_revb, INS_sve_revb, INS_sve_revb, INS_sve_revb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, RoundAwayFromZero, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frinta, INS_sve_frinta, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, RoundToNearest, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frintn, INS_sve_frintn, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, RoundToNegativeInfinity, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frintm, INS_sve_frintm, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, RoundToPositiveInfinity, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frintp, INS_sve_frintp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, RoundToZero, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frintz, INS_sve_frintz, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, SaturatingDecrementBy16BitElementCount, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdech, INS_sve_uqdech, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SaturatingDecrementBy32BitElementCount, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdecw, INS_sve_uqdecw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SaturatingDecrementBy64BitElementCount, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdecd, INS_sve_uqdecd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SaturatingDecrementBy8BitElementCount, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdecb, INS_sve_uqdecb, INS_sve_sqdecb, INS_sve_uqdecb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SaturatingDecrementByActiveElementCount, -1, 2, INS_invalid, INS_sve_sqdecp, INS_sve_sqdecp, INS_sve_sqdecp, INS_sve_sqdecp, INS_sve_sqdecp, INS_sve_sqdecp, INS_sve_sqdecp, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy16BitElementCount, -1, 3, INS_invalid, INS_invalid, INS_sve_sqinch, INS_sve_uqinch, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy32BitElementCount, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqincw, INS_sve_uqincw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy64BitElementCount, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqincd, INS_sve_uqincd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy8BitElementCount, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqincb, INS_sve_uqincb, INS_sve_sqincb, INS_sve_uqincb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SaturatingIncrementByActiveElementCount, -1, 2, INS_invalid, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, Scale, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fscale, INS_sve_fscale, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, Scatter, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1w, INS_sve_st1w, INS_sve_st1d, INS_sve_st1d, INS_sve_st1w, INS_sve_st1d, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, Scatter16BitNarrowing, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1h, INS_sve_st1h, INS_sve_st1h, INS_sve_st1h, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, Scatter16BitWithByteOffsetsNarrowing, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1h, INS_sve_st1h, INS_sve_st1h, INS_sve_st1h, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, Scatter32BitNarrowing, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1w, INS_sve_st1w, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, Scatter32BitWithByteOffsetsNarrowing, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1w, INS_sve_st1w, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, Scatter8BitNarrowing, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1b, INS_sve_st1b, INS_sve_st1b, INS_sve_st1b, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, Scatter8BitWithByteOffsetsNarrowing, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1b, INS_sve_st1b, INS_sve_st1b, INS_sve_st1b, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, ScatterWithByteOffsets, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1w, INS_sve_st1w, INS_sve_st1d, INS_sve_st1d, INS_sve_st1w, INS_sve_st1d, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, SetFfr, -1, 1, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialSideEffect_Other|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, ShiftLeftLogical, -1, -1, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, ShiftRightArithmetic, -1, -1, INS_sve_asr, INS_invalid, INS_sve_asr, INS_invalid, INS_sve_asr, INS_invalid, INS_sve_asr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, ShiftRightArithmeticForDivide, -1, -1, INS_sve_asrd, INS_invalid, INS_sve_asrd, INS_invalid, INS_sve_asrd, INS_invalid, INS_sve_asrd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve, ShiftRightLogical, -1, -1, INS_invalid, INS_sve_lsr, INS_invalid, INS_sve_lsr, INS_invalid, INS_sve_lsr, INS_invalid, INS_sve_lsr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SignExtend16, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sxth, INS_invalid, INS_sve_sxth, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, SignExtend32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sxtw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, SignExtend8, -1, -1, INS_invalid, INS_invalid, INS_sve_sxtb, INS_invalid, INS_sve_sxtb, INS_invalid, INS_sve_sxtb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, SignExtendWideningLower, -1, 1, INS_sve_sunpklo, INS_invalid, INS_sve_sunpklo, INS_invalid, INS_sve_sunpklo, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Sve, SignExtendWideningUpper, -1, 1, INS_sve_sunpkhi, INS_invalid, INS_sve_sunpkhi, INS_invalid, INS_sve_sunpkhi, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Sve, Splice, -1, 3, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, Sqrt, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fsqrt, INS_sve_fsqrt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, StoreAndZip, -1, 3, INS_sve_st1b, INS_sve_st1b, INS_sve_st1h, INS_sve_st1h, INS_sve_st1w, INS_sve_st1w, INS_sve_st1d, INS_sve_st1d, INS_sve_st1w, INS_sve_st1d, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, StoreNarrowing, -1, 3, INS_sve_st1b, INS_sve_st1b, INS_sve_st1h, INS_sve_st1h, INS_sve_st1w, INS_sve_st1w, INS_sve_st1d, INS_sve_st1d, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, StoreNonTemporal, -1, 3, INS_sve_stnt1b, INS_sve_stnt1b, INS_sve_stnt1h, INS_sve_stnt1h, INS_sve_stnt1w, INS_sve_stnt1w, INS_sve_stnt1d, INS_sve_stnt1d, INS_sve_stnt1w, INS_sve_stnt1d, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, Subtract, -1, 2, INS_sve_sub, INS_sve_sub, INS_sve_sub, INS_sve_sub, INS_sve_sub, INS_sve_sub, INS_sve_sub, INS_sve_sub, INS_sve_fsub, INS_sve_fsub, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, SubtractSaturate, -1, 2, INS_sve_sqsub, INS_sve_uqsub, INS_sve_sqsub, INS_sve_uqsub, INS_sve_sqsub, INS_sve_uqsub, INS_sve_sqsub, INS_sve_uqsub, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve, TestAnyTrue, -1, 2, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, TestFirstTrue, -1, 2, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, TestLastTrue, -1, 2, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, TransposeEven, -1, 2, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant) -HARDWARE_INTRINSIC(Sve, TransposeOdd, -1, 2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant) -HARDWARE_INTRINSIC(Sve, TrigonometricMultiplyAddCoefficient, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ftmad, INS_sve_ftmad, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, TrigonometricSelectCoefficient, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ftssel, INS_sve_ftssel, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Sve, TrigonometricStartingValue, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ftsmul, INS_sve_ftsmul, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Sve, UnzipEven, -1, 2, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant) -HARDWARE_INTRINSIC(Sve, UnzipOdd, -1, 2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant) -HARDWARE_INTRINSIC(Sve, VectorTableLookup, -1, 2, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve, Xor, -1, -1, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_HasAllMaskVariant) -HARDWARE_INTRINSIC(Sve, XorAcross, -1, -1, INS_sve_eorv, INS_sve_eorv, INS_sve_eorv, INS_sve_eorv, INS_sve_eorv, INS_sve_eorv, INS_sve_eorv, INS_sve_eorv, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) -HARDWARE_INTRINSIC(Sve, ZeroExtend16, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_uxth, INS_invalid, INS_sve_uxth, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, ZeroExtend32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_uxtw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, ZeroExtend8, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_sve_uxtb, INS_invalid, INS_sve_uxtb, INS_invalid, INS_sve_uxtb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, ZeroExtendWideningLower, -1, 1, INS_invalid, INS_sve_uunpklo, INS_invalid, INS_sve_uunpklo, INS_invalid, INS_sve_uunpklo, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Sve, ZeroExtendWideningUpper, -1, 1, INS_invalid, INS_sve_uunpkhi, INS_invalid, INS_sve_uunpkhi, INS_invalid, INS_sve_uunpkhi, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Sve, ZipHigh, -1, 2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant) -HARDWARE_INTRINSIC(Sve, ZipLow, -1, 2, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant) +HARDWARE_INTRINSIC(Sve, Abs, -1, -1, INS_sve_abs, INS_invalid, INS_sve_abs, INS_invalid, INS_sve_abs, INS_invalid, INS_sve_abs, INS_invalid, INS_sve_fabs, INS_sve_fabs, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, AbsoluteCompareGreaterThan, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_facgt, INS_sve_facgt, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, AbsoluteCompareGreaterThanOrEqual, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_facge, INS_sve_facge, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, AbsoluteCompareLessThan, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_faclt, INS_sve_faclt, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, AbsoluteCompareLessThanOrEqual, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_facle, INS_sve_facle, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, AbsoluteDifference, -1, -1, INS_sve_sabd, INS_sve_uabd, INS_sve_sabd, INS_sve_uabd, INS_sve_sabd, INS_sve_uabd, INS_sve_sabd, INS_sve_uabd, INS_sve_fabd, INS_sve_fabd, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, Add, -1, -1, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_fadd, INS_sve_fadd, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, AddAcross, -1, 1, INS_sve_saddv, INS_sve_uaddv, INS_sve_saddv, INS_sve_uaddv, INS_sve_saddv, INS_sve_uaddv, INS_sve_uaddv, INS_sve_uaddv, INS_sve_faddv, INS_sve_faddv, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) +HARDWARE_INTRINSIC(Sve, AddRotateComplex, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcadd, INS_sve_fcadd, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve, AddSaturate, -1, 2, INS_sve_sqadd, INS_sve_uqadd, INS_sve_sqadd, INS_sve_uqadd, INS_sve_sqadd, INS_sve_uqadd, INS_sve_sqadd, INS_sve_uqadd, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve, AddSequentialAcross, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fadda, INS_sve_fadda, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_ReduceOperation) +HARDWARE_INTRINSIC(Sve, And, -1, -1, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_HasAllMaskVariant) +HARDWARE_INTRINSIC(Sve, AndAcross, -1, -1, INS_sve_andv, INS_sve_andv, INS_sve_andv, INS_sve_andv, INS_sve_andv, INS_sve_andv, INS_sve_andv, INS_sve_andv, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) +HARDWARE_INTRINSIC(Sve, BitwiseClear, -1, -1, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_HasAllMaskVariant) +HARDWARE_INTRINSIC(Sve, BooleanNot, -1, -1, INS_sve_cnot, INS_sve_cnot, INS_sve_cnot, INS_sve_cnot, INS_sve_cnot, INS_sve_cnot, INS_sve_cnot, INS_sve_cnot, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, Compact, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_compact, INS_sve_compact, INS_sve_compact, INS_sve_compact, INS_sve_compact, INS_sve_compact, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, CompareEqual, -1, -1, INS_sve_cmpeq, INS_sve_cmpeq, INS_sve_cmpeq, INS_sve_cmpeq, INS_sve_cmpeq, INS_sve_cmpeq, INS_sve_cmpeq, INS_sve_cmpeq, INS_sve_fcmeq, INS_sve_fcmeq, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, CompareGreaterThan, -1, -1, INS_sve_cmpgt, INS_sve_cmphi, INS_sve_cmpgt, INS_sve_cmphi, INS_sve_cmpgt, INS_sve_cmphi, INS_sve_cmpgt, INS_sve_cmphi, INS_sve_fcmgt, INS_sve_fcmgt, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, CompareGreaterThanOrEqual, -1, -1, INS_sve_cmpge, INS_sve_cmphs, INS_sve_cmpge, INS_sve_cmphs, INS_sve_cmpge, INS_sve_cmphs, INS_sve_cmpge, INS_sve_cmphs, INS_sve_fcmge, INS_sve_fcmge, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, CompareLessThan, -1, -1, INS_sve_cmplt, INS_sve_cmplo, INS_sve_cmplt, INS_sve_cmplo, INS_sve_cmplt, INS_sve_cmplo, INS_sve_cmplt, INS_sve_cmplo, INS_sve_fcmlt, INS_sve_fcmlt, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, CompareLessThanOrEqual, -1, -1, INS_sve_cmple, INS_sve_cmpls, INS_sve_cmple, INS_sve_cmpls, INS_sve_cmple, INS_sve_cmpls, INS_sve_cmple, INS_sve_cmpls, INS_sve_fcmle, INS_sve_fcmle, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, CompareNotEqualTo, -1, -1, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_fcmne, INS_sve_fcmne, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, CompareUnordered, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcmuo, INS_sve_fcmuo, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, Compute16BitAddresses, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_adr, INS_invalid, INS_sve_adr, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, Compute32BitAddresses, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_adr, INS_invalid, INS_sve_adr, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, Compute64BitAddresses, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_adr, INS_invalid, INS_sve_adr, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, Compute8BitAddresses, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_adr, INS_invalid, INS_sve_adr, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, ConditionalExtractAfterLastActiveElement, -1, 3, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, ConditionalExtractAfterLastActiveElementAndReplicate, -1, 3, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, ConditionalExtractLastActiveElement, -1, 3, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, ConditionalExtractLastActiveElementAndReplicate, -1, 3, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, ConditionalSelect, -1, 3, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_SupportsContainment|HW_Flag_HasAllMaskVariant) +HARDWARE_INTRINSIC(Sve, ConvertToDouble, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_scvtf, INS_sve_ucvtf, INS_sve_scvtf, INS_sve_ucvtf, INS_sve_fcvt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ConvertToInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtzs, INS_sve_fcvtzs, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ConvertToInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtzs, INS_sve_fcvtzs, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ConvertToSingle, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_scvtf, INS_sve_ucvtf, INS_sve_scvtf, INS_sve_ucvtf, INS_invalid, INS_sve_fcvt, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ConvertToUInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtzu, INS_sve_fcvtzu, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ConvertToUInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtzu, INS_sve_fcvtzu, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, Count16BitElements, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_cnth, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(Sve, Count32BitElements, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_cntw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(Sve, Count64BitElements, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_cntd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(Sve, Count8BitElements, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_cntb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(Sve, CreateBreakAfterMask, -1, 2, INS_sve_brka, INS_sve_brka, INS_sve_brka, INS_sve_brka, INS_sve_brka, INS_sve_brka, INS_sve_brka, INS_sve_brka, INS_sve_brka, INS_sve_brka, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, CreateBreakAfterPropagateMask, -1, 3, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, CreateBreakBeforeMask, -1, 2, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, CreateBreakBeforePropagateMask, -1, 3, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, CreateBreakPropagateMask, -1, -1, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_HasRMWSemantics|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, CreateFalseMaskByte, -1, 0, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateFalseMaskDouble, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateFalseMaskInt16, -1, 0, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateFalseMaskInt32, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateFalseMaskInt64, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateFalseMaskSByte, -1, 0, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateFalseMaskSingle, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateFalseMaskUInt16, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateFalseMaskUInt32, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateFalseMaskUInt64, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateMaskForFirstActiveElement, -1, 2, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, CreateMaskForNextActiveElement, -1, 2, INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, CreateTrueMaskByte, -1, 1, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateTrueMaskDouble, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateTrueMaskInt16, -1, 1, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateTrueMaskInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateTrueMaskInt64, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateTrueMaskSByte, -1, 1, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateTrueMaskSingle, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateTrueMaskUInt16, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateTrueMaskUInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateTrueMaskUInt64, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskByte, -1, 2, INS_invalid, INS_sve_whilelt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskDouble, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilelt, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_whilelt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilelt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilelt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskSByte, -1, 2, INS_sve_whilelt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskSingle, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilelt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilelt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilelt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanMaskUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilelt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskByte, -1, 2, INS_invalid, INS_sve_whilele, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskDouble, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilele, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_whilele, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilele, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilele, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskSByte, -1, 2, INS_sve_whilele, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskSingle, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilele, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilele, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilele, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMaskUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilele, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, Divide, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sdiv, INS_sve_udiv, INS_sve_sdiv, INS_sve_udiv, INS_sve_fdiv, INS_sve_fdiv, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, DotProduct, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sdot, INS_sve_udot, INS_sve_sdot, INS_sve_udot, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, DotProductBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sdot, INS_sve_udot, INS_sve_sdot, INS_sve_udot, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve, DuplicateSelectedScalarToVector, -1, 2, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve, ExtractAfterLastActiveElement, -1, 2, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, ExtractAfterLastActiveElementScalar, 0, 2, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, ExtractLastActiveElement, -1, 2, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, ExtractLastActiveElementScalar, 0, 2, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, ExtractVector, -1, 3, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, FloatingPointExponentialAccelerator, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fexpa, INS_invalid, INS_sve_fexpa, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Sve, FusedMultiplyAdd, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmla, INS_sve_fmla, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, FusedMultiplyAddBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmla, INS_sve_fmla, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_FmaIntrinsic|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve, FusedMultiplyAddNegated, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fnmla, INS_sve_fnmla, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, FusedMultiplySubtract, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmls, INS_sve_fmls, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, FusedMultiplySubtractBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmls, INS_sve_fmls, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_FmaIntrinsic|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve, FusedMultiplySubtractNegated, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fnmls, INS_sve_fnmls, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, GatherPrefetch16Bit, -1, -1, INS_invalid, INS_invalid, INS_sve_prfh, INS_sve_prfh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherPrefetch32Bit, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_prfw, INS_sve_prfw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherPrefetch64Bit, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_prfd, INS_sve_prfd, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherPrefetch8Bit, -1, -1, INS_sve_prfb, INS_sve_prfb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherVector, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1d, INS_sve_ld1d, INS_sve_ld1w, INS_sve_ld1d, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, GatherVectorByteZeroExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_sve_ld1b, INS_sve_ld1b, INS_sve_ld1b, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, GatherVectorByteZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherVectorFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_sve_ldff1d, INS_sve_ldff1d, INS_sve_ldff1w, INS_sve_ldff1d, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherVectorInt16SignExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_sve_ld1sh, INS_sve_ld1sh, INS_sve_ld1sh, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, GatherVectorInt16SignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherVectorInt16WithByteOffsetsSignExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_sve_ld1sh, INS_sve_ld1sh, INS_sve_ld1sh, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, GatherVectorInt16WithByteOffsetsSignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherVectorInt32SignExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sw, INS_sve_ld1sw, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, GatherVectorInt32SignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sw, INS_sve_ldff1sw, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherVectorInt32WithByteOffsetsSignExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sw, INS_sve_ld1sw, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, GatherVectorInt32WithByteOffsetsSignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sw, INS_sve_ldff1sw, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherVectorSByteSignExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_sve_ld1sb, INS_sve_ld1sb, INS_sve_ld1sb, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, GatherVectorSByteSignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherVectorUInt16WithByteOffsetsZeroExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_sve_ld1h, INS_sve_ld1h, INS_sve_ld1h, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, GatherVectorUInt16WithByteOffsetsZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherVectorUInt16ZeroExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_sve_ld1h, INS_sve_ld1h, INS_sve_ld1h, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, GatherVectorUInt16ZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherVectorUInt32WithByteOffsetsZeroExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, GatherVectorUInt32WithByteOffsetsZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherVectorUInt32ZeroExtend, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, GatherVectorUInt32ZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherVectorWithByteOffsetFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_sve_ldff1d, INS_sve_ldff1d, INS_sve_ldff1w, INS_sve_ldff1d, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GatherVectorWithByteOffsets, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1d, INS_sve_ld1d, INS_sve_ld1w, INS_sve_ld1d, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, GetActiveElementCount, -1, 2, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation) +HARDWARE_INTRINSIC(Sve, GetFfrByte, -1, 0, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GetFfrDouble, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GetFfrInt16, -1, 0, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GetFfrInt32, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GetFfrInt64, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GetFfrSByte, -1, 0, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GetFfrSingle, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GetFfrUInt16, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GetFfrUInt32, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, GetFfrUInt64, -1, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, InsertIntoShiftedVector, -1, 2, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, LeadingSignCount, -1, -1, INS_sve_cls, INS_invalid, INS_sve_cls, INS_invalid, INS_sve_cls, INS_invalid, INS_sve_cls, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, LeadingZeroCount, -1, -1, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, Load2xVectorAndUnzip, -1, 2, INS_sve_ld2b, INS_sve_ld2b, INS_sve_ld2h, INS_sve_ld2h, INS_sve_ld2w, INS_sve_ld2w, INS_sve_ld2d, INS_sve_ld2d, INS_sve_ld2w, INS_sve_ld2d, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Sve, Load3xVectorAndUnzip, -1, 2, INS_sve_ld3b, INS_sve_ld3b, INS_sve_ld3h, INS_sve_ld3h, INS_sve_ld3w, INS_sve_ld3w, INS_sve_ld3d, INS_sve_ld3d, INS_sve_ld3w, INS_sve_ld3d, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Sve, Load4xVectorAndUnzip, -1, 2, INS_sve_ld4b, INS_sve_ld4b, INS_sve_ld4h, INS_sve_ld4h, INS_sve_ld4w, INS_sve_ld4w, INS_sve_ld4d, INS_sve_ld4d, INS_sve_ld4w, INS_sve_ld4d, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Sve, LoadVector, -1, 2, INS_sve_ld1b, INS_sve_ld1b, INS_sve_ld1h, INS_sve_ld1h, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1d, INS_sve_ld1d, INS_sve_ld1w, INS_sve_ld1d, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVector128AndReplicateToVector, -1, 2, INS_sve_ld1rqb, INS_sve_ld1rqb, INS_sve_ld1rqh, INS_sve_ld1rqh, INS_sve_ld1rqw, INS_sve_ld1rqw, INS_sve_ld1rqd, INS_sve_ld1rqd, INS_sve_ld1rqw, INS_sve_ld1rqd, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToInt16, -1, -1, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToUInt16, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToUInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToUInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorFirstFaulting, -1, -1, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1w, INS_sve_ldff1w, INS_sve_ldff1d, INS_sve_ldff1d, INS_sve_ldff1w, INS_sve_ldff1d, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToUInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToUInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorInt32NonFaultingSignExtendToInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorInt32NonFaultingSignExtendToUInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sw, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorInt32SignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sw, INS_sve_ldff1sw, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorInt32SignExtendToInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorInt32SignExtendToUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sw, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorNonFaulting, -1, -1, INS_sve_ldnf1b, INS_sve_ldnf1b, INS_sve_ldnf1h, INS_sve_ldnf1h, INS_sve_ldnf1w, INS_sve_ldnf1w, INS_sve_ldnf1d, INS_sve_ldnf1d, INS_sve_ldnf1w, INS_sve_ldnf1d, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorNonTemporal, -1, 2, INS_sve_ldnt1b, INS_sve_ldnt1b, INS_sve_ldnt1h, INS_sve_ldnt1h, INS_sve_ldnt1w, INS_sve_ldnt1w, INS_sve_ldnt1d, INS_sve_ldnt1d, INS_sve_ldnt1w, INS_sve_ldnt1d, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToInt16, -1, -1, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToUInt16, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToUInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToUInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToUInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToUInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt32NonFaultingZeroExtendToInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1w, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt32NonFaultingZeroExtendToUInt64, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1w, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendFirstFaulting, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve, Max, -1, -1, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_fmax, INS_sve_fmax, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, MaxAcross, -1, -1, INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_smaxv, INS_sve_umaxv, INS_sve_fmaxv, INS_sve_fmaxv, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) +HARDWARE_INTRINSIC(Sve, MaxNumber, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmaxnm, INS_sve_fmaxnm, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, MaxNumberAcross, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmaxnmv, INS_sve_fmaxnmv, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) +HARDWARE_INTRINSIC(Sve, Min, -1, -1, INS_sve_smin, INS_sve_umin, INS_sve_smin, INS_sve_umin, INS_sve_smin, INS_sve_umin, INS_sve_smin, INS_sve_umin, INS_sve_fmin, INS_sve_fmin, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, MinAcross, -1, -1, INS_sve_sminv, INS_sve_uminv, INS_sve_sminv, INS_sve_uminv, INS_sve_sminv, INS_sve_uminv, INS_sve_sminv, INS_sve_uminv, INS_sve_fminv, INS_sve_fminv, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) +HARDWARE_INTRINSIC(Sve, MinNumber, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fminnm, INS_sve_fminnm, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, MinNumberAcross, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fminnmv, INS_sve_fminnmv, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) +HARDWARE_INTRINSIC(Sve, Multiply, -1, 2, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_fmul, INS_sve_fmul, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, MultiplyAdd, -1, -1, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, MultiplyAddRotateComplex, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcmla, INS_sve_fcmla, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve, MultiplyAddRotateComplexBySelectedScalar, -1, 5, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcmla, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve, MultiplyBySelectedScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmul, INS_sve_fmul, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve, MultiplyExtended, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmulx, INS_sve_fmulx, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, MultiplySubtract, -1, -1, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, Negate, -1, -1, INS_sve_neg, INS_invalid, INS_sve_neg, INS_invalid, INS_sve_neg, INS_invalid, INS_sve_neg, INS_invalid, INS_sve_fneg, INS_sve_fneg, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, Not, -1, -1, INS_sve_not, INS_sve_not, INS_sve_not, INS_sve_not, INS_sve_not, INS_sve_not, INS_sve_not, INS_sve_not, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation) +HARDWARE_INTRINSIC(Sve, Or, -1, -1, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_HasAllMaskVariant) +HARDWARE_INTRINSIC(Sve, OrAcross, -1, -1, INS_sve_orv, INS_sve_orv, INS_sve_orv, INS_sve_orv, INS_sve_orv, INS_sve_orv, INS_sve_orv, INS_sve_orv, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) +HARDWARE_INTRINSIC(Sve, PopCount, -1, -1, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, Prefetch16Bit, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_sve_prfh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, Prefetch32Bit, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_prfw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, Prefetch64Bit, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_prfd, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, Prefetch8Bit, -1, 3, INS_invalid, INS_sve_prfb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Sve, ReciprocalEstimate, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frecpe, INS_sve_frecpe, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve, ReciprocalExponent, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frecpx, INS_sve_frecpx, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ReciprocalSqrtEstimate, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frsqrte, INS_sve_frsqrte, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve, ReciprocalSqrtStep, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frsqrts, INS_sve_frsqrts, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve, ReciprocalStep, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frecps, INS_sve_frecps, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve, ReverseBits, -1, -1, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ReverseElement, -1, 1, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant) +HARDWARE_INTRINSIC(Sve, ReverseElement16, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_revh, INS_sve_revh, INS_sve_revh, INS_sve_revh, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ReverseElement32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_revw, INS_sve_revw, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ReverseElement8, -1, -1, INS_invalid, INS_invalid, INS_sve_revb, INS_sve_revb, INS_sve_revb, INS_sve_revb, INS_sve_revb, INS_sve_revb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, RoundAwayFromZero, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frinta, INS_sve_frinta, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, RoundToNearest, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frintn, INS_sve_frintn, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, RoundToNegativeInfinity, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frintm, INS_sve_frintm, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, RoundToPositiveInfinity, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frintp, INS_sve_frintp, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, RoundToZero, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frintz, INS_sve_frintz, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, SaturatingDecrementBy16BitElementCount, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdech, INS_sve_uqdech, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, SaturatingDecrementBy32BitElementCount, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdecw, INS_sve_uqdecw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, SaturatingDecrementBy64BitElementCount, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdecd, INS_sve_uqdecd, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, SaturatingDecrementBy8BitElementCount, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdecb, INS_sve_uqdecb, INS_sve_sqdecb, INS_sve_uqdecb, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, SaturatingDecrementByActiveElementCount, -1, 2, INS_invalid, INS_sve_sqdecp, INS_sve_sqdecp, INS_sve_sqdecp, INS_sve_sqdecp, INS_sve_sqdecp, INS_sve_sqdecp, INS_sve_sqdecp, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy16BitElementCount, -1, 3, INS_invalid, INS_invalid, INS_sve_sqinch, INS_sve_uqinch, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy32BitElementCount, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqincw, INS_sve_uqincw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy64BitElementCount, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqincd, INS_sve_uqincd, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy8BitElementCount, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqincb, INS_sve_uqincb, INS_sve_sqincb, INS_sve_uqincb, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, SaturatingIncrementByActiveElementCount, -1, 2, INS_invalid, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, Scale, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fscale, INS_sve_fscale, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, Scatter, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1w, INS_sve_st1w, INS_sve_st1d, INS_sve_st1d, INS_sve_st1w, INS_sve_st1d, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, Scatter16BitNarrowing, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1h, INS_sve_st1h, INS_sve_st1h, INS_sve_st1h, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, Scatter16BitWithByteOffsetsNarrowing, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1h, INS_sve_st1h, INS_sve_st1h, INS_sve_st1h, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, Scatter32BitNarrowing, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1w, INS_sve_st1w, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, Scatter32BitWithByteOffsetsNarrowing, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1w, INS_sve_st1w, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, Scatter8BitNarrowing, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1b, INS_sve_st1b, INS_sve_st1b, INS_sve_st1b, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, Scatter8BitWithByteOffsetsNarrowing, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1b, INS_sve_st1b, INS_sve_st1b, INS_sve_st1b, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ScatterWithByteOffsets, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1w, INS_sve_st1w, INS_sve_st1d, INS_sve_st1d, INS_sve_st1w, INS_sve_st1d, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, SetFfr, -1, 1, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialSideEffect_Other|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, ShiftLeftLogical, -1, -1, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, ShiftRightArithmetic, -1, -1, INS_sve_asr, INS_invalid, INS_sve_asr, INS_invalid, INS_sve_asr, INS_invalid, INS_sve_asr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, ShiftRightArithmeticForDivide, -1, -1, INS_sve_asrd, INS_invalid, INS_sve_asrd, INS_invalid, INS_sve_asrd, INS_invalid, INS_sve_asrd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve, ShiftRightLogical, -1, -1, INS_invalid, INS_sve_lsr, INS_invalid, INS_sve_lsr, INS_invalid, INS_sve_lsr, INS_invalid, INS_sve_lsr, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, SignExtend16, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sxth, INS_invalid, INS_sve_sxth, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, SignExtend32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sxtw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, SignExtend8, -1, -1, INS_invalid, INS_invalid, INS_sve_sxtb, INS_invalid, INS_sve_sxtb, INS_invalid, INS_sve_sxtb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, SignExtendWideningLower, -1, 1, INS_sve_sunpklo, INS_invalid, INS_sve_sunpklo, INS_invalid, INS_sve_sunpklo, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Sve, SignExtendWideningUpper, -1, 1, INS_sve_sunpkhi, INS_invalid, INS_sve_sunpkhi, INS_invalid, INS_sve_sunpkhi, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Sve, Splice, -1, 3, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, Sqrt, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fsqrt, INS_sve_fsqrt, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, StoreAndZip, -1, 3, INS_sve_st1b, INS_sve_st1b, INS_sve_st1h, INS_sve_st1h, INS_sve_st1w, INS_sve_st1w, INS_sve_st1d, INS_sve_st1d, INS_sve_st1w, INS_sve_st1d, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, StoreNarrowing, -1, 3, INS_sve_st1b, INS_sve_st1b, INS_sve_st1h, INS_sve_st1h, INS_sve_st1w, INS_sve_st1w, INS_sve_st1d, INS_sve_st1d, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, StoreNonTemporal, -1, 3, INS_sve_stnt1b, INS_sve_stnt1b, INS_sve_stnt1h, INS_sve_stnt1h, INS_sve_stnt1w, INS_sve_stnt1w, INS_sve_stnt1d, INS_sve_stnt1d, INS_sve_stnt1w, INS_sve_stnt1d, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, Subtract, -1, 2, INS_sve_sub, INS_sve_sub, INS_sve_sub, INS_sve_sub, INS_sve_sub, INS_sve_sub, INS_sve_sub, INS_sve_sub, INS_sve_fsub, INS_sve_fsub, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, SubtractSaturate, -1, 2, INS_sve_sqsub, INS_sve_uqsub, INS_sve_sqsub, INS_sve_uqsub, INS_sve_sqsub, INS_sve_uqsub, INS_sve_sqsub, INS_sve_uqsub, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve, TestAnyTrue, -1, 2, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, TestFirstTrue, -1, 2, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, TestLastTrue, -1, 2, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, INS_sve_ptest, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, TransposeEven, -1, 2, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant) +HARDWARE_INTRINSIC(Sve, TransposeOdd, -1, 2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant) +HARDWARE_INTRINSIC(Sve, TrigonometricMultiplyAddCoefficient, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ftmad, INS_sve_ftmad, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, TrigonometricSelectCoefficient, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ftssel, INS_sve_ftssel, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Sve, TrigonometricStartingValue, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ftsmul, INS_sve_ftsmul, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Sve, UnzipEven, -1, 2, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant) +HARDWARE_INTRINSIC(Sve, UnzipOdd, -1, 2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant) +HARDWARE_INTRINSIC(Sve, VectorTableLookup, -1, 2, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve, Xor, -1, -1, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_HasAllMaskVariant) +HARDWARE_INTRINSIC(Sve, XorAcross, -1, -1, INS_sve_eorv, INS_sve_eorv, INS_sve_eorv, INS_sve_eorv, INS_sve_eorv, INS_sve_eorv, INS_sve_eorv, INS_sve_eorv, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) +HARDWARE_INTRINSIC(Sve, ZeroExtend16, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_uxth, INS_invalid, INS_sve_uxth, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ZeroExtend32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_uxtw, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ZeroExtend8, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_sve_uxtb, INS_invalid, INS_sve_uxtb, INS_invalid, INS_sve_uxtb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ZeroExtendWideningLower, -1, 1, INS_invalid, INS_sve_uunpklo, INS_invalid, INS_sve_uunpklo, INS_invalid, INS_sve_uunpklo, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Sve, ZeroExtendWideningUpper, -1, 1, INS_invalid, INS_sve_uunpkhi, INS_invalid, INS_sve_uunpkhi, INS_invalid, INS_sve_uunpkhi, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Sve, ZipHigh, -1, 2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant) +HARDWARE_INTRINSIC(Sve, ZipLow, -1, 2, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant) #define LAST_NI_Sve NI_Sve_ZipLow // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -328,209 +328,209 @@ HARDWARE_INTRINSIC(Sve, ZipLow, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // SVE2 Intrinsics #define FIRST_NI_Sve2 NI_Sve2_AbsSaturate -HARDWARE_INTRINSIC(Sve2, AbsSaturate, -1, -1, INS_sve_sqabs, INS_invalid, INS_sve_sqabs, INS_invalid, INS_sve_sqabs, INS_invalid, INS_sve_sqabs, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, AbsoluteDifferenceAdd, -1, 3, INS_sve_saba, INS_sve_uaba, INS_sve_saba, INS_sve_uaba, INS_sve_saba, INS_sve_uaba, INS_sve_saba, INS_sve_uaba, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, AbsoluteDifferenceWideningEven, -1, 2, INS_invalid, INS_invalid, INS_sve_sabdlb, INS_sve_uabdlb, INS_sve_sabdlb, INS_sve_uabdlb, INS_sve_sabdlb, INS_sve_uabdlb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, AbsoluteDifferenceWideningLowerAndAddEven, -1, 3, INS_invalid, INS_invalid, INS_sve_sabalb, INS_sve_uabalb, INS_sve_sabalb, INS_sve_uabalb, INS_sve_sabalb, INS_sve_uabalb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, AbsoluteDifferenceWideningLowerAndAddOdd, -1, 3, INS_invalid, INS_invalid, INS_sve_sabalt, INS_sve_uabalt, INS_sve_sabalt, INS_sve_uabalt, INS_sve_sabalt, INS_sve_uabalt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, AbsoluteDifferenceWideningOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_sabdlt, INS_sve_uabdlt, INS_sve_sabdlt, INS_sve_uabdlt, INS_sve_sabdlt, INS_sve_uabdlt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, AddCarryWideningEven, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_adclb, INS_invalid, INS_sve_adclb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, AddCarryWideningOdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_adclt, INS_invalid, INS_sve_adclt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, AddHighNarrowingEven, -1, 2, INS_sve_addhnb, INS_sve_addhnb, INS_sve_addhnb, INS_sve_addhnb, INS_sve_addhnb, INS_sve_addhnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, AddHighNarrowingOdd, -1, 3, INS_sve_addhnt, INS_sve_addhnt, INS_sve_addhnt, INS_sve_addhnt, INS_sve_addhnt, INS_sve_addhnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, AddPairwise, -1, -1, INS_sve_addp, INS_sve_addp, INS_sve_addp, INS_sve_addp, INS_sve_addp, INS_sve_addp, INS_sve_addp, INS_sve_addp, INS_sve_faddp, INS_sve_faddp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) -HARDWARE_INTRINSIC(Sve2, AddPairwiseWideningAndAdd, -1, -1, INS_invalid, INS_invalid, INS_sve_sadalp, INS_sve_uadalp, INS_sve_sadalp, INS_sve_uadalp, INS_sve_sadalp, INS_sve_uadalp, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, AddRotateComplex, -1, 3, INS_sve_cadd, INS_sve_cadd, INS_sve_cadd, INS_sve_cadd, INS_sve_cadd, INS_sve_cadd, INS_sve_cadd, INS_sve_cadd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve2, AddRoundedHighNarrowingEven, -1, 2, INS_sve_raddhnb, INS_sve_raddhnb, INS_sve_raddhnb, INS_sve_raddhnb, INS_sve_raddhnb, INS_sve_raddhnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, AddRoundedHighNarrowingOdd, -1, 3, INS_sve_raddhnt, INS_sve_raddhnt, INS_sve_raddhnt, INS_sve_raddhnt, INS_sve_raddhnt, INS_sve_raddhnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, AddSaturate, -1, -1, INS_sve_sqadd, INS_sve_uqadd, INS_sve_sqadd, INS_sve_uqadd, INS_sve_sqadd, INS_sve_uqadd, INS_sve_sqadd, INS_sve_uqadd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve2, AddSaturateRotateComplex, -1, 3, INS_sve_sqcadd, INS_invalid, INS_sve_sqcadd, INS_invalid, INS_sve_sqcadd, INS_invalid, INS_sve_sqcadd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve2, AddWideningEven, -1, 2, INS_invalid, INS_invalid, INS_sve_saddwb, INS_sve_uaddwb, INS_sve_saddwb, INS_sve_uaddwb, INS_sve_saddwb, INS_sve_uaddwb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve2, AddWideningEvenOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_saddlbt, INS_invalid, INS_sve_saddlbt, INS_invalid, INS_sve_saddlbt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, AddWideningOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_saddwt, INS_sve_uaddwt, INS_sve_saddwt, INS_sve_uaddwt, INS_sve_saddwt, INS_sve_uaddwt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve2, BitwiseClearXor, -1, 3, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, BitwiseSelect, -1, 3, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, BitwiseSelectLeftInverted, -1, 3, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, BitwiseSelectRightInverted, -1, 3, INS_sve_bsl2n, INS_sve_bsl2n, INS_sve_bsl2n, INS_sve_bsl2n, INS_sve_bsl2n, INS_sve_bsl2n, INS_sve_bsl2n, INS_sve_bsl2n, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ConvertToDoubleOdd, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtlt, INS_sve_fcvtlt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, ConvertToSingleEvenRoundToOdd, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtx, INS_sve_fcvtx, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, ConvertToSingleOdd, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtnt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ConvertToSingleOddRoundToOdd, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtxnt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, CountMatchingElements, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_histcnt, INS_sve_histcnt, INS_sve_histcnt, INS_sve_histcnt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve2, CountMatchingElementsIn128BitSegments, -1, 2, INS_sve_histseg, INS_sve_histseg, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskByte, -1, 2, INS_invalid, INS_sve_whilegt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskDouble, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilegt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_whilegt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilegt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilegt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskSByte, -1, 2, INS_sve_whilegt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskSingle, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilegt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilegt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilegt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilegt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskByte, -1, 2, INS_invalid, INS_sve_whilege, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskDouble, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilege, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_whilege, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilege, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilege, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskSByte, -1, 2, INS_sve_whilege, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskSingle, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilege, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilege, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilege, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilege, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskByte, -1, 2, INS_invalid, INS_sve_whilerw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskDouble, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilerw, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_whilerw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilerw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilerw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskSByte, -1, 2, INS_sve_whilerw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskSingle, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilerw, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilerw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilerw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilerw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskByte, -1, 2, INS_invalid, INS_sve_whilewr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskDouble, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilewr, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_whilewr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilewr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilewr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskSByte, -1, 2, INS_sve_whilewr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskSingle, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilewr, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilewr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilewr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilewr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve2, DotProductRotateComplex, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_cdot, INS_invalid, INS_sve_cdot, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve2, DotProductRotateComplexBySelectedIndex, -1, 5, INS_sve_cdot, INS_invalid, INS_sve_cdot, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(Sve2, FusedAddHalving, -1, -1, INS_sve_shadd, INS_sve_uhadd, INS_sve_shadd, INS_sve_uhadd, INS_sve_shadd, INS_sve_uhadd, INS_sve_shadd, INS_sve_uhadd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, FusedAddRoundedHalving, -1, -1, INS_sve_srhadd, INS_sve_urhadd, INS_sve_srhadd, INS_sve_urhadd, INS_sve_srhadd, INS_sve_urhadd, INS_sve_srhadd, INS_sve_urhadd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, FusedSubtractHalving, -1, -1, INS_sve_shsub, INS_sve_uhsub, INS_sve_shsub, INS_sve_uhsub, INS_sve_shsub, INS_sve_uhsub, INS_sve_shsub, INS_sve_uhsub, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, GatherVectorByteZeroExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1b, INS_sve_ldnt1b, INS_sve_ldnt1b, INS_sve_ldnt1b, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve2, GatherVectorInt16SignExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1sh, INS_sve_ldnt1sh, INS_sve_ldnt1sh, INS_sve_ldnt1sh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve2, GatherVectorInt16WithByteOffsetsSignExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1sh, INS_sve_ldnt1sh, INS_sve_ldnt1sh, INS_sve_ldnt1sh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve2, GatherVectorInt32SignExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1sw, INS_sve_ldnt1sw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve2, GatherVectorInt32WithByteOffsetsSignExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1sw, INS_sve_ldnt1sw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve2, GatherVectorNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1w, INS_sve_ldnt1w, INS_sve_ldnt1d, INS_sve_ldnt1d, INS_sve_ldnt1w, INS_sve_ldnt1d, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve2, GatherVectorSByteSignExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1sb, INS_sve_ldnt1sb, INS_sve_ldnt1sb, INS_sve_ldnt1sb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve2, GatherVectorUInt16WithByteOffsetsZeroExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1h, INS_sve_ldnt1h, INS_sve_ldnt1h, INS_sve_ldnt1h, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve2, GatherVectorUInt16ZeroExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1h, INS_sve_ldnt1h, INS_sve_ldnt1h, INS_sve_ldnt1h, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve2, GatherVectorUInt32WithByteOffsetsZeroExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1w, INS_sve_ldnt1w, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve2, GatherVectorUInt32ZeroExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1w, INS_sve_ldnt1w, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve2, GatherVectorWithByteOffsetsNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1w, INS_sve_ldnt1w, INS_sve_ldnt1d, INS_sve_ldnt1d, INS_sve_ldnt1w, INS_sve_ldnt1d, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve2, InterleavingXorEvenOdd, -1, 3, INS_sve_eorbt, INS_sve_eorbt, INS_sve_eorbt, INS_sve_eorbt, INS_sve_eorbt, INS_sve_eorbt, INS_sve_eorbt, INS_sve_eorbt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, InterleavingXorOddEven, -1, 3, INS_sve_eortb, INS_sve_eortb, INS_sve_eortb, INS_sve_eortb, INS_sve_eortb, INS_sve_eortb, INS_sve_eortb, INS_sve_eortb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, Log2, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_flogb, INS_invalid, INS_sve_flogb, INS_invalid, INS_sve_flogb, INS_sve_flogb, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, Match, -1, 3, INS_sve_match, INS_sve_match, INS_sve_match, INS_sve_match, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve2, MaxNumberPairwise, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmaxnmp, INS_sve_fmaxnmp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, MaxPairwise, -1, -1, INS_sve_smaxp, INS_sve_umaxp, INS_sve_smaxp, INS_sve_umaxp, INS_sve_smaxp, INS_sve_umaxp, INS_sve_smaxp, INS_sve_umaxp, INS_sve_fmaxp, INS_sve_fmaxp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_ReduceOperation) -HARDWARE_INTRINSIC(Sve2, MinNumberPairwise, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fminnmp, INS_sve_fminnmp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, MinPairwise, -1, -1, INS_sve_sminp, INS_sve_uminp, INS_sve_sminp, INS_sve_uminp, INS_sve_sminp, INS_sve_uminp, INS_sve_sminp, INS_sve_uminp, INS_sve_fminp, INS_sve_fminp, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_ReduceOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyAddBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, MultiplyAddRotateComplex, -1, 4, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve2, MultiplyAddRotateComplexBySelectedScalar, -1, 5, INS_invalid, INS_invalid, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve2, MultiplyAddRoundedDoublingSaturateHighRotateComplex, -1, 4, INS_sve_sqrdcmlah, INS_invalid, INS_sve_sqrdcmlah, INS_invalid, INS_sve_sqrdcmlah, INS_invalid, INS_sve_sqrdcmlah, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve2, MultiplyAddRoundedDoublingSaturateHighRotateComplexBySelectedScalar, -1, 5, INS_invalid, INS_invalid, INS_sve_sqrdcmlah, INS_invalid, INS_sve_sqrdcmlah, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalar, -1, 3, INS_invalid, INS_invalid, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalarWideningEven, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_smullb, INS_sve_umullb, INS_sve_smullb, INS_sve_umullb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalarWideningEvenAndAdd, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_smlalb, INS_sve_umlalb, INS_sve_smlalb, INS_sve_umlalb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalarWideningEvenAndSubtract, -1, 4, INS_invalid, INS_invalid, INS_sve_smlslb, INS_sve_umlslb, INS_sve_smlslb, INS_sve_umlslb, INS_sve_smlslb, INS_sve_umlslb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalarWideningOdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_smullt, INS_sve_umullt, INS_sve_smullt, INS_sve_umullt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalarWideningOddAndAdd, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_smlalt, INS_sve_umlalt, INS_sve_smlalt, INS_sve_umlalt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalarWideningOddAndSubtract, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_smlslt, INS_sve_umlslt, INS_sve_smlslt, INS_sve_umlslt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingBySelectedScalarSaturateHigh, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdmulh, INS_invalid, INS_sve_sqdmulh, INS_invalid, INS_sve_sqdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingSaturateHigh, -1, 2, INS_sve_sqdmulh, INS_invalid, INS_sve_sqdmulh, INS_invalid, INS_sve_sqdmulh, INS_invalid, INS_sve_sqdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningAndAddSaturateEven, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdmlalb, INS_invalid, INS_sve_sqdmlalb, INS_invalid, INS_sve_sqdmlalb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningAndAddSaturateEvenOdd, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdmlalbt, INS_invalid, INS_sve_sqdmlalbt, INS_invalid, INS_sve_sqdmlalbt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningAndAddSaturateOdd, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdmlalt, INS_invalid, INS_sve_sqdmlalt, INS_invalid, INS_sve_sqdmlalt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningAndSubtractSaturateEven, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdmlslb, INS_invalid, INS_sve_sqdmlslb, INS_invalid, INS_sve_sqdmlslb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningAndSubtractSaturateEvenOdd, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdmlslbt, INS_invalid, INS_sve_sqdmlslbt, INS_invalid, INS_sve_sqdmlslbt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningAndSubtractSaturateOdd, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdmlslt, INS_invalid, INS_sve_sqdmlslt, INS_invalid, INS_sve_sqdmlslt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningBySelectedScalarAndAddSaturateEven, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdmlalb, INS_invalid, INS_sve_sqdmlalb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningBySelectedScalarAndAddSaturateOdd, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdmlalt, INS_invalid, INS_sve_sqdmlalt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningBySelectedScalarAndSubtractSaturateEven, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdmlslb, INS_invalid, INS_sve_sqdmlslb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningBySelectedScalarAndSubtractSaturateOdd, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdmlslt, INS_invalid, INS_sve_sqdmlslt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningSaturateEven, -1, 2, INS_invalid, INS_invalid, INS_sve_sqdmullb, INS_invalid, INS_sve_sqdmullb, INS_invalid, INS_sve_sqdmullb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningSaturateEvenBySelectedScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdmullb, INS_invalid, INS_sve_sqdmullb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningSaturateOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_sqdmullt, INS_invalid, INS_sve_sqdmullt, INS_invalid, INS_sve_sqdmullt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningSaturateOddBySelectedScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdmullt, INS_invalid, INS_sve_sqdmullt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyRoundedDoublingBySelectedScalarSaturateHigh, -1, 3, INS_invalid, INS_invalid, INS_sve_sqrdmulh, INS_invalid, INS_sve_sqrdmulh, INS_invalid, INS_sve_sqrdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyRoundedDoublingSaturateAndAddHigh, -1, 3, INS_sve_sqrdmlah, INS_invalid, INS_sve_sqrdmlah, INS_invalid, INS_sve_sqrdmlah, INS_invalid, INS_sve_sqrdmlah, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, MultiplyRoundedDoublingSaturateAndSubtractHigh, -1, 3, INS_sve_sqrdmlsh, INS_invalid, INS_sve_sqrdmlsh, INS_invalid, INS_sve_sqrdmlsh, INS_invalid, INS_sve_sqrdmlsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, MultiplyRoundedDoublingSaturateBySelectedScalarAndAddHigh, -1, 4, INS_invalid, INS_invalid, INS_sve_sqrdmlah, INS_invalid, INS_sve_sqrdmlah, INS_invalid, INS_sve_sqrdmlah, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyRoundedDoublingSaturateBySelectedScalarAndSubtractHigh, -1, 4, INS_invalid, INS_invalid, INS_sve_sqrdmlsh, INS_invalid, INS_sve_sqrdmlsh, INS_invalid, INS_sve_sqrdmlsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyRoundedDoublingSaturateHigh, -1, 2, INS_sve_sqrdmulh, INS_invalid, INS_sve_sqrdmulh, INS_invalid, INS_sve_sqrdmulh, INS_invalid, INS_sve_sqrdmulh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, MultiplySubtractBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_LowVectorOperation) -HARDWARE_INTRINSIC(Sve2, MultiplyWideningEven, -1, 2, INS_invalid, INS_invalid, INS_sve_smullb, INS_sve_umullb, INS_sve_smullb, INS_sve_umullb, INS_sve_smullb, INS_sve_umullb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, MultiplyWideningEvenAndAdd, -1, 3, INS_invalid, INS_invalid, INS_sve_smlalb, INS_sve_umlalb, INS_sve_smlalb, INS_sve_umlalb, INS_sve_smlalb, INS_sve_umlalb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, MultiplyWideningEvenAndSubtract, -1, 3, INS_invalid, INS_invalid, INS_sve_smlslb, INS_sve_umlslb, INS_sve_smlslb, INS_sve_umlslb, INS_sve_smlslb, INS_sve_umlslb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, MultiplyWideningOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_smullt, INS_sve_umullt, INS_sve_smullt, INS_sve_umullt, INS_sve_smullt, INS_sve_umullt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, MultiplyWideningOddAndAdd, -1, 3, INS_invalid, INS_invalid, INS_sve_smlalt, INS_sve_umlalt, INS_sve_smlalt, INS_sve_umlalt, INS_sve_smlalt, INS_sve_umlalt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, MultiplyWideningOddAndSubtract, -1, 3, INS_invalid, INS_invalid, INS_sve_smlslt, INS_sve_umlslt, INS_sve_smlslt, INS_sve_umlslt, INS_sve_smlslt, INS_sve_umlslt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, NegateSaturate, -1, -1, INS_sve_sqneg, INS_invalid, INS_sve_sqneg, INS_invalid, INS_sve_sqneg, INS_invalid, INS_sve_sqneg, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, NoMatch, -1, 3, INS_sve_nmatch, INS_sve_nmatch, INS_sve_nmatch, INS_sve_nmatch, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) -HARDWARE_INTRINSIC(Sve2, PolynomialMultiply, -1, 2, INS_sve_pmul, INS_sve_pmul, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, PolynomialMultiplyWideningEven, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_pmullb, INS_invalid, INS_invalid, INS_invalid, INS_sve_pmullb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, PolynomialMultiplyWideningOdd, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_pmullt, INS_invalid, INS_invalid, INS_invalid, INS_sve_pmullt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, ReciprocalEstimate, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_urecpe, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, ReciprocalSqrtEstimate, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ursqrte, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, SaturatingExtractNarrowingLower, -1, 1, INS_sve_sqxtnb, INS_sve_uqxtnb, INS_sve_sqxtnb, INS_sve_uqxtnb, INS_sve_sqxtnb, INS_sve_uqxtnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, SaturatingExtractNarrowingUpper, -1, 2, INS_sve_sqxtnt, INS_sve_uqxtnt, INS_sve_sqxtnt, INS_sve_uqxtnt, INS_sve_sqxtnt, INS_sve_uqxtnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, SaturatingExtractUnsignedNarrowingLower, -1, 1, INS_invalid, INS_sve_sqxtunb, INS_invalid, INS_sve_sqxtunb, INS_invalid, INS_sve_sqxtunb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, SaturatingExtractUnsignedNarrowingUpper, -1, 2, INS_invalid, INS_sve_sqxtunt, INS_invalid, INS_sve_sqxtunt, INS_invalid, INS_sve_sqxtunt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, Scatter16BitNarrowingNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1h, INS_sve_stnt1h, INS_sve_stnt1h, INS_sve_stnt1h, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, Scatter16BitWithByteOffsetsNarrowingNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1h, INS_sve_stnt1h, INS_sve_stnt1h, INS_sve_stnt1h, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, Scatter32BitNarrowingNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1w, INS_sve_stnt1w, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, Scatter32BitWithByteOffsetsNarrowingNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1w, INS_sve_stnt1w, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, Scatter8BitNarrowingNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1b, INS_sve_stnt1b, INS_sve_stnt1b, INS_sve_stnt1b, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, Scatter8BitWithByteOffsetsNarrowingNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1b, INS_sve_stnt1b, INS_sve_stnt1b, INS_sve_stnt1b, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, ScatterNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1w, INS_sve_stnt1w, INS_sve_stnt1d, INS_sve_stnt1d, INS_sve_stnt1w, INS_sve_stnt1d, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, ScatterWithByteOffsetsNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1w, INS_sve_stnt1w, INS_sve_stnt1d, INS_sve_stnt1d, INS_sve_stnt1w, INS_sve_stnt1d, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, ShiftArithmeticRounded, -1, -1, INS_sve_srshl, INS_invalid, INS_sve_srshl, INS_invalid, INS_sve_srshl, INS_invalid, INS_sve_srshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, ShiftArithmeticRoundedSaturate, -1, -1, INS_sve_sqrshl, INS_invalid, INS_sve_sqrshl, INS_invalid, INS_sve_sqrshl, INS_invalid, INS_sve_sqrshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, ShiftArithmeticSaturate, -1, -1, INS_sve_sqshl, INS_invalid, INS_sve_sqshl, INS_invalid, INS_sve_sqshl, INS_invalid, INS_sve_sqshl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve2, ShiftLeftAndInsert, -1, 3, INS_sve_sli, INS_sve_sli, INS_sve_sli, INS_sve_sli, INS_sve_sli, INS_sve_sli, INS_sve_sli, INS_sve_sli, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftLeftLogicalSaturate, -1, -1, INS_invalid, INS_sve_uqshl, INS_invalid, INS_sve_uqshl, INS_invalid, INS_sve_uqshl, INS_invalid, INS_sve_uqshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftLeftLogicalSaturateUnsigned, -1, -1, INS_invalid, INS_sve_sqshlu, INS_invalid, INS_sve_sqshlu, INS_invalid, INS_sve_sqshlu, INS_invalid, INS_sve_sqshlu, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftLeftLogicalWideningEven, -1, 2, INS_invalid, INS_invalid, INS_sve_sshllb, INS_sve_ushllb, INS_sve_sshllb, INS_sve_ushllb, INS_sve_sshllb, INS_sve_ushllb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve2, ShiftLeftLogicalWideningOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_sshllt, INS_sve_ushllt, INS_sve_sshllt, INS_sve_ushllt, INS_sve_sshllt, INS_sve_ushllt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve2, ShiftLogicalRounded, -1, -1, INS_invalid, INS_sve_urshl, INS_invalid, INS_sve_urshl, INS_invalid, INS_sve_urshl, INS_invalid, INS_sve_urshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftLogicalRoundedSaturate, -1, -1, INS_invalid, INS_sve_uqrshl, INS_invalid, INS_sve_uqrshl, INS_invalid, INS_sve_uqrshl, INS_invalid, INS_sve_uqrshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftRightAndInsert, -1, 3, INS_sve_sri, INS_sve_sri, INS_sve_sri, INS_sve_sri, INS_sve_sri, INS_sve_sri, INS_sve_sri, INS_sve_sri, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticAdd, -1, 3, INS_sve_ssra, INS_invalid, INS_sve_ssra, INS_invalid, INS_sve_ssra, INS_invalid, INS_sve_ssra, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticNarrowingSaturateEven, -1, 2, INS_sve_sqshrnb, INS_invalid, INS_sve_sqshrnb, INS_invalid, INS_sve_sqshrnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticNarrowingSaturateOdd, -1, 3, INS_sve_sqshrnt, INS_invalid, INS_sve_sqshrnt, INS_invalid, INS_sve_sqshrnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticNarrowingSaturateUnsignedEven, -1, 2, INS_invalid, INS_sve_sqshrunb, INS_invalid, INS_sve_sqshrunb, INS_invalid, INS_sve_sqshrunb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticNarrowingSaturateUnsignedOdd, -1, 3, INS_invalid, INS_sve_sqshrunt, INS_invalid, INS_sve_sqshrunt, INS_invalid, INS_sve_sqshrunt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticRounded, -1, -1, INS_sve_srshr, INS_invalid, INS_sve_srshr, INS_invalid, INS_sve_srshr, INS_invalid, INS_sve_srshr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticRoundedAdd, -1, 3, INS_sve_srsra, INS_invalid, INS_sve_srsra, INS_invalid, INS_sve_srsra, INS_invalid, INS_sve_srsra, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticRoundedNarrowingSaturateEven, -1, 2, INS_sve_sqrshrnb, INS_invalid, INS_sve_sqrshrnb, INS_invalid, INS_sve_sqrshrnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticRoundedNarrowingSaturateOdd, -1, 3, INS_sve_sqrshrnt, INS_invalid, INS_sve_sqrshrnt, INS_invalid, INS_sve_sqrshrnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticRoundedNarrowingSaturateUnsignedEven, -1, 2, INS_invalid, INS_sve_sqrshrunb, INS_invalid, INS_sve_sqrshrunb, INS_invalid, INS_sve_sqrshrunb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticRoundedNarrowingSaturateUnsignedOdd, -1, 3, INS_invalid, INS_sve_sqrshrunt, INS_invalid, INS_sve_sqrshrunt, INS_invalid, INS_sve_sqrshrunt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalAdd, -1, 3, INS_invalid, INS_sve_usra, INS_invalid, INS_sve_usra, INS_invalid, INS_sve_usra, INS_invalid, INS_sve_usra, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalNarrowingEven, -1, 2, INS_sve_shrnb, INS_sve_shrnb, INS_sve_shrnb, INS_sve_shrnb, INS_sve_shrnb, INS_sve_shrnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalNarrowingOdd, -1, 3, INS_sve_shrnt, INS_sve_shrnt, INS_sve_shrnt, INS_sve_shrnt, INS_sve_shrnt, INS_sve_shrnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalNarrowingSaturateEven, -1, 2, INS_invalid, INS_sve_uqshrnb, INS_invalid, INS_sve_uqshrnb, INS_invalid, INS_sve_uqshrnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalNarrowingSaturateOdd, -1, 3, INS_invalid, INS_sve_uqshrnt, INS_invalid, INS_sve_uqshrnt, INS_invalid, INS_sve_uqshrnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalRounded, -1, -1, INS_invalid, INS_sve_urshr, INS_invalid, INS_sve_urshr, INS_invalid, INS_sve_urshr, INS_invalid, INS_sve_urshr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalRoundedAdd, -1, 3, INS_invalid, INS_sve_ursra, INS_invalid, INS_sve_ursra, INS_invalid, INS_sve_ursra, INS_invalid, INS_sve_ursra, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalRoundedNarrowingEven, -1, 2, INS_sve_rshrnb, INS_sve_rshrnb, INS_sve_rshrnb, INS_sve_rshrnb, INS_sve_rshrnb, INS_sve_rshrnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalRoundedNarrowingOdd, -1, 3, INS_sve_rshrnt, INS_sve_rshrnt, INS_sve_rshrnt, INS_sve_rshrnt, INS_sve_rshrnt, INS_sve_rshrnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalRoundedNarrowingSaturateEven, -1, 2, INS_invalid, INS_sve_uqrshrnb, INS_invalid, INS_sve_uqrshrnb, INS_invalid, INS_sve_uqrshrnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) -HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalRoundedNarrowingSaturateOdd, -1, 3, INS_invalid, INS_sve_uqrshrnt, INS_invalid, INS_sve_uqrshrnt, INS_invalid, INS_sve_uqrshrnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, SubtractBorrowWideningEven, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sbclb, INS_invalid, INS_sve_sbclb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, SubtractBorrowWideningOdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sbclt, INS_invalid, INS_sve_sbclt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, SubtractHighNarrowingEven, -1, 2, INS_sve_subhnb, INS_sve_subhnb, INS_sve_subhnb, INS_sve_subhnb, INS_sve_subhnb, INS_sve_subhnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, SubtractHighNarrowingOdd, -1, 3, INS_sve_subhnt, INS_sve_subhnt, INS_sve_subhnt, INS_sve_subhnt, INS_sve_subhnt, INS_sve_subhnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, SubtractRoundedHighNarrowingEven, -1, 2, INS_sve_rsubhnb, INS_sve_rsubhnb, INS_sve_rsubhnb, INS_sve_rsubhnb, INS_sve_rsubhnb, INS_sve_rsubhnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, SubtractRoundedHighNarrowingOdd, -1, 3, INS_sve_rsubhnt, INS_sve_rsubhnt, INS_sve_rsubhnt, INS_sve_rsubhnt, INS_sve_rsubhnt, INS_sve_rsubhnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, SubtractSaturate, -1, -1, INS_sve_sqsub, INS_sve_uqsub, INS_sve_sqsub, INS_sve_uqsub, INS_sve_sqsub, INS_sve_uqsub, INS_sve_sqsub, INS_sve_uqsub, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, SubtractWideningEven, -1, 2, INS_invalid, INS_invalid, INS_sve_ssubwb, INS_sve_usubwb, INS_sve_ssubwb, INS_sve_usubwb, INS_sve_ssubwb, INS_sve_usubwb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve2, SubtractWideningEvenOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_ssublbt, INS_invalid, INS_sve_ssublbt, INS_invalid, INS_sve_ssublbt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, SubtractWideningOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_ssubwt, INS_sve_usubwt, INS_sve_ssubwt, INS_sve_usubwt, INS_sve_ssubwt, INS_sve_usubwt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve2, SubtractWideningOddEven, -1, 2, INS_invalid, INS_invalid, INS_sve_ssubltb, INS_invalid, INS_sve_ssubltb, INS_invalid, INS_sve_ssubltb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve2, VectorTableLookup, -1, 2, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve2, VectorTableLookupExtension, -1, 3, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, Xor, -1, 3, INS_sve_eor3, INS_sve_eor3, INS_sve_eor3, INS_sve_eor3, INS_sve_eor3, INS_sve_eor3, INS_sve_eor3, INS_sve_eor3, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve2, XorRotateRight, -1, 3, INS_sve_xar, INS_sve_xar, INS_sve_xar, INS_sve_xar, INS_sve_xar, INS_sve_xar, INS_sve_xar, INS_sve_xar, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve2, AbsSaturate, -1, -1, INS_sve_sqabs, INS_invalid, INS_sve_sqabs, INS_invalid, INS_sve_sqabs, INS_invalid, INS_sve_sqabs, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, AbsoluteDifferenceAdd, -1, 3, INS_sve_saba, INS_sve_uaba, INS_sve_saba, INS_sve_uaba, INS_sve_saba, INS_sve_uaba, INS_sve_saba, INS_sve_uaba, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, AbsoluteDifferenceWideningEven, -1, 2, INS_invalid, INS_invalid, INS_sve_sabdlb, INS_sve_uabdlb, INS_sve_sabdlb, INS_sve_uabdlb, INS_sve_sabdlb, INS_sve_uabdlb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, AbsoluteDifferenceWideningLowerAndAddEven, -1, 3, INS_invalid, INS_invalid, INS_sve_sabalb, INS_sve_uabalb, INS_sve_sabalb, INS_sve_uabalb, INS_sve_sabalb, INS_sve_uabalb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, AbsoluteDifferenceWideningLowerAndAddOdd, -1, 3, INS_invalid, INS_invalid, INS_sve_sabalt, INS_sve_uabalt, INS_sve_sabalt, INS_sve_uabalt, INS_sve_sabalt, INS_sve_uabalt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, AbsoluteDifferenceWideningOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_sabdlt, INS_sve_uabdlt, INS_sve_sabdlt, INS_sve_uabdlt, INS_sve_sabdlt, INS_sve_uabdlt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, AddCarryWideningEven, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_adclb, INS_invalid, INS_sve_adclb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, AddCarryWideningOdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_adclt, INS_invalid, INS_sve_adclt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, AddHighNarrowingEven, -1, 2, INS_sve_addhnb, INS_sve_addhnb, INS_sve_addhnb, INS_sve_addhnb, INS_sve_addhnb, INS_sve_addhnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, AddHighNarrowingOdd, -1, 3, INS_sve_addhnt, INS_sve_addhnt, INS_sve_addhnt, INS_sve_addhnt, INS_sve_addhnt, INS_sve_addhnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, AddPairwise, -1, -1, INS_sve_addp, INS_sve_addp, INS_sve_addp, INS_sve_addp, INS_sve_addp, INS_sve_addp, INS_sve_addp, INS_sve_addp, INS_sve_faddp, INS_sve_faddp, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation) +HARDWARE_INTRINSIC(Sve2, AddPairwiseWideningAndAdd, -1, -1, INS_invalid, INS_invalid, INS_sve_sadalp, INS_sve_uadalp, INS_sve_sadalp, INS_sve_uadalp, INS_sve_sadalp, INS_sve_uadalp, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, AddRotateComplex, -1, 3, INS_sve_cadd, INS_sve_cadd, INS_sve_cadd, INS_sve_cadd, INS_sve_cadd, INS_sve_cadd, INS_sve_cadd, INS_sve_cadd, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve2, AddRoundedHighNarrowingEven, -1, 2, INS_sve_raddhnb, INS_sve_raddhnb, INS_sve_raddhnb, INS_sve_raddhnb, INS_sve_raddhnb, INS_sve_raddhnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, AddRoundedHighNarrowingOdd, -1, 3, INS_sve_raddhnt, INS_sve_raddhnt, INS_sve_raddhnt, INS_sve_raddhnt, INS_sve_raddhnt, INS_sve_raddhnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, AddSaturate, -1, -1, INS_sve_sqadd, INS_sve_uqadd, INS_sve_sqadd, INS_sve_uqadd, INS_sve_sqadd, INS_sve_uqadd, INS_sve_sqadd, INS_sve_uqadd, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve2, AddSaturateRotateComplex, -1, 3, INS_sve_sqcadd, INS_invalid, INS_sve_sqcadd, INS_invalid, INS_sve_sqcadd, INS_invalid, INS_sve_sqcadd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve2, AddWideningEven, -1, 2, INS_invalid, INS_invalid, INS_sve_saddwb, INS_sve_uaddwb, INS_sve_saddwb, INS_sve_uaddwb, INS_sve_saddwb, INS_sve_uaddwb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve2, AddWideningEvenOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_saddlbt, INS_invalid, INS_sve_saddlbt, INS_invalid, INS_sve_saddlbt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, AddWideningOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_saddwt, INS_sve_uaddwt, INS_sve_saddwt, INS_sve_uaddwt, INS_sve_saddwt, INS_sve_uaddwt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve2, BitwiseClearXor, -1, 3, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, BitwiseSelect, -1, 3, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, BitwiseSelectLeftInverted, -1, 3, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, BitwiseSelectRightInverted, -1, 3, INS_sve_bsl2n, INS_sve_bsl2n, INS_sve_bsl2n, INS_sve_bsl2n, INS_sve_bsl2n, INS_sve_bsl2n, INS_sve_bsl2n, INS_sve_bsl2n, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ConvertToDoubleOdd, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtlt, INS_sve_fcvtlt, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, ConvertToSingleEvenRoundToOdd, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtx, INS_sve_fcvtx, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, ConvertToSingleOdd, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtnt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ConvertToSingleOddRoundToOdd, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtxnt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, CountMatchingElements, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_histcnt, INS_sve_histcnt, INS_sve_histcnt, INS_sve_histcnt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve2, CountMatchingElementsIn128BitSegments, -1, 2, INS_sve_histseg, INS_sve_histseg, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskByte, -1, 2, INS_invalid, INS_sve_whilegt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskDouble, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilegt, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_whilegt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilegt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilegt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskSByte, -1, 2, INS_sve_whilegt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskSingle, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilegt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilegt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilegt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanMaskUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilegt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskByte, -1, 2, INS_invalid, INS_sve_whilege, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskDouble, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilege, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_whilege, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilege, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilege, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskSByte, -1, 2, INS_sve_whilege, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskSingle, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilege, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilege, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilege, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileGreaterThanOrEqualMaskUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilege, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskByte, -1, 2, INS_invalid, INS_sve_whilerw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskDouble, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilerw, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_whilerw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilerw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilerw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskSByte, -1, 2, INS_sve_whilerw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskSingle, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilerw, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilerw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilerw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileReadAfterWriteMaskUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilerw, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskByte, -1, 2, INS_invalid, INS_sve_whilewr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskDouble, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilewr, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskInt16, -1, 2, INS_invalid, INS_invalid, INS_sve_whilewr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilewr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilewr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskSByte, -1, 2, INS_sve_whilewr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskSingle, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilewr, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskUInt16, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilewr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskUInt32, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilewr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, CreateWhileWriteAfterReadMaskUInt64, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_whilewr, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve2, DotProductRotateComplex, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_cdot, INS_invalid, INS_sve_cdot, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve2, DotProductRotateComplexBySelectedIndex, -1, 5, INS_sve_cdot, INS_invalid, INS_sve_cdot, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(Sve2, FusedAddHalving, -1, -1, INS_sve_shadd, INS_sve_uhadd, INS_sve_shadd, INS_sve_uhadd, INS_sve_shadd, INS_sve_uhadd, INS_sve_shadd, INS_sve_uhadd, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, FusedAddRoundedHalving, -1, -1, INS_sve_srhadd, INS_sve_urhadd, INS_sve_srhadd, INS_sve_urhadd, INS_sve_srhadd, INS_sve_urhadd, INS_sve_srhadd, INS_sve_urhadd, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, FusedSubtractHalving, -1, -1, INS_sve_shsub, INS_sve_uhsub, INS_sve_shsub, INS_sve_uhsub, INS_sve_shsub, INS_sve_uhsub, INS_sve_shsub, INS_sve_uhsub, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, GatherVectorByteZeroExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1b, INS_sve_ldnt1b, INS_sve_ldnt1b, INS_sve_ldnt1b, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve2, GatherVectorInt16SignExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1sh, INS_sve_ldnt1sh, INS_sve_ldnt1sh, INS_sve_ldnt1sh, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve2, GatherVectorInt16WithByteOffsetsSignExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1sh, INS_sve_ldnt1sh, INS_sve_ldnt1sh, INS_sve_ldnt1sh, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve2, GatherVectorInt32SignExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1sw, INS_sve_ldnt1sw, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve2, GatherVectorInt32WithByteOffsetsSignExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1sw, INS_sve_ldnt1sw, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve2, GatherVectorNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1w, INS_sve_ldnt1w, INS_sve_ldnt1d, INS_sve_ldnt1d, INS_sve_ldnt1w, INS_sve_ldnt1d, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve2, GatherVectorSByteSignExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1sb, INS_sve_ldnt1sb, INS_sve_ldnt1sb, INS_sve_ldnt1sb, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve2, GatherVectorUInt16WithByteOffsetsZeroExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1h, INS_sve_ldnt1h, INS_sve_ldnt1h, INS_sve_ldnt1h, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve2, GatherVectorUInt16ZeroExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1h, INS_sve_ldnt1h, INS_sve_ldnt1h, INS_sve_ldnt1h, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve2, GatherVectorUInt32WithByteOffsetsZeroExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1w, INS_sve_ldnt1w, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve2, GatherVectorUInt32ZeroExtendNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1w, INS_sve_ldnt1w, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve2, GatherVectorWithByteOffsetsNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnt1w, INS_sve_ldnt1w, INS_sve_ldnt1d, INS_sve_ldnt1d, INS_sve_ldnt1w, INS_sve_ldnt1d, -1, -1, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve2, InterleavingXorEvenOdd, -1, 3, INS_sve_eorbt, INS_sve_eorbt, INS_sve_eorbt, INS_sve_eorbt, INS_sve_eorbt, INS_sve_eorbt, INS_sve_eorbt, INS_sve_eorbt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, InterleavingXorOddEven, -1, 3, INS_sve_eortb, INS_sve_eortb, INS_sve_eortb, INS_sve_eortb, INS_sve_eortb, INS_sve_eortb, INS_sve_eortb, INS_sve_eortb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, Log2, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_flogb, INS_invalid, INS_sve_flogb, INS_invalid, INS_sve_flogb, INS_sve_flogb, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, Match, -1, 3, INS_sve_match, INS_sve_match, INS_sve_match, INS_sve_match, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve2, MaxNumberPairwise, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmaxnmp, INS_sve_fmaxnmp, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, MaxPairwise, -1, -1, INS_sve_smaxp, INS_sve_umaxp, INS_sve_smaxp, INS_sve_umaxp, INS_sve_smaxp, INS_sve_umaxp, INS_sve_smaxp, INS_sve_umaxp, INS_sve_fmaxp, INS_sve_fmaxp, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_ReduceOperation) +HARDWARE_INTRINSIC(Sve2, MinNumberPairwise, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fminnmp, INS_sve_fminnmp, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, MinPairwise, -1, -1, INS_sve_sminp, INS_sve_uminp, INS_sve_sminp, INS_sve_uminp, INS_sve_sminp, INS_sve_uminp, INS_sve_sminp, INS_sve_uminp, INS_sve_fminp, INS_sve_fminp, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_ReduceOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyAddBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, MultiplyAddRotateComplex, -1, 4, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve2, MultiplyAddRotateComplexBySelectedScalar, -1, 5, INS_invalid, INS_invalid, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_sve_cmla, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve2, MultiplyAddRoundedDoublingSaturateHighRotateComplex, -1, 4, INS_sve_sqrdcmlah, INS_invalid, INS_sve_sqrdcmlah, INS_invalid, INS_sve_sqrdcmlah, INS_invalid, INS_sve_sqrdcmlah, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve2, MultiplyAddRoundedDoublingSaturateHighRotateComplexBySelectedScalar, -1, 5, INS_invalid, INS_invalid, INS_sve_sqrdcmlah, INS_invalid, INS_sve_sqrdcmlah, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalar, -1, 3, INS_invalid, INS_invalid, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalarWideningEven, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_smullb, INS_sve_umullb, INS_sve_smullb, INS_sve_umullb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalarWideningEvenAndAdd, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_smlalb, INS_sve_umlalb, INS_sve_smlalb, INS_sve_umlalb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalarWideningEvenAndSubtract, -1, 4, INS_invalid, INS_invalid, INS_sve_smlslb, INS_sve_umlslb, INS_sve_smlslb, INS_sve_umlslb, INS_sve_smlslb, INS_sve_umlslb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalarWideningOdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_smullt, INS_sve_umullt, INS_sve_smullt, INS_sve_umullt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalarWideningOddAndAdd, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_smlalt, INS_sve_umlalt, INS_sve_smlalt, INS_sve_umlalt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalarWideningOddAndSubtract, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_smlslt, INS_sve_umlslt, INS_sve_smlslt, INS_sve_umlslt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingBySelectedScalarSaturateHigh, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdmulh, INS_invalid, INS_sve_sqdmulh, INS_invalid, INS_sve_sqdmulh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingSaturateHigh, -1, 2, INS_sve_sqdmulh, INS_invalid, INS_sve_sqdmulh, INS_invalid, INS_sve_sqdmulh, INS_invalid, INS_sve_sqdmulh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningAndAddSaturateEven, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdmlalb, INS_invalid, INS_sve_sqdmlalb, INS_invalid, INS_sve_sqdmlalb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningAndAddSaturateEvenOdd, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdmlalbt, INS_invalid, INS_sve_sqdmlalbt, INS_invalid, INS_sve_sqdmlalbt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningAndAddSaturateOdd, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdmlalt, INS_invalid, INS_sve_sqdmlalt, INS_invalid, INS_sve_sqdmlalt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningAndSubtractSaturateEven, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdmlslb, INS_invalid, INS_sve_sqdmlslb, INS_invalid, INS_sve_sqdmlslb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningAndSubtractSaturateEvenOdd, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdmlslbt, INS_invalid, INS_sve_sqdmlslbt, INS_invalid, INS_sve_sqdmlslbt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningAndSubtractSaturateOdd, -1, 3, INS_invalid, INS_invalid, INS_sve_sqdmlslt, INS_invalid, INS_sve_sqdmlslt, INS_invalid, INS_sve_sqdmlslt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningBySelectedScalarAndAddSaturateEven, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdmlalb, INS_invalid, INS_sve_sqdmlalb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningBySelectedScalarAndAddSaturateOdd, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdmlalt, INS_invalid, INS_sve_sqdmlalt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningBySelectedScalarAndSubtractSaturateEven, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdmlslb, INS_invalid, INS_sve_sqdmlslb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningBySelectedScalarAndSubtractSaturateOdd, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdmlslt, INS_invalid, INS_sve_sqdmlslt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningSaturateEven, -1, 2, INS_invalid, INS_invalid, INS_sve_sqdmullb, INS_invalid, INS_sve_sqdmullb, INS_invalid, INS_sve_sqdmullb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningSaturateEvenBySelectedScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdmullb, INS_invalid, INS_sve_sqdmullb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningSaturateOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_sqdmullt, INS_invalid, INS_sve_sqdmullt, INS_invalid, INS_sve_sqdmullt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningSaturateOddBySelectedScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdmullt, INS_invalid, INS_sve_sqdmullt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyRoundedDoublingBySelectedScalarSaturateHigh, -1, 3, INS_invalid, INS_invalid, INS_sve_sqrdmulh, INS_invalid, INS_sve_sqrdmulh, INS_invalid, INS_sve_sqrdmulh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyRoundedDoublingSaturateAndAddHigh, -1, 3, INS_sve_sqrdmlah, INS_invalid, INS_sve_sqrdmlah, INS_invalid, INS_sve_sqrdmlah, INS_invalid, INS_sve_sqrdmlah, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, MultiplyRoundedDoublingSaturateAndSubtractHigh, -1, 3, INS_sve_sqrdmlsh, INS_invalid, INS_sve_sqrdmlsh, INS_invalid, INS_sve_sqrdmlsh, INS_invalid, INS_sve_sqrdmlsh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, MultiplyRoundedDoublingSaturateBySelectedScalarAndAddHigh, -1, 4, INS_invalid, INS_invalid, INS_sve_sqrdmlah, INS_invalid, INS_sve_sqrdmlah, INS_invalid, INS_sve_sqrdmlah, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyRoundedDoublingSaturateBySelectedScalarAndSubtractHigh, -1, 4, INS_invalid, INS_invalid, INS_sve_sqrdmlsh, INS_invalid, INS_sve_sqrdmlsh, INS_invalid, INS_sve_sqrdmlsh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyRoundedDoublingSaturateHigh, -1, 2, INS_sve_sqrdmulh, INS_invalid, INS_sve_sqrdmulh, INS_invalid, INS_sve_sqrdmulh, INS_invalid, INS_sve_sqrdmulh, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, MultiplySubtractBySelectedScalar, -1, 4, INS_invalid, INS_invalid, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve2, MultiplyWideningEven, -1, 2, INS_invalid, INS_invalid, INS_sve_smullb, INS_sve_umullb, INS_sve_smullb, INS_sve_umullb, INS_sve_smullb, INS_sve_umullb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, MultiplyWideningEvenAndAdd, -1, 3, INS_invalid, INS_invalid, INS_sve_smlalb, INS_sve_umlalb, INS_sve_smlalb, INS_sve_umlalb, INS_sve_smlalb, INS_sve_umlalb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, MultiplyWideningEvenAndSubtract, -1, 3, INS_invalid, INS_invalid, INS_sve_smlslb, INS_sve_umlslb, INS_sve_smlslb, INS_sve_umlslb, INS_sve_smlslb, INS_sve_umlslb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, MultiplyWideningOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_smullt, INS_sve_umullt, INS_sve_smullt, INS_sve_umullt, INS_sve_smullt, INS_sve_umullt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, MultiplyWideningOddAndAdd, -1, 3, INS_invalid, INS_invalid, INS_sve_smlalt, INS_sve_umlalt, INS_sve_smlalt, INS_sve_umlalt, INS_sve_smlalt, INS_sve_umlalt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, MultiplyWideningOddAndSubtract, -1, 3, INS_invalid, INS_invalid, INS_sve_smlslt, INS_sve_umlslt, INS_sve_smlslt, INS_sve_umlslt, INS_sve_smlslt, INS_sve_umlslt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, NegateSaturate, -1, -1, INS_sve_sqneg, INS_invalid, INS_sve_sqneg, INS_invalid, INS_sve_sqneg, INS_invalid, INS_sve_sqneg, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, NoMatch, -1, 3, INS_sve_nmatch, INS_sve_nmatch, INS_sve_nmatch, INS_sve_nmatch, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve2, PolynomialMultiply, -1, 2, INS_sve_pmul, INS_sve_pmul, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, PolynomialMultiplyWideningEven, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_pmullb, INS_invalid, INS_invalid, INS_invalid, INS_sve_pmullb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, PolynomialMultiplyWideningOdd, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_sve_pmullt, INS_invalid, INS_invalid, INS_invalid, INS_sve_pmullt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, ReciprocalEstimate, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_urecpe, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, ReciprocalSqrtEstimate, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ursqrte, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, SaturatingExtractNarrowingLower, -1, 1, INS_sve_sqxtnb, INS_sve_uqxtnb, INS_sve_sqxtnb, INS_sve_uqxtnb, INS_sve_sqxtnb, INS_sve_uqxtnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, SaturatingExtractNarrowingUpper, -1, 2, INS_sve_sqxtnt, INS_sve_uqxtnt, INS_sve_sqxtnt, INS_sve_uqxtnt, INS_sve_sqxtnt, INS_sve_uqxtnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, SaturatingExtractUnsignedNarrowingLower, -1, 1, INS_invalid, INS_sve_sqxtunb, INS_invalid, INS_sve_sqxtunb, INS_invalid, INS_sve_sqxtunb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, SaturatingExtractUnsignedNarrowingUpper, -1, 2, INS_invalid, INS_sve_sqxtunt, INS_invalid, INS_sve_sqxtunt, INS_invalid, INS_sve_sqxtunt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, Scatter16BitNarrowingNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1h, INS_sve_stnt1h, INS_sve_stnt1h, INS_sve_stnt1h, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, Scatter16BitWithByteOffsetsNarrowingNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1h, INS_sve_stnt1h, INS_sve_stnt1h, INS_sve_stnt1h, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, Scatter32BitNarrowingNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1w, INS_sve_stnt1w, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, Scatter32BitWithByteOffsetsNarrowingNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1w, INS_sve_stnt1w, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, Scatter8BitNarrowingNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1b, INS_sve_stnt1b, INS_sve_stnt1b, INS_sve_stnt1b, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, Scatter8BitWithByteOffsetsNarrowingNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1b, INS_sve_stnt1b, INS_sve_stnt1b, INS_sve_stnt1b, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, ScatterNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1w, INS_sve_stnt1w, INS_sve_stnt1d, INS_sve_stnt1d, INS_sve_stnt1w, INS_sve_stnt1d, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, ScatterWithByteOffsetsNonTemporal, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_stnt1w, INS_sve_stnt1w, INS_sve_stnt1d, INS_sve_stnt1d, INS_sve_stnt1w, INS_sve_stnt1d, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, ShiftArithmeticRounded, -1, -1, INS_sve_srshl, INS_invalid, INS_sve_srshl, INS_invalid, INS_sve_srshl, INS_invalid, INS_sve_srshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, ShiftArithmeticRoundedSaturate, -1, -1, INS_sve_sqrshl, INS_invalid, INS_sve_sqrshl, INS_invalid, INS_sve_sqrshl, INS_invalid, INS_sve_sqrshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, ShiftArithmeticSaturate, -1, -1, INS_sve_sqshl, INS_invalid, INS_sve_sqshl, INS_invalid, INS_sve_sqshl, INS_invalid, INS_sve_sqshl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve2, ShiftLeftAndInsert, -1, 3, INS_sve_sli, INS_sve_sli, INS_sve_sli, INS_sve_sli, INS_sve_sli, INS_sve_sli, INS_sve_sli, INS_sve_sli, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftLeftLogicalSaturate, -1, -1, INS_invalid, INS_sve_uqshl, INS_invalid, INS_sve_uqshl, INS_invalid, INS_sve_uqshl, INS_invalid, INS_sve_uqshl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftLeftLogicalSaturateUnsigned, -1, -1, INS_invalid, INS_sve_sqshlu, INS_invalid, INS_sve_sqshlu, INS_invalid, INS_sve_sqshlu, INS_invalid, INS_sve_sqshlu, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftLeftLogicalWideningEven, -1, 2, INS_invalid, INS_invalid, INS_sve_sshllb, INS_sve_ushllb, INS_sve_sshllb, INS_sve_ushllb, INS_sve_sshllb, INS_sve_ushllb, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve2, ShiftLeftLogicalWideningOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_sshllt, INS_sve_ushllt, INS_sve_sshllt, INS_sve_ushllt, INS_sve_sshllt, INS_sve_ushllt, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftLeftByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve2, ShiftLogicalRounded, -1, -1, INS_invalid, INS_sve_urshl, INS_invalid, INS_sve_urshl, INS_invalid, INS_sve_urshl, INS_invalid, INS_sve_urshl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftLogicalRoundedSaturate, -1, -1, INS_invalid, INS_sve_uqrshl, INS_invalid, INS_sve_uqrshl, INS_invalid, INS_sve_uqrshl, INS_invalid, INS_sve_uqrshl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftRightAndInsert, -1, 3, INS_sve_sri, INS_sve_sri, INS_sve_sri, INS_sve_sri, INS_sve_sri, INS_sve_sri, INS_sve_sri, INS_sve_sri, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticAdd, -1, 3, INS_sve_ssra, INS_invalid, INS_sve_ssra, INS_invalid, INS_sve_ssra, INS_invalid, INS_sve_ssra, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticNarrowingSaturateEven, -1, 2, INS_sve_sqshrnb, INS_invalid, INS_sve_sqshrnb, INS_invalid, INS_sve_sqshrnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticNarrowingSaturateOdd, -1, 3, INS_sve_sqshrnt, INS_invalid, INS_sve_sqshrnt, INS_invalid, INS_sve_sqshrnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticNarrowingSaturateUnsignedEven, -1, 2, INS_invalid, INS_sve_sqshrunb, INS_invalid, INS_sve_sqshrunb, INS_invalid, INS_sve_sqshrunb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticNarrowingSaturateUnsignedOdd, -1, 3, INS_invalid, INS_sve_sqshrunt, INS_invalid, INS_sve_sqshrunt, INS_invalid, INS_sve_sqshrunt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticRounded, -1, -1, INS_sve_srshr, INS_invalid, INS_sve_srshr, INS_invalid, INS_sve_srshr, INS_invalid, INS_sve_srshr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticRoundedAdd, -1, 3, INS_sve_srsra, INS_invalid, INS_sve_srsra, INS_invalid, INS_sve_srsra, INS_invalid, INS_sve_srsra, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticRoundedNarrowingSaturateEven, -1, 2, INS_sve_sqrshrnb, INS_invalid, INS_sve_sqrshrnb, INS_invalid, INS_sve_sqrshrnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticRoundedNarrowingSaturateOdd, -1, 3, INS_sve_sqrshrnt, INS_invalid, INS_sve_sqrshrnt, INS_invalid, INS_sve_sqrshrnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticRoundedNarrowingSaturateUnsignedEven, -1, 2, INS_invalid, INS_sve_sqrshrunb, INS_invalid, INS_sve_sqrshrunb, INS_invalid, INS_sve_sqrshrunb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve2, ShiftRightArithmeticRoundedNarrowingSaturateUnsignedOdd, -1, 3, INS_invalid, INS_sve_sqrshrunt, INS_invalid, INS_sve_sqrshrunt, INS_invalid, INS_sve_sqrshrunt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalAdd, -1, 3, INS_invalid, INS_sve_usra, INS_invalid, INS_sve_usra, INS_invalid, INS_sve_usra, INS_invalid, INS_sve_usra, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalNarrowingEven, -1, 2, INS_sve_shrnb, INS_sve_shrnb, INS_sve_shrnb, INS_sve_shrnb, INS_sve_shrnb, INS_sve_shrnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalNarrowingOdd, -1, 3, INS_sve_shrnt, INS_sve_shrnt, INS_sve_shrnt, INS_sve_shrnt, INS_sve_shrnt, INS_sve_shrnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalNarrowingSaturateEven, -1, 2, INS_invalid, INS_sve_uqshrnb, INS_invalid, INS_sve_uqshrnb, INS_invalid, INS_sve_uqshrnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalNarrowingSaturateOdd, -1, 3, INS_invalid, INS_sve_uqshrnt, INS_invalid, INS_sve_uqshrnt, INS_invalid, INS_sve_uqshrnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalRounded, -1, -1, INS_invalid, INS_sve_urshr, INS_invalid, INS_sve_urshr, INS_invalid, INS_sve_urshr, INS_invalid, INS_sve_urshr, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalRoundedAdd, -1, 3, INS_invalid, INS_sve_ursra, INS_invalid, INS_sve_ursra, INS_invalid, INS_sve_ursra, INS_invalid, INS_sve_ursra, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalRoundedNarrowingEven, -1, 2, INS_sve_rshrnb, INS_sve_rshrnb, INS_sve_rshrnb, INS_sve_rshrnb, INS_sve_rshrnb, INS_sve_rshrnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalRoundedNarrowingOdd, -1, 3, INS_sve_rshrnt, INS_sve_rshrnt, INS_sve_rshrnt, INS_sve_rshrnt, INS_sve_rshrnt, INS_sve_rshrnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalRoundedNarrowingSaturateEven, -1, 2, INS_invalid, INS_sve_uqrshrnb, INS_invalid, INS_sve_uqrshrnb, INS_invalid, INS_sve_uqrshrnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve2, ShiftRightLogicalRoundedNarrowingSaturateOdd, -1, 3, INS_invalid, INS_sve_uqrshrnt, INS_invalid, INS_sve_uqrshrnt, INS_invalid, INS_sve_uqrshrnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, SubtractBorrowWideningEven, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sbclb, INS_invalid, INS_sve_sbclb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, SubtractBorrowWideningOdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sbclt, INS_invalid, INS_sve_sbclt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, SubtractHighNarrowingEven, -1, 2, INS_sve_subhnb, INS_sve_subhnb, INS_sve_subhnb, INS_sve_subhnb, INS_sve_subhnb, INS_sve_subhnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, SubtractHighNarrowingOdd, -1, 3, INS_sve_subhnt, INS_sve_subhnt, INS_sve_subhnt, INS_sve_subhnt, INS_sve_subhnt, INS_sve_subhnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, SubtractRoundedHighNarrowingEven, -1, 2, INS_sve_rsubhnb, INS_sve_rsubhnb, INS_sve_rsubhnb, INS_sve_rsubhnb, INS_sve_rsubhnb, INS_sve_rsubhnb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, SubtractRoundedHighNarrowingOdd, -1, 3, INS_sve_rsubhnt, INS_sve_rsubhnt, INS_sve_rsubhnt, INS_sve_rsubhnt, INS_sve_rsubhnt, INS_sve_rsubhnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, SubtractSaturate, -1, -1, INS_sve_sqsub, INS_sve_uqsub, INS_sve_sqsub, INS_sve_uqsub, INS_sve_sqsub, INS_sve_uqsub, INS_sve_sqsub, INS_sve_uqsub, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, SubtractWideningEven, -1, 2, INS_invalid, INS_invalid, INS_sve_ssubwb, INS_sve_usubwb, INS_sve_ssubwb, INS_sve_usubwb, INS_sve_ssubwb, INS_sve_usubwb, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve2, SubtractWideningEvenOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_ssublbt, INS_invalid, INS_sve_ssublbt, INS_invalid, INS_sve_ssublbt, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, SubtractWideningOdd, -1, 2, INS_invalid, INS_invalid, INS_sve_ssubwt, INS_sve_usubwt, INS_sve_ssubwt, INS_sve_usubwt, INS_sve_ssubwt, INS_sve_usubwt, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve2, SubtractWideningOddEven, -1, 2, INS_invalid, INS_invalid, INS_sve_ssubltb, INS_invalid, INS_sve_ssubltb, INS_invalid, INS_sve_ssubltb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, VectorTableLookup, -1, 2, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, INS_sve_tbl, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve2, VectorTableLookupExtension, -1, 3, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, INS_sve_tbx, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, Xor, -1, 3, INS_sve_eor3, INS_sve_eor3, INS_sve_eor3, INS_sve_eor3, INS_sve_eor3, INS_sve_eor3, INS_sve_eor3, INS_sve_eor3, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, XorRotateRight, -1, 3, INS_sve_xar, INS_sve_xar, INS_sve_xar, INS_sve_xar, INS_sve_xar, INS_sve_xar, INS_sve_xar, INS_sve_xar, INS_invalid, INS_invalid, -1, -1, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand) #define LAST_NI_Sve2 NI_Sve2_XorRotateRight // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -539,7 +539,7 @@ HARDWARE_INTRINSIC(Sve2, XorRotateRight, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // SVE SHA3 Intrinsics #define FIRST_NI_SveSha3 NI_SveSha3_BitwiseRotateLeftBy1AndXor -HARDWARE_INTRINSIC(SveSha3, BitwiseRotateLeftBy1AndXor, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rax1, INS_sve_rax1, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(SveSha3, BitwiseRotateLeftBy1AndXor, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rax1, INS_sve_rax1, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen) #define LAST_NI_SveSha3 NI_SveSha3_BitwiseRotateLeftBy1AndXor // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -548,35 +548,35 @@ HARDWARE_INTRINSIC(SveSha3, BitwiseRotateLeftBy1AndXor, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Special intrinsics that are generated during importing or lowering #define SPECIAL_NI_Sve NI_Sve_ConditionalExtractAfterLastActiveElementScalar -HARDWARE_INTRINSIC(Sve, ConditionalExtractAfterLastActiveElementScalar, 0, 3, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, ConditionalExtractLastActiveElementScalar, 0, 3, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, ConvertMaskToVector, -1, 1, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_Scalable) -HARDWARE_INTRINSIC(Sve, ConvertVectorToMask, -1, 2, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ConditionalExtractAfterLastActiveElementScalar, 0, 3, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, INS_sve_clasta, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ConditionalExtractLastActiveElementScalar, 0, 3, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, INS_sve_clastb, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ConvertMaskToVector, -1, 1, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, -1, -1, HW_Category_Helper, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve, ConvertVectorToMask, -1, 2, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, -1, -1, HW_Category_Helper, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_LowMaskedOperation) // True mask only used inside a ConvertVectorToMask -HARDWARE_INTRINSIC(Sve, ConversionTrueMask, -1, 0, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, ConversionTrueMask, -1, 0, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, -1, -1, HW_Category_Helper, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) // Scalar variants of Saturating*By*BitElementCount. There is 8bit versions as the generic version is scalar only. -HARDWARE_INTRINSIC(Sve, SaturatingDecrementBy16BitElementCountScalar, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdech, INS_sve_uqdech, INS_sve_sqdech, INS_sve_uqdech, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SaturatingDecrementBy32BitElementCountScalar, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdecw, INS_sve_uqdecw, INS_sve_sqdecw, INS_sve_uqdecw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SaturatingDecrementBy64BitElementCountScalar, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdecd, INS_sve_uqdecd, INS_sve_sqdecd, INS_sve_uqdecd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy16BitElementCountScalar, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqinch, INS_sve_uqinch, INS_sve_sqinch, INS_sve_uqinch, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy32BitElementCountScalar, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqincw, INS_sve_uqincw, INS_sve_sqincw, INS_sve_uqincw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy64BitElementCountScalar, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqincd, INS_sve_uqincd, INS_sve_sqincd, INS_sve_uqincd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, StoreAndZipx2, -1, 3, INS_sve_st2b, INS_sve_st2b, INS_sve_st2h, INS_sve_st2h, INS_sve_st2w, INS_sve_st2w, INS_sve_st2d, INS_sve_st2d, INS_sve_st2w, INS_sve_st2d, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(Sve, StoreAndZipx3, -1, 3, INS_sve_st3b, INS_sve_st3b, INS_sve_st3h, INS_sve_st3h, INS_sve_st3w, INS_sve_st3w, INS_sve_st3d, INS_sve_st3d, INS_sve_st3w, INS_sve_st3d, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters) -HARDWARE_INTRINSIC(Sve, StoreAndZipx4, -1, 3, INS_sve_st4b, INS_sve_st4b, INS_sve_st4h, INS_sve_st4h, INS_sve_st4w, INS_sve_st4w, INS_sve_st4d, INS_sve_st4d, INS_sve_st4w, INS_sve_st4d, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(Sve, SaturatingDecrementBy16BitElementCountScalar, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdech, INS_sve_uqdech, INS_sve_sqdech, INS_sve_uqdech, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, SaturatingDecrementBy32BitElementCountScalar, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdecw, INS_sve_uqdecw, INS_sve_sqdecw, INS_sve_uqdecw, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, SaturatingDecrementBy64BitElementCountScalar, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdecd, INS_sve_uqdecd, INS_sve_sqdecd, INS_sve_uqdecd, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy16BitElementCountScalar, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqinch, INS_sve_uqinch, INS_sve_sqinch, INS_sve_uqinch, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy32BitElementCountScalar, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqincw, INS_sve_uqincw, INS_sve_sqincw, INS_sve_uqincw, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy64BitElementCountScalar, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqincd, INS_sve_uqincd, INS_sve_sqincd, INS_sve_uqincd, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, StoreAndZipx2, -1, 3, INS_sve_st2b, INS_sve_st2b, INS_sve_st2h, INS_sve_st2h, INS_sve_st2w, INS_sve_st2w, INS_sve_st2d, INS_sve_st2d, INS_sve_st2w, INS_sve_st2d, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(Sve, StoreAndZipx3, -1, 3, INS_sve_st3b, INS_sve_st3b, INS_sve_st3h, INS_sve_st3h, INS_sve_st3w, INS_sve_st3w, INS_sve_st3d, INS_sve_st3d, INS_sve_st3w, INS_sve_st3d, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters) +HARDWARE_INTRINSIC(Sve, StoreAndZipx4, -1, 3, INS_sve_st4b, INS_sve_st4b, INS_sve_st4h, INS_sve_st4h, INS_sve_st4w, INS_sve_st4w, INS_sve_st4d, INS_sve_st4d, INS_sve_st4w, INS_sve_st4d, -1, -1, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters) // Predicate variants of intrinsics, these are specialized for operating on TYP_MASK type values. -HARDWARE_INTRINSIC(Sve, And_Predicates, -1, 2, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_EmbeddedMaskedOperation|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, BitwiseClear_Predicates, -1, 2, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_EmbeddedMaskedOperation|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, Or_Predicates, -1, 2, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_EmbeddedMaskedOperation|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, Xor_Predicates, -1, 2, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_EmbeddedMaskedOperation|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, ConditionalSelect_Predicates, -1, 3, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_ExplicitMaskedOperation|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Sve, ZipHigh_Predicates, -1, 2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, ZipLow_Predicates, -1, 2, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, UnzipEven_Predicates, -1, 2, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, UnzipOdd_Predicates, -1, 2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, TransposeEven_Predicates, -1, 2, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, TransposeOdd_Predicates, -1, 2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, ReverseElement_Predicates, -1, 1, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, And_Predicates, -1, 2, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_sve_and, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_EmbeddedMaskedOperation|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, BitwiseClear_Predicates, -1, 2, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_sve_bic, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_EmbeddedMaskedOperation|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, Or_Predicates, -1, 2, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_EmbeddedMaskedOperation|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, Xor_Predicates, -1, 2, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_sve_eor, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_EmbeddedMaskedOperation|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, ConditionalSelect_Predicates, -1, 3, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_ExplicitMaskedOperation|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, ZipHigh_Predicates, -1, 2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, INS_sve_zip2, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, ZipLow_Predicates, -1, 2, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, INS_sve_zip1, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, UnzipEven_Predicates, -1, 2, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, INS_sve_uzp1, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, UnzipOdd_Predicates, -1, 2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, INS_sve_uzp2, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, TransposeEven_Predicates, -1, 2, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, INS_sve_trn1, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, TransposeOdd_Predicates, -1, 2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, INS_sve_trn2, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, ReverseElement_Predicates, -1, 1, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, -1, -1, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) #endif // FEATURE_HW_INTRINSIC diff --git a/src/coreclr/jit/hwintrinsiclistwasm.h b/src/coreclr/jit/hwintrinsiclistwasm.h index 53cdb3824a40d1..ae0eb686c75edd 100644 --- a/src/coreclr/jit/hwintrinsiclistwasm.h +++ b/src/coreclr/jit/hwintrinsiclistwasm.h @@ -16,70 +16,70 @@ // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // PackedSimd Intrinsics #define FIRST_NI_PackedSimd NI_PackedSimd_Abs -HARDWARE_INTRINSIC(PackedSimd, Abs, 16, 1, INS_i8x16_abs, INS_invalid, INS_i16x8_abs, INS_invalid, INS_i32x4_abs, INS_invalid, INS_i64x2_abs, INS_invalid, INS_f32x4_abs, INS_f64x2_abs, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, Add, 16, 2, INS_i8x16_add, INS_i8x16_add, INS_i16x8_add, INS_i16x8_add, INS_i32x4_add, INS_i32x4_add, INS_i64x2_add, INS_i64x2_add, INS_f32x4_add, INS_f64x2_add, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(PackedSimd, AddPairwiseWidening, 16, 1, INS_i16x8_extadd_pairwise_s_i8x16, INS_i16x8_extadd_pairwise_u_i8x16, INS_i32x4_extadd_pairwise_s_i16x8, INS_i32x4_extadd_pairwise_u_i16x8, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, AddSaturate, 16, 2, INS_i8x16_add_sat_s, INS_i8x16_add_sat_u, INS_i16x8_add_sat_s, INS_i16x8_add_sat_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(PackedSimd, AllTrue, 16, 1, INS_i8x16_all_true, INS_i8x16_all_true, INS_i16x8_all_true, INS_i16x8_all_true, INS_i32x4_all_true, INS_i32x4_all_true, INS_i64x2_all_true, INS_i64x2_all_true, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, And, 16, 2, INS_v128_and, INS_v128_and, INS_v128_and, INS_v128_and, INS_v128_and, INS_v128_and, INS_v128_and, INS_v128_and, INS_v128_and, INS_v128_and, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(PackedSimd, AndNot, 16, 2, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, AnyTrue, 16, 1, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(PackedSimd, AverageRounded, 16, 2, INS_invalid, INS_i8x16_avgr_u, INS_invalid, INS_i16x8_avgr_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(PackedSimd, Bitmask, 16, 1, INS_i8x16_bitmask, INS_i8x16_bitmask, INS_i16x8_bitmask, INS_i16x8_bitmask, INS_i32x4_bitmask, INS_i32x4_bitmask, INS_i64x2_bitmask, INS_i64x2_bitmask, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, BitwiseSelect, 16, 3, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, Ceiling, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_ceil, INS_f64x2_ceil, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, CompareEqual, 16, 2, INS_i8x16_eq, INS_i8x16_eq, INS_i16x8_eq, INS_i16x8_eq, INS_i32x4_eq, INS_i32x4_eq, INS_i64x2_eq, INS_i64x2_eq, INS_f32x4_eq, INS_f64x2_eq, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(PackedSimd, CompareGreaterThan, 16, 2, INS_i8x16_gt_s, INS_i8x16_gt_u, INS_i16x8_gt_s, INS_i16x8_gt_u, INS_i32x4_gt_s, INS_i32x4_gt_u, INS_i64x2_gt_s, INS_invalid, INS_f32x4_gt, INS_f64x2_gt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(PackedSimd, CompareGreaterThanOrEqual, 16, 2, INS_i8x16_ge_s, INS_i8x16_ge_u, INS_i16x8_ge_s, INS_i16x8_ge_u, INS_i32x4_ge_s, INS_i32x4_ge_u, INS_i64x2_ge_s, INS_invalid, INS_f32x4_ge, INS_f64x2_ge, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(PackedSimd, CompareLessThan, 16, 2, INS_i8x16_lt_s, INS_i8x16_lt_u, INS_i16x8_lt_s, INS_i16x8_lt_u, INS_i32x4_lt_s, INS_i32x4_lt_u, INS_i64x2_lt_s, INS_invalid, INS_f32x4_lt, INS_f64x2_lt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(PackedSimd, CompareLessThanOrEqual, 16, 2, INS_i8x16_le_s, INS_i8x16_le_u, INS_i16x8_le_s, INS_i16x8_le_u, INS_i32x4_le_s, INS_i32x4_le_u, INS_i64x2_le_s, INS_invalid, INS_f32x4_le, INS_f64x2_le, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(PackedSimd, CompareNotEqual, 16, 2, INS_i8x16_ne, INS_i8x16_ne, INS_i16x8_ne, INS_i16x8_ne, INS_i32x4_ne, INS_i32x4_ne, INS_i64x2_ne, INS_i64x2_ne, INS_f32x4_ne, INS_f64x2_ne, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(PackedSimd, ConvertNarrowingSaturateSigned, 16, 2, INS_invalid, INS_invalid, INS_i8x16_narrow_i16x8_s, INS_invalid, INS_i16x8_narrow_i32x4_s, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, ConvertNarrowingSaturateUnsigned, 16, 2, INS_invalid, INS_invalid, INS_i8x16_narrow_i16x8_u, INS_invalid, INS_i16x8_narrow_i32x4_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, ConvertToDoubleLower, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f64x2_convert_low_s_i32x4, INS_f64x2_convert_low_u_i32x4, INS_invalid, INS_invalid, INS_f64x2_promote_low_f32x4, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, ConvertToInt32Saturate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_i32x4_trunc_sat_s_f32x4, INS_i32x4_trunc_sat_s_f64x2_zero, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, ConvertToSingle, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_convert_s_i32x4, INS_f32x4_convert_u_i32x4, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_demote_f64x2_zero, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, ConvertToUInt32Saturate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_i32x4_trunc_sat_u_f32x4, INS_i32x4_trunc_sat_u_f64x2_zero, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, Divide, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_div, INS_f64x2_div, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, Dot, 16, 2, INS_invalid, INS_invalid, INS_i32x4_dot_i16x8_s, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, ExtractScalar, 16, 2, INS_i8x16_extract_lane_s, INS_i8x16_extract_lane_u, INS_i16x8_extract_lane_s, INS_i16x8_extract_lane_u, INS_i32x4_extract_lane, INS_i32x4_extract_lane, INS_i64x2_extract_lane, INS_i64x2_extract_lane, INS_f32x4_extract_lane, INS_f64x2_extract_lane, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(PackedSimd, Floor, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_floor, INS_f64x2_floor, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndInsert, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndSplatVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(PackedSimd, LoadScalarVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(PackedSimd, LoadVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(PackedSimd, LoadWideningVector128, 16, 1, INS_v128_load8x8_s, INS_v128_load8x8_u, INS_v128_load16x4_s, INS_v128_load16x4_u, INS_v128_load32x2_s, INS_v128_load32x2_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(PackedSimd, Max, 16, 2, INS_i8x16_max_s, INS_i8x16_max_u, INS_i16x8_max_s, INS_i16x8_max_u, INS_i32x4_max_s, INS_i32x4_max_u, INS_invalid, INS_invalid, INS_f32x4_max, INS_f64x2_max, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(PackedSimd, Min, 16, 2, INS_i8x16_min_s, INS_i8x16_min_u, INS_i16x8_min_s, INS_i16x8_min_u, INS_i32x4_min_s, INS_i32x4_min_u, INS_invalid, INS_invalid, INS_f32x4_min, INS_f64x2_min, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(PackedSimd, Multiply, 16, 2, INS_invalid, INS_invalid, INS_i16x8_mul, INS_i16x8_mul, INS_i32x4_mul, INS_i32x4_mul, INS_i64x2_mul, INS_i64x2_mul, INS_f32x4_mul, INS_f64x2_mul, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(PackedSimd, MultiplyRoundedSaturateQ15, 16, 2, INS_invalid, INS_invalid, INS_i16x8_q15mulr_sat_s, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(PackedSimd, MultiplyWideningLower, 16, 2, INS_i16x8_extmul_low_s_i8x16, INS_i16x8_extmul_low_u_i8x16, INS_i32x4_extmul_low_s_i16x8, INS_i32x4_extmul_low_u_i16x8, INS_i64x2_extmul_low_s_i32x4, INS_i64x2_extmul_low_u_i32x4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, MultiplyWideningUpper, 16, 2, INS_i16x8_extmul_high_s_i8x16, INS_i16x8_extmul_high_u_i8x16, INS_i32x4_extmul_high_s_i16x8, INS_i32x4_extmul_high_u_i16x8, INS_i64x2_extmul_high_s_i32x4, INS_i64x2_extmul_high_u_i32x4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, Negate, 16, 1, INS_i8x16_neg, INS_i8x16_neg, INS_i16x8_neg, INS_i16x8_neg, INS_i32x4_neg, INS_i32x4_neg, INS_i64x2_neg, INS_i64x2_neg, INS_f32x4_neg, INS_f64x2_neg, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, Not, 16, 1, INS_v128_not, INS_v128_not, INS_v128_not, INS_v128_not, INS_v128_not, INS_v128_not, INS_v128_not, INS_v128_not, INS_v128_not, INS_v128_not, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, Or, 16, 2, INS_v128_or, INS_v128_or, INS_v128_or, INS_v128_or, INS_v128_or, INS_v128_or, INS_v128_or, INS_v128_or, INS_v128_or, INS_v128_or, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(PackedSimd, PopCount, 16, 1, INS_invalid, INS_i8x16_popcnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, PseudoMax, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_pmax, INS_f64x2_pmax, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, PseudoMin, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_pmin, INS_f64x2_pmin, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, ReplaceScalar, 16, 3, INS_i8x16_replace_lane, INS_i8x16_replace_lane, INS_i16x8_replace_lane, INS_i16x8_replace_lane, INS_i32x4_replace_lane, INS_i32x4_replace_lane, INS_i64x2_replace_lane, INS_i64x2_replace_lane, INS_f32x4_replace_lane, INS_f64x2_replace_lane, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(PackedSimd, RoundToNearest, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_nearest, INS_f64x2_nearest, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, ShiftLeft, 16, 2, INS_i8x16_shl, INS_i8x16_shl, INS_i16x8_shl, INS_i16x8_shl, INS_i32x4_shl, INS_i32x4_shl, INS_i64x2_shl, INS_i64x2_shl, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(PackedSimd, ShiftRightArithmetic, 16, 2, INS_i8x16_shr_s, INS_i8x16_shr_s, INS_i16x8_shr_s, INS_i16x8_shr_s, INS_i32x4_shr_s, INS_i32x4_shr_s, INS_i64x2_shr_s, INS_i64x2_shr_s, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(PackedSimd, ShiftRightLogical, 16, 2, INS_i8x16_shr_u, INS_i8x16_shr_u, INS_i16x8_shr_u, INS_i16x8_shr_u, INS_i32x4_shr_u, INS_i32x4_shr_u, INS_i64x2_shr_u, INS_i64x2_shr_u, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(PackedSimd, Shuffle, 16, 3, INS_i8x16_shuffle, INS_i8x16_shuffle, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, SignExtendWideningLower, 16, 1, INS_i16x8_extend_low_s_i8x16, INS_i16x8_extend_low_s_i8x16, INS_i32x4_extend_low_s_i16x8, INS_i32x4_extend_low_s_i16x8, INS_i64x2_extend_low_s_i32x4, INS_i64x2_extend_low_s_i32x4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, SignExtendWideningUpper, 16, 1, INS_i16x8_extend_high_s_i8x16, INS_i16x8_extend_high_s_i8x16, INS_i32x4_extend_high_s_i16x8, INS_i32x4_extend_high_s_i16x8, INS_i64x2_extend_high_s_i32x4, INS_i64x2_extend_high_s_i32x4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, Splat, 16, 1, INS_i8x16_splat, INS_i8x16_splat, INS_i16x8_splat, INS_i16x8_splat, INS_i32x4_splat, INS_i32x4_splat, INS_i64x2_splat, INS_i64x2_splat, INS_f32x4_splat, INS_f64x2_splat, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, Sqrt, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_sqrt, INS_f64x2_sqrt, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, Store, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(PackedSimd, StoreSelectedScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(PackedSimd, Subtract, 16, 2, INS_i8x16_sub, INS_i8x16_sub, INS_i16x8_sub, INS_i16x8_sub, INS_i32x4_sub, INS_i32x4_sub, INS_i64x2_sub, INS_i64x2_sub, INS_f32x4_sub, INS_f64x2_sub, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, SubtractSaturate, 16, 2, INS_i8x16_sub_sat_s, INS_i8x16_sub_sat_u, INS_i16x8_sub_sat_s, INS_i16x8_sub_sat_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, Swizzle, 16, 2, INS_i8x16_swizzle, INS_i8x16_swizzle, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, Truncate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_trunc, INS_f64x2_trunc, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, Xor, 16, 2, INS_v128_xor, INS_v128_xor, INS_v128_xor, INS_v128_xor, INS_v128_xor, INS_v128_xor, INS_v128_xor, INS_v128_xor, INS_v128_xor, INS_v128_xor, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) -HARDWARE_INTRINSIC(PackedSimd, ZeroExtendWideningLower, 16, 1, INS_i16x8_extend_low_u_i8x16, INS_i16x8_extend_low_u_i8x16, INS_i32x4_extend_low_u_i16x8, INS_i32x4_extend_low_u_i16x8, INS_i64x2_extend_low_u_i32x4, INS_i64x2_extend_low_u_i32x4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, ZeroExtendWideningUpper, 16, 1, INS_i16x8_extend_high_u_i8x16, INS_i16x8_extend_high_u_i8x16, INS_i32x4_extend_high_u_i16x8, INS_i32x4_extend_high_u_i16x8, INS_i64x2_extend_high_u_i32x4, INS_i64x2_extend_high_u_i32x4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, Abs, 16, 1, INS_i8x16_abs, INS_invalid, INS_i16x8_abs, INS_invalid, INS_i32x4_abs, INS_invalid, INS_i64x2_abs, INS_invalid, INS_f32x4_abs, INS_f64x2_abs, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, Add, 16, 2, INS_i8x16_add, INS_i8x16_add, INS_i16x8_add, INS_i16x8_add, INS_i32x4_add, INS_i32x4_add, INS_i64x2_add, INS_i64x2_add, INS_f32x4_add, INS_f64x2_add, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(PackedSimd, AddPairwiseWidening, 16, 1, INS_i16x8_extadd_pairwise_s_i8x16, INS_i16x8_extadd_pairwise_u_i8x16, INS_i32x4_extadd_pairwise_s_i16x8, INS_i32x4_extadd_pairwise_u_i16x8, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, AddSaturate, 16, 2, INS_i8x16_add_sat_s, INS_i8x16_add_sat_u, INS_i16x8_add_sat_s, INS_i16x8_add_sat_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(PackedSimd, AllTrue, 16, 1, INS_i8x16_all_true, INS_i8x16_all_true, INS_i16x8_all_true, INS_i16x8_all_true, INS_i32x4_all_true, INS_i32x4_all_true, INS_i64x2_all_true, INS_i64x2_all_true, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, And, 16, 2, INS_v128_and, INS_v128_and, INS_v128_and, INS_v128_and, INS_v128_and, INS_v128_and, INS_v128_and, INS_v128_and, INS_v128_and, INS_v128_and, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(PackedSimd, AndNot, 16, 2, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, INS_v128_andnot, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, AnyTrue, 16, 1, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, INS_v128_any_true, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(PackedSimd, AverageRounded, 16, 2, INS_invalid, INS_i8x16_avgr_u, INS_invalid, INS_i16x8_avgr_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(PackedSimd, Bitmask, 16, 1, INS_i8x16_bitmask, INS_i8x16_bitmask, INS_i16x8_bitmask, INS_i16x8_bitmask, INS_i32x4_bitmask, INS_i32x4_bitmask, INS_i64x2_bitmask, INS_i64x2_bitmask, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, BitwiseSelect, 16, 3, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, INS_v128_bitselect, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, Ceiling, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_ceil, INS_f64x2_ceil, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, CompareEqual, 16, 2, INS_i8x16_eq, INS_i8x16_eq, INS_i16x8_eq, INS_i16x8_eq, INS_i32x4_eq, INS_i32x4_eq, INS_i64x2_eq, INS_i64x2_eq, INS_f32x4_eq, INS_f64x2_eq, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(PackedSimd, CompareGreaterThan, 16, 2, INS_i8x16_gt_s, INS_i8x16_gt_u, INS_i16x8_gt_s, INS_i16x8_gt_u, INS_i32x4_gt_s, INS_i32x4_gt_u, INS_i64x2_gt_s, INS_invalid, INS_f32x4_gt, INS_f64x2_gt, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(PackedSimd, CompareGreaterThanOrEqual, 16, 2, INS_i8x16_ge_s, INS_i8x16_ge_u, INS_i16x8_ge_s, INS_i16x8_ge_u, INS_i32x4_ge_s, INS_i32x4_ge_u, INS_i64x2_ge_s, INS_invalid, INS_f32x4_ge, INS_f64x2_ge, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(PackedSimd, CompareLessThan, 16, 2, INS_i8x16_lt_s, INS_i8x16_lt_u, INS_i16x8_lt_s, INS_i16x8_lt_u, INS_i32x4_lt_s, INS_i32x4_lt_u, INS_i64x2_lt_s, INS_invalid, INS_f32x4_lt, INS_f64x2_lt, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(PackedSimd, CompareLessThanOrEqual, 16, 2, INS_i8x16_le_s, INS_i8x16_le_u, INS_i16x8_le_s, INS_i16x8_le_u, INS_i32x4_le_s, INS_i32x4_le_u, INS_i64x2_le_s, INS_invalid, INS_f32x4_le, INS_f64x2_le, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(PackedSimd, CompareNotEqual, 16, 2, INS_i8x16_ne, INS_i8x16_ne, INS_i16x8_ne, INS_i16x8_ne, INS_i32x4_ne, INS_i32x4_ne, INS_i64x2_ne, INS_i64x2_ne, INS_f32x4_ne, INS_f64x2_ne, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(PackedSimd, ConvertNarrowingSaturateSigned, 16, 2, INS_invalid, INS_invalid, INS_i8x16_narrow_i16x8_s, INS_invalid, INS_i16x8_narrow_i32x4_s, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, ConvertNarrowingSaturateUnsigned, 16, 2, INS_invalid, INS_invalid, INS_i8x16_narrow_i16x8_u, INS_invalid, INS_i16x8_narrow_i32x4_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, ConvertToDoubleLower, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f64x2_convert_low_s_i32x4, INS_f64x2_convert_low_u_i32x4, INS_invalid, INS_invalid, INS_f64x2_promote_low_f32x4, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, ConvertToInt32Saturate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_i32x4_trunc_sat_s_f32x4, INS_i32x4_trunc_sat_s_f64x2_zero, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, ConvertToSingle, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_convert_s_i32x4, INS_f32x4_convert_u_i32x4, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_demote_f64x2_zero, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, ConvertToUInt32Saturate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_i32x4_trunc_sat_u_f32x4, INS_i32x4_trunc_sat_u_f64x2_zero, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, Divide, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_div, INS_f64x2_div, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, Dot, 16, 2, INS_invalid, INS_invalid, INS_i32x4_dot_i16x8_s, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, ExtractScalar, 16, 2, INS_i8x16_extract_lane_s, INS_i8x16_extract_lane_u, INS_i16x8_extract_lane_s, INS_i16x8_extract_lane_u, INS_i32x4_extract_lane, INS_i32x4_extract_lane, INS_i64x2_extract_lane, INS_i64x2_extract_lane, INS_f32x4_extract_lane, INS_f64x2_extract_lane, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(PackedSimd, Floor, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_floor, INS_f64x2_floor, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndInsert, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndSplatVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(PackedSimd, LoadScalarVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(PackedSimd, LoadVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(PackedSimd, LoadWideningVector128, 16, 1, INS_v128_load8x8_s, INS_v128_load8x8_u, INS_v128_load16x4_s, INS_v128_load16x4_u, INS_v128_load32x2_s, INS_v128_load32x2_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(PackedSimd, Max, 16, 2, INS_i8x16_max_s, INS_i8x16_max_u, INS_i16x8_max_s, INS_i16x8_max_u, INS_i32x4_max_s, INS_i32x4_max_u, INS_invalid, INS_invalid, INS_f32x4_max, INS_f64x2_max, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(PackedSimd, Min, 16, 2, INS_i8x16_min_s, INS_i8x16_min_u, INS_i16x8_min_s, INS_i16x8_min_u, INS_i32x4_min_s, INS_i32x4_min_u, INS_invalid, INS_invalid, INS_f32x4_min, INS_f64x2_min, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(PackedSimd, Multiply, 16, 2, INS_invalid, INS_invalid, INS_i16x8_mul, INS_i16x8_mul, INS_i32x4_mul, INS_i32x4_mul, INS_i64x2_mul, INS_i64x2_mul, INS_f32x4_mul, INS_f64x2_mul, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(PackedSimd, MultiplyRoundedSaturateQ15, 16, 2, INS_invalid, INS_invalid, INS_i16x8_q15mulr_sat_s, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(PackedSimd, MultiplyWideningLower, 16, 2, INS_i16x8_extmul_low_s_i8x16, INS_i16x8_extmul_low_u_i8x16, INS_i32x4_extmul_low_s_i16x8, INS_i32x4_extmul_low_u_i16x8, INS_i64x2_extmul_low_s_i32x4, INS_i64x2_extmul_low_u_i32x4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, MultiplyWideningUpper, 16, 2, INS_i16x8_extmul_high_s_i8x16, INS_i16x8_extmul_high_u_i8x16, INS_i32x4_extmul_high_s_i16x8, INS_i32x4_extmul_high_u_i16x8, INS_i64x2_extmul_high_s_i32x4, INS_i64x2_extmul_high_u_i32x4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, Negate, 16, 1, INS_i8x16_neg, INS_i8x16_neg, INS_i16x8_neg, INS_i16x8_neg, INS_i32x4_neg, INS_i32x4_neg, INS_i64x2_neg, INS_i64x2_neg, INS_f32x4_neg, INS_f64x2_neg, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, Not, 16, 1, INS_v128_not, INS_v128_not, INS_v128_not, INS_v128_not, INS_v128_not, INS_v128_not, INS_v128_not, INS_v128_not, INS_v128_not, INS_v128_not, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, Or, 16, 2, INS_v128_or, INS_v128_or, INS_v128_or, INS_v128_or, INS_v128_or, INS_v128_or, INS_v128_or, INS_v128_or, INS_v128_or, INS_v128_or, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(PackedSimd, PopCount, 16, 1, INS_invalid, INS_i8x16_popcnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, PseudoMax, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_pmax, INS_f64x2_pmax, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, PseudoMin, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_pmin, INS_f64x2_pmin, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, ReplaceScalar, 16, 3, INS_i8x16_replace_lane, INS_i8x16_replace_lane, INS_i16x8_replace_lane, INS_i16x8_replace_lane, INS_i32x4_replace_lane, INS_i32x4_replace_lane, INS_i64x2_replace_lane, INS_i64x2_replace_lane, INS_f32x4_replace_lane, INS_f64x2_replace_lane, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(PackedSimd, RoundToNearest, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_nearest, INS_f64x2_nearest, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, ShiftLeft, 16, 2, INS_i8x16_shl, INS_i8x16_shl, INS_i16x8_shl, INS_i16x8_shl, INS_i32x4_shl, INS_i32x4_shl, INS_i64x2_shl, INS_i64x2_shl, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(PackedSimd, ShiftRightArithmetic, 16, 2, INS_i8x16_shr_s, INS_i8x16_shr_s, INS_i16x8_shr_s, INS_i16x8_shr_s, INS_i32x4_shr_s, INS_i32x4_shr_s, INS_i64x2_shr_s, INS_i64x2_shr_s, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(PackedSimd, ShiftRightLogical, 16, 2, INS_i8x16_shr_u, INS_i8x16_shr_u, INS_i16x8_shr_u, INS_i16x8_shr_u, INS_i32x4_shr_u, INS_i32x4_shr_u, INS_i64x2_shr_u, INS_i64x2_shr_u, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(PackedSimd, Shuffle, 16, 3, INS_i8x16_shuffle, INS_i8x16_shuffle, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, SignExtendWideningLower, 16, 1, INS_i16x8_extend_low_s_i8x16, INS_i16x8_extend_low_s_i8x16, INS_i32x4_extend_low_s_i16x8, INS_i32x4_extend_low_s_i16x8, INS_i64x2_extend_low_s_i32x4, INS_i64x2_extend_low_s_i32x4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, SignExtendWideningUpper, 16, 1, INS_i16x8_extend_high_s_i8x16, INS_i16x8_extend_high_s_i8x16, INS_i32x4_extend_high_s_i16x8, INS_i32x4_extend_high_s_i16x8, INS_i64x2_extend_high_s_i32x4, INS_i64x2_extend_high_s_i32x4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, Splat, 16, 1, INS_i8x16_splat, INS_i8x16_splat, INS_i16x8_splat, INS_i16x8_splat, INS_i32x4_splat, INS_i32x4_splat, INS_i64x2_splat, INS_i64x2_splat, INS_f32x4_splat, INS_f64x2_splat, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, Sqrt, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_sqrt, INS_f64x2_sqrt, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, Store, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(PackedSimd, StoreSelectedScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(PackedSimd, Subtract, 16, 2, INS_i8x16_sub, INS_i8x16_sub, INS_i16x8_sub, INS_i16x8_sub, INS_i32x4_sub, INS_i32x4_sub, INS_i64x2_sub, INS_i64x2_sub, INS_f32x4_sub, INS_f64x2_sub, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, SubtractSaturate, 16, 2, INS_i8x16_sub_sat_s, INS_i8x16_sub_sat_u, INS_i16x8_sub_sat_s, INS_i16x8_sub_sat_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, Swizzle, 16, 2, INS_i8x16_swizzle, INS_i8x16_swizzle, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, Truncate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_trunc, INS_f64x2_trunc, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, Xor, 16, 2, INS_v128_xor, INS_v128_xor, INS_v128_xor, INS_v128_xor, INS_v128_xor, INS_v128_xor, INS_v128_xor, INS_v128_xor, INS_v128_xor, INS_v128_xor, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) +HARDWARE_INTRINSIC(PackedSimd, ZeroExtendWideningLower, 16, 1, INS_i16x8_extend_low_u_i8x16, INS_i16x8_extend_low_u_i8x16, INS_i32x4_extend_low_u_i16x8, INS_i32x4_extend_low_u_i16x8, INS_i64x2_extend_low_u_i32x4, INS_i64x2_extend_low_u_i32x4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, ZeroExtendWideningUpper, 16, 1, INS_i16x8_extend_high_u_i8x16, INS_i16x8_extend_high_u_i8x16, INS_i32x4_extend_high_u_i16x8, INS_i32x4_extend_high_u_i16x8, INS_i64x2_extend_high_u_i32x4, INS_i64x2_extend_high_u_i32x4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) #define LAST_NI_PackedSimd NI_PackedSimd_ZeroExtendWideningUpper #endif // FEATURE_HW_INTRINSICS diff --git a/src/coreclr/jit/hwintrinsiclistxarch.h b/src/coreclr/jit/hwintrinsiclistxarch.h index cb755c6af754a2..195a0de7b6c232 100644 --- a/src/coreclr/jit/hwintrinsiclistxarch.h +++ b/src/coreclr/jit/hwintrinsiclistxarch.h @@ -28,165 +28,165 @@ // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Intrinsics for X86Base, SSE, SSE2, SSE3, SSSE3, SSE41, SSE42, POPCNT #define FIRST_NI_X86Base NI_X86Base_Abs -HARDWARE_INTRINSIC(X86Base, Abs, 16, 1, INS_pabsb, INS_invalid, INS_pabsw, INS_invalid, INS_pabsd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, Add, 16, 2, INS_paddb, INS_paddb, INS_paddw, INS_paddw, INS_paddd, INS_paddd, INS_paddq, INS_paddq, INS_addps, INS_addpd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(X86Base, AddSaturate, 16, 2, INS_paddsb, INS_paddusb, INS_paddsw, INS_paddusw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(X86Base, AddScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_addss, INS_addsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(X86Base, AddSubtract, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_addsubps, INS_addsubpd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, AlignRight, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_palignr, INS_palignr, INS_palignr, INS_palignr, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(X86Base, And, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pandd, INS_pandd, INS_pandd, INS_pandd, INS_andps, INS_andpd, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(X86Base, AndNot, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pandnd, INS_pandnd, INS_pandnd, INS_pandnd, INS_andnps, INS_andnpd, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(X86Base, Average, 16, 2, INS_invalid, INS_pavgb, INS_invalid, INS_pavgw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(X86Base, BitScanForward, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bsf, INS_bsf, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, BitScanReverse, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bsr, INS_bsr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, Blend, 16, 3, INS_invalid, INS_invalid, INS_pblendw, INS_pblendw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blendps, INS_blendpd, INS_invalid, 1, 1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, BlendVariable, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pblendvb, INS_pblendvb, INS_pblendvb, INS_pblendvb, INS_blendvps, INS_blendvpd, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(X86Base, Ceiling, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, INS_invalid, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, CeilingScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundss, INS_roundsd, INS_invalid, -1, 8, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(X86Base, CompareEqual, 16, 2, INS_pcmpeqb, INS_pcmpeqb, INS_pcmpeqw, INS_pcmpeqw, INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqq, INS_pcmpeqq, INS_cmpps, INS_cmppd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(X86Base, CompareGreaterThan, 16, 2, INS_pcmpgtb, INS_invalid, INS_pcmpgtw, INS_invalid, INS_pcmpgtd, INS_invalid, INS_pcmpgtq, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(X86Base, CompareGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(X86Base, CompareLessThan, 16, 2, INS_pcmpgtb, INS_invalid, INS_pcmpgtw, INS_invalid, INS_pcmpgtd, INS_invalid, INS_pcmpgtq, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(X86Base, CompareLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(X86Base, CompareNotEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(X86Base, CompareNotGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(X86Base, CompareNotGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(X86Base, CompareNotLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(X86Base, CompareNotLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(X86Base, CompareOrdered, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(X86Base, CompareScalarEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, CompareScalarGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, CompareScalarGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, CompareScalarLessThan, 16, 2, INS_pcmpgtb, INS_invalid, INS_pcmpgtw, INS_invalid, INS_pcmpgtd, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, INS_invalid, 1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, CompareScalarLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, CompareScalarNotEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, CompareScalarNotGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, CompareScalarNotGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, CompareScalarNotLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, CompareScalarNotLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, CompareScalarOrdered, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, CompareScalarOrderedEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_comiss, INS_comisd, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_Commutative|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(X86Base, CompareScalarOrderedGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_comiss, INS_comisd, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(X86Base, CompareScalarOrderedGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_comiss, INS_comisd, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(X86Base, CompareScalarOrderedLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_comiss, INS_comisd, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(X86Base, CompareScalarOrderedLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_comiss, INS_comisd, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(X86Base, CompareScalarOrderedNotEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_comiss, INS_comisd, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_Commutative|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(X86Base, CompareScalarUnordered, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, CompareScalarUnorderedEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ucomiss, INS_ucomisd, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_Commutative|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(X86Base, CompareScalarUnorderedGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ucomiss, INS_ucomisd, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(X86Base, CompareScalarUnorderedGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ucomiss, INS_ucomisd, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(X86Base, CompareScalarUnorderedLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ucomiss, INS_ucomisd, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(X86Base, CompareScalarUnorderedLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ucomiss, INS_ucomisd, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(X86Base, CompareScalarUnorderedNotEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ucomiss, INS_ucomisd, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_Commutative|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(X86Base, CompareUnordered, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(X86Base, ConvertScalarToVector128Double, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2sd32, INS_invalid, INS_invalid, INS_invalid, INS_cvtss2sd, INS_invalid, INS_invalid, 5, 4, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(X86Base, ConvertScalarToVector128Int32, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, ConvertScalarToVector128Single, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2ss32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsd2ss, INS_invalid, 5, 4, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(X86Base, ConvertScalarToVector128UInt32, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, ConvertToInt32, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_invalid, INS_invalid, INS_invalid, INS_cvtss2si32, INS_cvtsd2si32, INS_invalid, 3, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, ConvertToInt32WithTruncation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvttss2si32, INS_cvttsd2si32, INS_invalid, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, ConvertToUInt32, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, ConvertToVector128Double, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtdq2pd, INS_invalid, INS_invalid, INS_invalid, INS_cvtps2pd, INS_invalid, INS_invalid, 5, 5, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, ConvertToVector128Int16, 16, 1, INS_pmovsxbw, INS_pmovzxbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_MaybeMemoryLoad) -HARDWARE_INTRINSIC(X86Base, ConvertToVector128Int32, 16, 1, INS_pmovsxbd, INS_pmovzxbd, INS_pmovsxwd, INS_pmovzxwd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtps2dq, INS_cvtpd2dq, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_MaybeMemoryLoad) -HARDWARE_INTRINSIC(X86Base, ConvertToVector128Int32WithTruncation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvttps2dq, INS_cvttpd2dq, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, ConvertToVector128Int64, 16, 1, INS_pmovsxbq, INS_pmovzxbq, INS_pmovsxwq, INS_pmovzxwq, INS_pmovsxdq, INS_pmovzxdq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_MaybeMemoryLoad) -HARDWARE_INTRINSIC(X86Base, ConvertToVector128Single, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtdq2ps, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtpd2ps, INS_invalid, 4, 5, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, Crc32, 0, 2, INS_invalid, INS_crc32, INS_invalid, INS_crc32, INS_invalid, INS_crc32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(X86Base, DivRem, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_idiv, INS_div, INS_idiv, INS_div, INS_invalid, INS_invalid, INS_invalid, 25, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_BaseTypeFromSecondArg|HW_Flag_MultiReg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(X86Base, Divide, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_divps, INS_divpd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(X86Base, DivideScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_divss, INS_divsd, INS_invalid, -1, -1, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(X86Base, DotProduct, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_dpps, INS_dppd, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, Extract, 16, 2, INS_pextrb, INS_pextrb, INS_pextrw, INS_pextrw, INS_pextrd, INS_pextrd, INS_invalid, INS_invalid, INS_extractps, INS_invalid, INS_invalid, 4, 4, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsScalarT) -HARDWARE_INTRINSIC(X86Base, Floor, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, INS_invalid, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, FloorScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundss, INS_roundsd, INS_invalid, -1, 8, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(X86Base, HorizontalAdd, 16, 2, INS_invalid, INS_invalid, INS_phaddw, INS_phaddw, INS_phaddd, INS_phaddd, INS_invalid, INS_invalid, INS_haddps, INS_haddpd, INS_invalid, 3, 6, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, HorizontalAddSaturate, 16, 2, INS_invalid, INS_invalid, INS_phaddsw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, HorizontalSubtract, 16, 2, INS_invalid, INS_invalid, INS_phsubw, INS_invalid, INS_phsubd, INS_invalid, INS_invalid, INS_invalid, INS_hsubps, INS_hsubpd, INS_invalid, 3, 6, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, HorizontalSubtractSaturate, 16, 2, INS_invalid, INS_invalid, INS_phsubsw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, Insert, 16, 3, INS_pinsrb, INS_pinsrb, INS_pinsrw, INS_pinsrw, INS_pinsrd, INS_pinsrd, INS_invalid, INS_invalid, INS_insertps, INS_invalid, INS_invalid, 4, 1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(X86Base, LoadAlignedVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movaps, INS_movapd, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoRMWSemantics|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(X86Base, LoadAlignedVector128NonTemporal, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movntdqa, INS_movntdqa, INS_movntdqa, INS_movntdqa, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoRMWSemantics|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(X86Base, LoadAndDuplicateToVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movddup, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, LoadDquVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_lddqu, INS_lddqu, INS_lddqu, INS_lddqu, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoRMWSemantics|HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(X86Base, LoadFence, 0, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Barrier) -HARDWARE_INTRINSIC(X86Base, LoadHigh, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movhps, INS_movhpd, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, LoadLow, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movlps, INS_movlpd, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, LoadScalarVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_movd32, INS_movq, INS_movq, INS_movss, INS_movsd_simd, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, LoadVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(X86Base, MaskMove, 16, 3, INS_maskmovdqu, INS_maskmovdqu, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(X86Base, Max, 16, 2, INS_pmaxsb, INS_pmaxub, INS_pmaxsw, INS_pmaxuw, INS_pmaxsd, INS_pmaxud, INS_invalid, INS_invalid, INS_maxps, INS_maxpd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative) -HARDWARE_INTRINSIC(X86Base, MaxScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_maxss, INS_maxsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(X86Base, MemoryFence, 0, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Barrier) -HARDWARE_INTRINSIC(X86Base, Min, 16, 2, INS_pminsb, INS_pminub, INS_pminsw, INS_pminuw, INS_pminsd, INS_pminud, INS_invalid, INS_invalid, INS_minps, INS_minpd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative) -HARDWARE_INTRINSIC(X86Base, MinHorizontal, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_phminposuw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, MinScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_minss, INS_minsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(X86Base, MoveAndDuplicate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movddup, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, MoveHighAndDuplicate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movshdup, INS_invalid, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, MoveHighToLow, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movhlps, INS_invalid, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment) -HARDWARE_INTRINSIC(X86Base, MoveLowAndDuplicate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movsldup, INS_invalid, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, MoveLowToHigh, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movlhps, INS_invalid, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment) -HARDWARE_INTRINSIC(X86Base, MoveMask, 16, 1, INS_pmovmskb, INS_pmovmskb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movmskps, INS_movmskpd, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, MoveScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movq, INS_movq, INS_movss, INS_movsd_simd, INS_invalid, 1, 1, HW_Category_SIMDScalar, HW_Flag_NoContainment) -HARDWARE_INTRINSIC(X86Base, MultipleSumAbsoluteDifferences, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_mpsadbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, Multiply, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pmuldq, INS_pmuludq, INS_mulps, INS_mulpd, INS_invalid, 5, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(X86Base, MultiplyAddAdjacent, 16, 2, INS_invalid, INS_invalid, INS_pmaddubsw, INS_invalid, INS_pmaddwd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative) -HARDWARE_INTRINSIC(X86Base, MultiplyHigh, 16, 2, INS_invalid, INS_invalid, INS_pmulhw, INS_pmulhuw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(X86Base, MultiplyHighRoundScale, 16, 2, INS_invalid, INS_invalid, INS_pmulhrsw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(X86Base, MultiplyLow, 16, 2, INS_invalid, INS_invalid, INS_pmullw, INS_pmullw, INS_pmulld, INS_pmulld, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(X86Base, MultiplyScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mulss, INS_mulsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(X86Base, Or, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pord, INS_pord, INS_pord, INS_pord, INS_orps, INS_orpd, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(X86Base, PackSignedSaturate, 16, 2, INS_packsswb, INS_invalid, INS_packssdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(X86Base, PackUnsignedSaturate, 16, 2, INS_invalid, INS_packuswb, INS_invalid, INS_packusdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(X86Base, Pause, 0, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(X86Base, PopCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_popcnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(X86Base, Prefetch0, 0, 1, INS_invalid, INS_prefetcht0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(X86Base, Prefetch1, 0, 1, INS_invalid, INS_prefetcht1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(X86Base, Prefetch2, 0, 1, INS_invalid, INS_prefetcht2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(X86Base, PrefetchNonTemporal, 0, 1, INS_invalid, INS_prefetchnta, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Other) -HARDWARE_INTRINSIC(X86Base, Reciprocal, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rcpps, INS_invalid, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, ReciprocalScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rcpss, INS_invalid, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, ReciprocalSqrt, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rsqrtps, INS_invalid, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, ReciprocalSqrtScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rsqrtss, INS_invalid, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, RoundCurrentDirection, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, INS_invalid, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, RoundCurrentDirectionScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundss, INS_roundsd, INS_invalid, -1, 8, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(X86Base, RoundToNearestInteger, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, INS_invalid, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, RoundToNearestIntegerScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundss, INS_roundsd, INS_invalid, -1, 8, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(X86Base, RoundToNegativeInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, INS_invalid, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, RoundToNegativeInfinityScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundss, INS_roundsd, INS_invalid, -1, 8, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(X86Base, RoundToPositiveInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, INS_invalid, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, RoundToPositiveInfinityScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundss, INS_roundsd, INS_invalid, -1, 8, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(X86Base, RoundToZero, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, INS_invalid, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, RoundToZeroScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundss, INS_roundsd, INS_invalid, -1, 8, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(X86Base, ShiftLeftLogical, 16, 2, INS_invalid, INS_invalid, INS_psllw, INS_psllw, INS_pslld, INS_pslld, INS_psllq, INS_psllq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(X86Base, ShiftLeftLogical128BitLane, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pslldq, INS_pslldq, INS_pslldq, INS_pslldq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(X86Base, ShiftRightArithmetic, 16, 2, INS_invalid, INS_invalid, INS_psraw, INS_invalid, INS_psrad, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(X86Base, ShiftRightLogical, 16, 2, INS_invalid, INS_invalid, INS_psrlw, INS_psrlw, INS_psrld, INS_psrld, INS_psrlq, INS_psrlq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(X86Base, ShiftRightLogical128BitLane, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_psrldq, INS_psrldq, INS_psrldq, INS_psrldq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(X86Base, Shuffle, 16, -1, INS_pshufb, INS_pshufb, INS_invalid, INS_invalid, INS_pshufd, INS_pshufd, INS_invalid, INS_invalid, INS_shufps, INS_shufpd, INS_invalid, 1, 1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(X86Base, ShuffleHigh, 16, 2, INS_invalid, INS_invalid, INS_pshufhw, INS_pshufhw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(X86Base, ShuffleLow, 16, 2, INS_invalid, INS_invalid, INS_pshuflw, INS_pshuflw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(X86Base, Sign, 16, 2, INS_psignb, INS_invalid, INS_psignw, INS_invalid, INS_psignd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, Sqrt, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqrtps, INS_sqrtpd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, SqrtScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqrtss, INS_sqrtsd, INS_invalid, -1, -1, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(X86Base, Store, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(X86Base, StoreAligned, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movaps, INS_movapd, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromSecondArg|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(X86Base, StoreAlignedNonTemporal, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movntdq, INS_movntdq, INS_movntdq, INS_movntdq, INS_movntps, INS_movntpd, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromSecondArg|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(X86Base, StoreFence, 0, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Barrier) -HARDWARE_INTRINSIC(X86Base, StoreHigh, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movhps, INS_movhpd, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(X86Base, StoreLow, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movlps, INS_movlpd, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(X86Base, StoreNonTemporal, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movnti32, INS_movnti32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(X86Base, StoreScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_movd32, INS_movq, INS_movq, INS_movss, INS_movsd_simd, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(X86Base, Subtract, 16, 2, INS_psubb, INS_psubb, INS_psubw, INS_psubw, INS_psubd, INS_psubd, INS_psubq, INS_psubq, INS_subps, INS_subpd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(X86Base, SubtractSaturate, 16, 2, INS_psubsb, INS_psubusb, INS_psubsw, INS_psubusw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(X86Base, SubtractScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_subss, INS_subsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(X86Base, SumAbsoluteDifferences, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_psadbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(X86Base, TestC, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(X86Base, TestNotZAndNotC, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(X86Base, TestZ, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(X86Base, UnpackHigh, 16, 2, INS_punpckhbw, INS_punpckhbw, INS_punpckhwd, INS_punpckhwd, INS_punpckhdq, INS_punpckhdq, INS_punpckhqdq, INS_punpckhqdq, INS_unpckhps, INS_unpckhpd, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(X86Base, UnpackLow, 16, 2, INS_punpcklbw, INS_punpcklbw, INS_punpcklwd, INS_punpcklwd, INS_punpckldq, INS_punpckldq, INS_punpcklqdq, INS_punpcklqdq, INS_unpcklps, INS_unpcklpd, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(X86Base, Xor, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pxord, INS_pxord, INS_pxord, INS_pxord, INS_xorps, INS_xorpd, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(X86Base, Abs, 16, 1, INS_pabsb, INS_invalid, INS_pabsw, INS_invalid, INS_pabsd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, Add, 16, 2, INS_paddb, INS_paddb, INS_paddw, INS_paddw, INS_paddd, INS_paddd, INS_paddq, INS_paddq, INS_addps, INS_addpd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(X86Base, AddSaturate, 16, 2, INS_paddsb, INS_paddusb, INS_paddsw, INS_paddusw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(X86Base, AddScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_addss, INS_addsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(X86Base, AddSubtract, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_addsubps, INS_addsubpd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, AlignRight, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_palignr, INS_palignr, INS_palignr, INS_palignr, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(X86Base, And, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pandd, INS_pandd, INS_pandd, INS_pandd, INS_andps, INS_andpd, 1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(X86Base, AndNot, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pandnd, INS_pandnd, INS_pandnd, INS_pandnd, INS_andnps, INS_andnpd, 1, 1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(X86Base, Average, 16, 2, INS_invalid, INS_pavgb, INS_invalid, INS_pavgw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(X86Base, BitScanForward, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bsf, INS_bsf, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, BitScanReverse, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bsr, INS_bsr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, Blend, 16, 3, INS_invalid, INS_invalid, INS_pblendw, INS_pblendw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blendps, INS_blendpd, 1, 1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, BlendVariable, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pblendvb, INS_pblendvb, INS_pblendvb, INS_pblendvb, INS_blendvps, INS_blendvpd, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(X86Base, Ceiling, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, CeilingScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundss, INS_roundsd, -1, 8, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(X86Base, CompareEqual, 16, 2, INS_pcmpeqb, INS_pcmpeqb, INS_pcmpeqw, INS_pcmpeqw, INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqq, INS_pcmpeqq, INS_cmpps, INS_cmppd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(X86Base, CompareGreaterThan, 16, 2, INS_pcmpgtb, INS_invalid, INS_pcmpgtw, INS_invalid, INS_pcmpgtd, INS_invalid, INS_pcmpgtq, INS_invalid, INS_cmpps, INS_cmppd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(X86Base, CompareGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(X86Base, CompareLessThan, 16, 2, INS_pcmpgtb, INS_invalid, INS_pcmpgtw, INS_invalid, INS_pcmpgtd, INS_invalid, INS_pcmpgtq, INS_invalid, INS_cmpps, INS_cmppd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(X86Base, CompareLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(X86Base, CompareNotEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(X86Base, CompareNotGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(X86Base, CompareNotGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(X86Base, CompareNotLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(X86Base, CompareNotLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(X86Base, CompareOrdered, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(X86Base, CompareScalarEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, CompareScalarGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, CompareScalarGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, CompareScalarLessThan, 16, 2, INS_pcmpgtb, INS_invalid, INS_pcmpgtw, INS_invalid, INS_pcmpgtd, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, 1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, CompareScalarLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, CompareScalarNotEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, CompareScalarNotGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, CompareScalarNotGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, CompareScalarNotLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, CompareScalarNotLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, CompareScalarOrdered, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, CompareScalarOrderedEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_comiss, INS_comisd, -1, 3, HW_Category_SIMDScalar, HW_Flag_Commutative|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(X86Base, CompareScalarOrderedGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_comiss, INS_comisd, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(X86Base, CompareScalarOrderedGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_comiss, INS_comisd, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(X86Base, CompareScalarOrderedLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_comiss, INS_comisd, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(X86Base, CompareScalarOrderedLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_comiss, INS_comisd, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(X86Base, CompareScalarOrderedNotEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_comiss, INS_comisd, -1, 3, HW_Category_SIMDScalar, HW_Flag_Commutative|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(X86Base, CompareScalarUnordered, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, CompareScalarUnorderedEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ucomiss, INS_ucomisd, -1, 3, HW_Category_SIMDScalar, HW_Flag_Commutative|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(X86Base, CompareScalarUnorderedGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ucomiss, INS_ucomisd, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(X86Base, CompareScalarUnorderedGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ucomiss, INS_ucomisd, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(X86Base, CompareScalarUnorderedLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ucomiss, INS_ucomisd, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(X86Base, CompareScalarUnorderedLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ucomiss, INS_ucomisd, -1, 3, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(X86Base, CompareScalarUnorderedNotEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ucomiss, INS_ucomisd, -1, 3, HW_Category_SIMDScalar, HW_Flag_Commutative|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(X86Base, CompareUnordered, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(X86Base, ConvertScalarToVector128Double, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2sd32, INS_invalid, INS_invalid, INS_invalid, INS_cvtss2sd, INS_invalid, 5, 4, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(X86Base, ConvertScalarToVector128Int32, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, ConvertScalarToVector128Single, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2ss32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsd2ss, 5, 4, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(X86Base, ConvertScalarToVector128UInt32, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, ConvertToInt32, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_invalid, INS_invalid, INS_invalid, INS_cvtss2si32, INS_cvtsd2si32, 3, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, ConvertToInt32WithTruncation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvttss2si32, INS_cvttsd2si32, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, ConvertToUInt32, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, ConvertToVector128Double, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtdq2pd, INS_invalid, INS_invalid, INS_invalid, INS_cvtps2pd, INS_invalid, 5, 5, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, ConvertToVector128Int16, 16, 1, INS_pmovsxbw, INS_pmovzxbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_MaybeMemoryLoad) +HARDWARE_INTRINSIC(X86Base, ConvertToVector128Int32, 16, 1, INS_pmovsxbd, INS_pmovzxbd, INS_pmovsxwd, INS_pmovzxwd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtps2dq, INS_cvtpd2dq, 1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_MaybeMemoryLoad) +HARDWARE_INTRINSIC(X86Base, ConvertToVector128Int32WithTruncation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvttps2dq, INS_cvttpd2dq, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, ConvertToVector128Int64, 16, 1, INS_pmovsxbq, INS_pmovzxbq, INS_pmovsxwq, INS_pmovzxwq, INS_pmovsxdq, INS_pmovzxdq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_MaybeMemoryLoad) +HARDWARE_INTRINSIC(X86Base, ConvertToVector128Single, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtdq2ps, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtpd2ps, 4, 5, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, Crc32, 0, 2, INS_invalid, INS_crc32, INS_invalid, INS_crc32, INS_invalid, INS_crc32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(X86Base, DivRem, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_idiv, INS_div, INS_idiv, INS_div, INS_invalid, INS_invalid, 25, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_BaseTypeFromSecondArg|HW_Flag_MultiReg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(X86Base, Divide, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_divps, INS_divpd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(X86Base, DivideScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_divss, INS_divsd, -1, -1, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(X86Base, DotProduct, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_dpps, INS_dppd, -1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, Extract, 16, 2, INS_pextrb, INS_pextrb, INS_pextrw, INS_pextrw, INS_pextrd, INS_pextrd, INS_invalid, INS_invalid, INS_extractps, INS_invalid, 4, 4, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsScalarT) +HARDWARE_INTRINSIC(X86Base, Floor, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, FloorScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundss, INS_roundsd, -1, 8, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(X86Base, HorizontalAdd, 16, 2, INS_invalid, INS_invalid, INS_phaddw, INS_phaddw, INS_phaddd, INS_phaddd, INS_invalid, INS_invalid, INS_haddps, INS_haddpd, 3, 6, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, HorizontalAddSaturate, 16, 2, INS_invalid, INS_invalid, INS_phaddsw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, HorizontalSubtract, 16, 2, INS_invalid, INS_invalid, INS_phsubw, INS_invalid, INS_phsubd, INS_invalid, INS_invalid, INS_invalid, INS_hsubps, INS_hsubpd, 3, 6, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, HorizontalSubtractSaturate, 16, 2, INS_invalid, INS_invalid, INS_phsubsw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, Insert, 16, 3, INS_pinsrb, INS_pinsrb, INS_pinsrw, INS_pinsrw, INS_pinsrd, INS_pinsrd, INS_invalid, INS_invalid, INS_insertps, INS_invalid, 4, 1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(X86Base, LoadAlignedVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movaps, INS_movapd, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoRMWSemantics|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(X86Base, LoadAlignedVector128NonTemporal, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movntdqa, INS_movntdqa, INS_movntdqa, INS_movntdqa, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoRMWSemantics|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(X86Base, LoadAndDuplicateToVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movddup, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, LoadDquVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_lddqu, INS_lddqu, INS_lddqu, INS_lddqu, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoRMWSemantics|HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(X86Base, LoadFence, 0, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Barrier) +HARDWARE_INTRINSIC(X86Base, LoadHigh, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movhps, INS_movhpd, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, LoadLow, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movlps, INS_movlpd, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, LoadScalarVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_movd32, INS_movq, INS_movq, INS_movss, INS_movsd_simd, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, LoadVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(X86Base, MaskMove, 16, 3, INS_maskmovdqu, INS_maskmovdqu, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(X86Base, Max, 16, 2, INS_pmaxsb, INS_pmaxub, INS_pmaxsw, INS_pmaxuw, INS_pmaxsd, INS_pmaxud, INS_invalid, INS_invalid, INS_maxps, INS_maxpd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative) +HARDWARE_INTRINSIC(X86Base, MaxScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_maxss, INS_maxsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(X86Base, MemoryFence, 0, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Barrier) +HARDWARE_INTRINSIC(X86Base, Min, 16, 2, INS_pminsb, INS_pminub, INS_pminsw, INS_pminuw, INS_pminsd, INS_pminud, INS_invalid, INS_invalid, INS_minps, INS_minpd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative) +HARDWARE_INTRINSIC(X86Base, MinHorizontal, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_phminposuw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, MinScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_minss, INS_minsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(X86Base, MoveAndDuplicate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movddup, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, MoveHighAndDuplicate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movshdup, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, MoveHighToLow, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movhlps, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment) +HARDWARE_INTRINSIC(X86Base, MoveLowAndDuplicate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movsldup, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, MoveLowToHigh, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movlhps, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment) +HARDWARE_INTRINSIC(X86Base, MoveMask, 16, 1, INS_pmovmskb, INS_pmovmskb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movmskps, INS_movmskpd, 3, 3, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, MoveScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movq, INS_movq, INS_movss, INS_movsd_simd, 1, 1, HW_Category_SIMDScalar, HW_Flag_NoContainment) +HARDWARE_INTRINSIC(X86Base, MultipleSumAbsoluteDifferences, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_mpsadbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, Multiply, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pmuldq, INS_pmuludq, INS_mulps, INS_mulpd, 5, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(X86Base, MultiplyAddAdjacent, 16, 2, INS_invalid, INS_invalid, INS_pmaddubsw, INS_invalid, INS_pmaddwd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative) +HARDWARE_INTRINSIC(X86Base, MultiplyHigh, 16, 2, INS_invalid, INS_invalid, INS_pmulhw, INS_pmulhuw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(X86Base, MultiplyHighRoundScale, 16, 2, INS_invalid, INS_invalid, INS_pmulhrsw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(X86Base, MultiplyLow, 16, 2, INS_invalid, INS_invalid, INS_pmullw, INS_pmullw, INS_pmulld, INS_pmulld, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(X86Base, MultiplyScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mulss, INS_mulsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(X86Base, Or, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pord, INS_pord, INS_pord, INS_pord, INS_orps, INS_orpd, 1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(X86Base, PackSignedSaturate, 16, 2, INS_packsswb, INS_invalid, INS_packssdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(X86Base, PackUnsignedSaturate, 16, 2, INS_invalid, INS_packuswb, INS_invalid, INS_packusdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(X86Base, Pause, 0, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(X86Base, PopCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_popcnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(X86Base, Prefetch0, 0, 1, INS_invalid, INS_prefetcht0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(X86Base, Prefetch1, 0, 1, INS_invalid, INS_prefetcht1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(X86Base, Prefetch2, 0, 1, INS_invalid, INS_prefetcht2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(X86Base, PrefetchNonTemporal, 0, 1, INS_invalid, INS_prefetchnta, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(X86Base, Reciprocal, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rcpps, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, ReciprocalScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rcpss, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, ReciprocalSqrt, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rsqrtps, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, ReciprocalSqrtScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rsqrtss, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, RoundCurrentDirection, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, RoundCurrentDirectionScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundss, INS_roundsd, -1, 8, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(X86Base, RoundToNearestInteger, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, RoundToNearestIntegerScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundss, INS_roundsd, -1, 8, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(X86Base, RoundToNegativeInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, RoundToNegativeInfinityScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundss, INS_roundsd, -1, 8, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(X86Base, RoundToPositiveInfinity, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, RoundToPositiveInfinityScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundss, INS_roundsd, -1, 8, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(X86Base, RoundToZero, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, RoundToZeroScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundss, INS_roundsd, -1, 8, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(X86Base, ShiftLeftLogical, 16, 2, INS_invalid, INS_invalid, INS_psllw, INS_psllw, INS_pslld, INS_pslld, INS_psllq, INS_psllq, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(X86Base, ShiftLeftLogical128BitLane, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pslldq, INS_pslldq, INS_pslldq, INS_pslldq, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(X86Base, ShiftRightArithmetic, 16, 2, INS_invalid, INS_invalid, INS_psraw, INS_invalid, INS_psrad, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(X86Base, ShiftRightLogical, 16, 2, INS_invalid, INS_invalid, INS_psrlw, INS_psrlw, INS_psrld, INS_psrld, INS_psrlq, INS_psrlq, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(X86Base, ShiftRightLogical128BitLane, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_psrldq, INS_psrldq, INS_psrldq, INS_psrldq, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(X86Base, Shuffle, 16, -1, INS_pshufb, INS_pshufb, INS_invalid, INS_invalid, INS_pshufd, INS_pshufd, INS_invalid, INS_invalid, INS_shufps, INS_shufpd, 1, 1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(X86Base, ShuffleHigh, 16, 2, INS_invalid, INS_invalid, INS_pshufhw, INS_pshufhw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(X86Base, ShuffleLow, 16, 2, INS_invalid, INS_invalid, INS_pshuflw, INS_pshuflw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(X86Base, Sign, 16, 2, INS_psignb, INS_invalid, INS_psignw, INS_invalid, INS_psignd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, Sqrt, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqrtps, INS_sqrtpd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, SqrtScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqrtss, INS_sqrtsd, -1, -1, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(X86Base, Store, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(X86Base, StoreAligned, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movaps, INS_movapd, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromSecondArg|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(X86Base, StoreAlignedNonTemporal, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movntdq, INS_movntdq, INS_movntdq, INS_movntdq, INS_movntps, INS_movntpd, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromSecondArg|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(X86Base, StoreFence, 0, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Barrier) +HARDWARE_INTRINSIC(X86Base, StoreHigh, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movhps, INS_movhpd, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(X86Base, StoreLow, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movlps, INS_movlpd, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(X86Base, StoreNonTemporal, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movnti32, INS_movnti32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(X86Base, StoreScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_movd32, INS_movq, INS_movq, INS_movss, INS_movsd_simd, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(X86Base, Subtract, 16, 2, INS_psubb, INS_psubb, INS_psubw, INS_psubw, INS_psubd, INS_psubd, INS_psubq, INS_psubq, INS_subps, INS_subpd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(X86Base, SubtractSaturate, 16, 2, INS_psubsb, INS_psubusb, INS_psubsw, INS_psubusw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(X86Base, SubtractScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_subss, INS_subsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(X86Base, SumAbsoluteDifferences, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_psadbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(X86Base, TestC, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(X86Base, TestNotZAndNotC, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(X86Base, TestZ, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(X86Base, UnpackHigh, 16, 2, INS_punpckhbw, INS_punpckhbw, INS_punpckhwd, INS_punpckhwd, INS_punpckhdq, INS_punpckhdq, INS_punpckhqdq, INS_punpckhqdq, INS_unpckhps, INS_unpckhpd, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(X86Base, UnpackLow, 16, 2, INS_punpcklbw, INS_punpcklbw, INS_punpcklwd, INS_punpcklwd, INS_punpckldq, INS_punpckldq, INS_punpcklqdq, INS_punpcklqdq, INS_unpcklps, INS_unpcklpd, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(X86Base, Xor, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pxord, INS_pxord, INS_pxord, INS_pxord, INS_xorps, INS_xorpd, 1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp|HW_Flag_NormalizeSmallTypeToInt) #define LAST_NI_X86Base NI_X86Base_Xor // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -195,98 +195,98 @@ HARDWARE_INTRINSIC(X86Base, Xor, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // 64-bit only Intrinsics for X86Base, SSE, SSE2, SSE3, SSSE3, SSE41, SSE42, POPCNT #define FIRST_NI_X86Base_X64 NI_X86Base_X64_BigMul -HARDWARE_INTRINSIC(X86Base_X64, BigMul, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_imulEAX, INS_mulEAX, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_BaseTypeFromSecondArg|HW_Flag_MultiReg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_RmwIntrinsic|HW_Flag_Commutative) -HARDWARE_INTRINSIC(X86Base_X64, BitScanForward, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bsf, INS_bsf, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base_X64, BitScanReverse, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bsr, INS_bsr, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base_X64, ConvertScalarToVector128Double, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2sd64, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(X86Base_X64, ConvertScalarToVector128Int64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd64, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(X86Base_X64, ConvertScalarToVector128Single, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2ss64, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_CopyUpperBits|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(X86Base_X64, ConvertScalarToVector128UInt64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd64, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(X86Base_X64, ConvertToInt64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd64, INS_invalid, INS_cvtss2si64, INS_cvtsd2si64, INS_invalid, 3, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(X86Base_X64, ConvertToInt64WithTruncation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvttss2si64, INS_cvttsd2si64, INS_invalid, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(X86Base_X64, ConvertToUInt64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd64, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(X86Base_X64, Crc32, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_crc32, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(X86Base_X64, DivRem, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_idiv, INS_div, INS_invalid, INS_invalid, INS_invalid, 57, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_BaseTypeFromSecondArg|HW_Flag_MultiReg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(X86Base_X64, Extract, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pextrq, INS_pextrq, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsScalarT) -HARDWARE_INTRINSIC(X86Base_X64, Insert, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pinsrq, INS_pinsrq, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(X86Base_X64, PopCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_popcnt, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(X86Base_X64, StoreNonTemporal, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movnti64, INS_movnti64, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(X86Base_X64, BigMul, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_imulEAX, INS_mulEAX, INS_invalid, INS_invalid, 4, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_BaseTypeFromSecondArg|HW_Flag_MultiReg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_RmwIntrinsic|HW_Flag_Commutative) +HARDWARE_INTRINSIC(X86Base_X64, BitScanForward, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bsf, INS_bsf, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base_X64, BitScanReverse, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bsr, INS_bsr, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base_X64, ConvertScalarToVector128Double, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2sd64, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(X86Base_X64, ConvertScalarToVector128Int64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd64, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(X86Base_X64, ConvertScalarToVector128Single, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2ss64, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_CopyUpperBits|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(X86Base_X64, ConvertScalarToVector128UInt64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd64, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(X86Base_X64, ConvertToInt64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd64, INS_invalid, INS_cvtss2si64, INS_cvtsd2si64, 3, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(X86Base_X64, ConvertToInt64WithTruncation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvttss2si64, INS_cvttsd2si64, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(X86Base_X64, ConvertToUInt64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd64, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(X86Base_X64, Crc32, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_crc32, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(X86Base_X64, DivRem, 0, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_idiv, INS_div, INS_invalid, INS_invalid, 57, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_BaseTypeFromSecondArg|HW_Flag_MultiReg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(X86Base_X64, Extract, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pextrq, INS_pextrq, INS_invalid, INS_invalid, 4, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics|HW_Flag_ReturnsScalarT) +HARDWARE_INTRINSIC(X86Base_X64, Insert, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pinsrq, INS_pinsrq, INS_invalid, INS_invalid, 4, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(X86Base_X64, PopCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_popcnt, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(X86Base_X64, StoreNonTemporal, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movnti64, INS_movnti64, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromSecondArg) #define LAST_NI_X86Base_X64 NI_X86Base_X64_StoreNonTemporal // Intrinsics for AVX #define FIRST_NI_AVX NI_AVX_Add -HARDWARE_INTRINSIC(AVX, Add, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_addps, INS_addpd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX, AddSubtract, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_addsubps, INS_addsubpd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX, And, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_andps, INS_andpd, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX, AndNot, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_andnps, INS_andnpd, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, Blend, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blendps, INS_blendpd, INS_invalid, -1, 1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX, BlendVariable, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vblendvps, INS_vblendvpd, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, BroadcastScalarToVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcastss, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, BroadcastScalarToVector256, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcastss, INS_vbroadcastsd, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, BroadcastVector128ToVector256, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcastf32x4, INS_vbroadcastf32x4, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, Ceiling, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, INS_invalid, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, Compare, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_IMM, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, CompareEqual, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, CompareGreaterThan, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, CompareGreaterThanOrEqual, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, CompareLessThan, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, CompareLessThanOrEqual, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, CompareNotEqual, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, CompareNotGreaterThan, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, CompareNotGreaterThanOrEqual, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, CompareNotLessThan, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, CompareNotLessThanOrEqual, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, CompareOrdered, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, CompareScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, INS_invalid, -1, 4, HW_Category_IMM, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, CompareUnordered, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX, ConvertToVector128Int32, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtpd2dq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtpd2dq, INS_invalid, 7, 7, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, ConvertToVector128Int32WithTruncation, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvttpd2dq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 7, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, ConvertToVector128Single, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtpd2ps, INS_invalid, -1, 7, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX, ConvertToVector256Double, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtdq2pd, INS_invalid, INS_invalid, INS_invalid, INS_cvtps2pd, INS_invalid, INS_invalid, 7, 7, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX, ConvertToVector256Int32, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtps2dq, INS_invalid, INS_invalid, INS_invalid, INS_cvtps2dq, INS_invalid, INS_invalid, 4, 4, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, ConvertToVector256Int32WithTruncation, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvttps2dq, INS_invalid, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX, ConvertToVector256Single, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtdq2ps, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX, Divide, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_divps, INS_divpd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, DotProduct, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_dpps, INS_invalid, INS_invalid, -1, 13, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX, DuplicateEvenIndexed, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movsldup, INS_movddup, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, DuplicateOddIndexed, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movshdup, INS_invalid, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, ExtractVector128, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vextractf32x4, INS_vextractf32x4, INS_vextractf32x4, INS_vextractf32x4, INS_vextractf32x4, INS_vextractf32x4, INS_invalid, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX, Floor, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, INS_invalid, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, HorizontalAdd, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_haddps, INS_haddpd, INS_invalid, -1, 6, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX, HorizontalSubtract, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_hsubps, INS_hsubpd, INS_invalid, -1, 6, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX, InsertVector128, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vinsertf32x4, INS_vinsertf32x4, INS_vinsertf32x4, INS_vinsertf32x4, INS_vinsertf32x4, INS_vinsertf32x4, INS_invalid, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX, LoadAlignedVector256, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movaps, INS_movapd, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX, LoadDquVector256, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_lddqu, INS_lddqu, INS_lddqu, INS_lddqu, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX, LoadVector256, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX, MaskLoad, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vmaskmovps, INS_vmaskmovpd, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX, MaskStore, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vmaskmovps, INS_vmaskmovpd, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoContainment|HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX, Max, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_maxps, INS_maxpd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative) -HARDWARE_INTRINSIC(AVX, Min, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_minps, INS_minpd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative) -HARDWARE_INTRINSIC(AVX, MoveMask, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movmskps, INS_movmskpd, INS_invalid, -1, 5, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX, Multiply, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mulps, INS_mulpd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX, Or, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_orps, INS_orpd, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX, Permute, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermilps, INS_vpermilpd, INS_invalid, -1, 1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX, Permute2x128, 32, 3, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, INS_invalid, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX, PermuteVar, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermilpsvar, INS_vpermilpdvar, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, Reciprocal, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rcpps, INS_invalid, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX, ReciprocalSqrt, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rsqrtps, INS_invalid, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX, RoundCurrentDirection, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, INS_invalid, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, RoundToNearestInteger, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, INS_invalid, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, RoundToNegativeInfinity, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, INS_invalid, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, RoundToPositiveInfinity, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, INS_invalid, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, RoundToZero, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, INS_invalid, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, Shuffle, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_shufps, INS_shufpd, INS_invalid, -1, 1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX, Sqrt, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqrtps, INS_sqrtpd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, Store, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(AVX, StoreAligned, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movaps, INS_movapd, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX, StoreAlignedNonTemporal, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movntdq, INS_movntdq, INS_movntdq, INS_movntdq, INS_movntps, INS_movntpd, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX, Subtract, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_subps, INS_subpd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, TestC, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_vtestps, INS_vtestpd, INS_invalid, 6, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(AVX, TestNotZAndNotC, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_vtestps, INS_vtestpd, INS_invalid, 6, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(AVX, TestZ, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_vtestps, INS_vtestpd, INS_invalid, 6, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_ReturnsBoolean) -HARDWARE_INTRINSIC(AVX, UnpackHigh, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_unpckhps, INS_unpckhpd, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, UnpackLow, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_unpcklps, INS_unpcklpd, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX, Xor, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_xorps, INS_xorpd, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AVX, Add, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_addps, INS_addpd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX, AddSubtract, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_addsubps, INS_addsubpd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX, And, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_andps, INS_andpd, -1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX, AndNot, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_andnps, INS_andnpd, -1, 1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, Blend, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blendps, INS_blendpd, -1, 1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX, BlendVariable, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vblendvps, INS_vblendvpd, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, BroadcastScalarToVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcastss, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, BroadcastScalarToVector256, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcastss, INS_vbroadcastsd, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, BroadcastVector128ToVector256, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcastf32x4, INS_vbroadcastf32x4, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, Ceiling, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, Compare, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_IMM, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, CompareEqual, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, CompareGreaterThan, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, CompareGreaterThanOrEqual, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, CompareLessThan, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, CompareLessThanOrEqual, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, CompareNotEqual, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, CompareNotGreaterThan, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, CompareNotGreaterThanOrEqual, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, CompareNotLessThan, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, CompareNotLessThanOrEqual, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, CompareOrdered, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, CompareScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpss, INS_cmpsd, -1, 4, HW_Category_IMM, HW_Flag_CopyUpperBits|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, CompareUnordered, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cmpps, INS_cmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX, ConvertToVector128Int32, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtpd2dq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtpd2dq, 7, 7, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, ConvertToVector128Int32WithTruncation, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvttpd2dq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 7, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, ConvertToVector128Single, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtpd2ps, -1, 7, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX, ConvertToVector256Double, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtdq2pd, INS_invalid, INS_invalid, INS_invalid, INS_cvtps2pd, INS_invalid, 7, 7, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX, ConvertToVector256Int32, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtps2dq, INS_invalid, INS_invalid, INS_invalid, INS_cvtps2dq, INS_invalid, 4, 4, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, ConvertToVector256Int32WithTruncation, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvttps2dq, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX, ConvertToVector256Single, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtdq2ps, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX, Divide, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_divps, INS_divpd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, DotProduct, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_dpps, INS_invalid, -1, 13, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX, DuplicateEvenIndexed, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movsldup, INS_movddup, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, DuplicateOddIndexed, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movshdup, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, ExtractVector128, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vextractf32x4, INS_vextractf32x4, INS_vextractf32x4, INS_vextractf32x4, INS_vextractf32x4, INS_vextractf32x4, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX, Floor, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, HorizontalAdd, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_haddps, INS_haddpd, -1, 6, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX, HorizontalSubtract, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_hsubps, INS_hsubpd, -1, 6, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX, InsertVector128, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vinsertf32x4, INS_vinsertf32x4, INS_vinsertf32x4, INS_vinsertf32x4, INS_vinsertf32x4, INS_vinsertf32x4, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX, LoadAlignedVector256, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movaps, INS_movapd, -1, -1, HW_Category_MemoryLoad, HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX, LoadDquVector256, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_lddqu, INS_lddqu, INS_lddqu, INS_lddqu, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX, LoadVector256, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX, MaskLoad, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vmaskmovps, INS_vmaskmovpd, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX, MaskStore, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vmaskmovps, INS_vmaskmovpd, -1, -1, HW_Category_MemoryStore, HW_Flag_NoContainment|HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX, Max, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_maxps, INS_maxpd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative) +HARDWARE_INTRINSIC(AVX, Min, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_minps, INS_minpd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative) +HARDWARE_INTRINSIC(AVX, MoveMask, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movmskps, INS_movmskpd, -1, 5, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX, Multiply, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mulps, INS_mulpd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX, Or, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_orps, INS_orpd, -1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX, Permute, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermilps, INS_vpermilpd, -1, 1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX, Permute2x128, 32, 3, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, INS_vperm2f128, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX, PermuteVar, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermilpsvar, INS_vpermilpdvar, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, Reciprocal, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rcpps, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX, ReciprocalSqrt, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rsqrtps, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX, RoundCurrentDirection, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, RoundToNearestInteger, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, RoundToNegativeInfinity, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, RoundToPositiveInfinity, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, RoundToZero, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_roundps, INS_roundpd, -1, 8, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, Shuffle, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_shufps, INS_shufpd, -1, 1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX, Sqrt, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqrtps, INS_sqrtpd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, Store, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(AVX, StoreAligned, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movdqa32, INS_movaps, INS_movapd, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX, StoreAlignedNonTemporal, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movntdq, INS_movntdq, INS_movntdq, INS_movntdq, INS_movntps, INS_movntpd, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX, Subtract, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_subps, INS_subpd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, TestC, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_vtestps, INS_vtestpd, 6, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(AVX, TestNotZAndNotC, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_vtestps, INS_vtestpd, 6, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(AVX, TestZ, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_vtestps, INS_vtestpd, 6, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_ReturnsBoolean) +HARDWARE_INTRINSIC(AVX, UnpackHigh, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_unpckhps, INS_unpckhpd, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, UnpackLow, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_unpcklps, INS_unpcklpd, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX, Xor, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_xorps, INS_xorpd, -1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp) #define LAST_NI_AVX NI_AVX_Xor // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -295,96 +295,96 @@ HARDWARE_INTRINSIC(AVX, Xor, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Intrinsics for AVX2, BMI1, BMI2, F16C, FMA, LZCNT, MOVBE #define FIRST_NI_AVX2 NI_AVX2_Abs -HARDWARE_INTRINSIC(AVX2, Abs, 32, 1, INS_pabsb, INS_invalid, INS_pabsw, INS_invalid, INS_pabsd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX2, Add, 32, 2, INS_paddb, INS_paddb, INS_paddw, INS_paddw, INS_paddd, INS_paddd, INS_paddq, INS_paddq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX2, AddSaturate, 32, 2, INS_paddsb, INS_paddusb, INS_paddsw, INS_paddusw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX2, AlignRight, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_palignr, INS_palignr, INS_palignr, INS_palignr, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX2, And, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pandd, INS_pandd, INS_pandd, INS_pandd, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX2, AndNot, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_InvalidNodeId|HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(AVX2, Average, 32, 2, INS_invalid, INS_pavgb, INS_invalid, INS_pavgw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX2, BitFieldExtract, 0, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bextr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 2, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, Blend, -1, 3, INS_invalid, INS_invalid, INS_pblendw, INS_pblendw, INS_vpblendd, INS_vpblendd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, BlendVariable, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpblendvb, INS_vpblendvb, INS_vpblendvb, INS_vpblendvb, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX2, BroadcastScalarToVector128, 16, 1, INS_vpbroadcastb, INS_vpbroadcastb, INS_vpbroadcastw, INS_vpbroadcastw, INS_vpbroadcastd, INS_vpbroadcastd, INS_vpbroadcastq, INS_vpbroadcastq, INS_vbroadcastss, INS_movddup, INS_invalid, 1, 1, HW_Category_SIMDScalar, HW_Flag_MaybeMemoryLoad) -HARDWARE_INTRINSIC(AVX2, BroadcastScalarToVector256, 32, 1, INS_vpbroadcastb, INS_vpbroadcastb, INS_vpbroadcastw, INS_vpbroadcastw, INS_vpbroadcastd, INS_vpbroadcastd, INS_vpbroadcastq, INS_vpbroadcastq, INS_vbroadcastss, INS_vbroadcastsd, INS_invalid, 3, 3, HW_Category_SIMDScalar, HW_Flag_MaybeMemoryLoad) -HARDWARE_INTRINSIC(AVX2, BroadcastVector128ToVector256, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcasti32x4, INS_vbroadcasti32x4, INS_vbroadcasti32x4, INS_vbroadcasti32x4, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX2, CompareEqual, 32, 2, INS_pcmpeqb, INS_pcmpeqb, INS_pcmpeqw, INS_pcmpeqw, INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqq, INS_pcmpeqq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX2, CompareGreaterThan, 32, 2, INS_pcmpgtb, INS_invalid, INS_pcmpgtw, INS_invalid, INS_pcmpgtd, INS_invalid, INS_pcmpgtq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX2, CompareLessThan, 32, 2, INS_pcmpgtb, INS_invalid, INS_pcmpgtw, INS_invalid, INS_pcmpgtd, INS_invalid, INS_pcmpgtq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX2, ConvertToInt32, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX2, ConvertToUInt32, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX2, ConvertToVector128Half, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2ph, INS_invalid, INS_invalid, -1, 5, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX2, ConvertToVector128Single, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_vcvtph2ps, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX2, ConvertToVector256Half, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2ph, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX2, ConvertToVector256Int16, 32, 1, INS_pmovsxbw, INS_pmovzxbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_MaybeMemoryLoad) -HARDWARE_INTRINSIC(AVX2, ConvertToVector256Int32, 32, 1, INS_pmovsxbd, INS_pmovzxbd, INS_pmovsxwd, INS_pmovzxwd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_MaybeMemoryLoad) -HARDWARE_INTRINSIC(AVX2, ConvertToVector256Int64, 32, 1, INS_pmovsxbq, INS_pmovzxbq, INS_pmovsxwq, INS_pmovzxwq, INS_pmovsxdq, INS_pmovzxdq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_MaybeMemoryLoad) -HARDWARE_INTRINSIC(AVX2, ConvertToVector256Single, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_vcvtph2ps, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 7, 7, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX2, ExtractLowestSetBit, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blsi, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, ExtractVector128, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vextracti32x4, INS_vextracti32x4, INS_vextracti32x4, INS_vextracti32x4, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX2, GatherMaskVector128, 16, 5, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpgatherdd, INS_vpgatherdd, INS_vpgatherdq, INS_vpgatherdq, INS_vgatherdps, INS_vgatherdpd, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_MaybeMemoryLoad|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_NoContainment|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, GatherMaskVector256, 32, 5, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpgatherdd, INS_vpgatherdd, INS_vpgatherdq, INS_vpgatherdq, INS_vgatherdps, INS_vgatherdpd, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_MaybeMemoryLoad|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_NoContainment|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, GatherVector128, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpgatherdd, INS_vpgatherdd, INS_vpgatherdq, INS_vpgatherdq, INS_vgatherdps, INS_vgatherdpd, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_MaybeMemoryLoad|HW_Flag_SpecialCodeGen|HW_Flag_NoContainment|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, GatherVector256, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpgatherdd, INS_vpgatherdd, INS_vpgatherdq, INS_vpgatherdq, INS_vgatherdps, INS_vgatherdpd, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_MaybeMemoryLoad|HW_Flag_SpecialCodeGen|HW_Flag_NoContainment|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, GetMaskUpToLowestSetBit, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blsmsk, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(AVX2, HorizontalAdd, 32, 2, INS_invalid, INS_invalid, INS_phaddw, INS_phaddw, INS_phaddd, INS_phaddd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, HorizontalAddSaturate, 32, 2, INS_invalid, INS_invalid, INS_phaddsw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, HorizontalSubtract, 32, 2, INS_invalid, INS_invalid, INS_phsubw, INS_invalid, INS_phsubd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, HorizontalSubtractSaturate, 32, 2, INS_invalid, INS_invalid, INS_phsubsw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, InsertVector128, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vinserti32x4, INS_vinserti32x4, INS_vinserti32x4, INS_vinserti32x4, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX2, LeadingZeroCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_lzcnt, INS_lzcnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX2, LoadAlignedVector256NonTemporal, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movntdqa, INS_movntdqa, INS_movntdqa, INS_movntdqa, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX2, MaskLoad, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmaskmovd, INS_vpmaskmovd, INS_vpmaskmovq, INS_vpmaskmovq, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, MaskStore, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmaskmovd, INS_vpmaskmovd, INS_vpmaskmovq, INS_vpmaskmovq, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoContainment|HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, Max, 32, 2, INS_pmaxsb, INS_pmaxub, INS_pmaxsw, INS_pmaxuw, INS_pmaxsd, INS_pmaxud, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX2, Min, 32, 2, INS_pminsb, INS_pminub, INS_pminsw, INS_pminuw, INS_pminsd, INS_pminud, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX2, MoveMask, 32, 1, INS_pmovmskb, INS_pmovmskb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, MultipleSumAbsoluteDifferences, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_mpsadbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, Multiply, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pmuldq, INS_pmuludq, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX2, MultiplyAdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmadd213ps, INS_vfmadd213pd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX2, MultiplyAddAdjacent, 32, 2, INS_invalid, INS_invalid, INS_pmaddubsw, INS_invalid, INS_pmaddwd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX2, MultiplyAddNegated, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmadd213ps, INS_vfnmadd213pd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX2, MultiplyAddNegatedScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmadd213ss, INS_vfnmadd213sd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX2, MultiplyAddScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmadd213ss, INS_vfmadd213sd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX2, MultiplyAddSubtract, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmaddsub213ps, INS_vfmaddsub213pd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX2, MultiplyHigh, 32, 2, INS_invalid, INS_invalid, INS_pmulhw, INS_pmulhuw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX2, MultiplyHighRoundScale, 32, 2, INS_invalid, INS_invalid, INS_pmulhrsw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX2, MultiplyLow, 32, 2, INS_invalid, INS_invalid, INS_pmullw, INS_pmullw, INS_pmulld, INS_pmulld, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX2, MultiplyNoFlags, 0, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mulx, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_Scalar, HW_Flag_NoContainment|HW_Flag_MaybeMemoryStore|HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics|HW_Flag_MaybeCommutative) -HARDWARE_INTRINSIC(AVX2, MultiplySubtract, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmsub213ps, INS_vfmsub213pd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX2, MultiplySubtractAdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmsubadd213ps, INS_vfmsubadd213pd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX2, MultiplySubtractNegated, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmsub213ps, INS_vfnmsub213pd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX2, MultiplySubtractNegatedScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmsub213ss, INS_vfnmsub213sd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX2, MultiplySubtractScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmsub213ss, INS_vfmsub213sd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX2, Or, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pord, INS_pord, INS_pord, INS_pord, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX2, PackSignedSaturate, 32, 2, INS_packsswb, INS_invalid, INS_packssdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX2, PackUnsignedSaturate, 32, 2, INS_invalid, INS_packuswb, INS_invalid, INS_packusdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX2, ParallelBitDeposit, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pdep, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, ParallelBitExtract, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pext, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, Permute2x128, 32, 3, INS_vperm2i128, INS_vperm2i128, INS_vperm2i128, INS_vperm2i128, INS_vperm2i128, INS_vperm2i128, INS_vperm2i128, INS_vperm2i128, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, Permute4x64, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermq, INS_vpermq, INS_invalid, INS_vpermpd, INS_invalid, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX2, PermuteVar8x32, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermd, INS_vpermd, INS_invalid, INS_invalid, INS_vpermps, INS_invalid, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX2, ResetLowestSetBit, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blsr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(AVX2, ShiftLeftLogical, 32, 2, INS_invalid, INS_invalid, INS_psllw, INS_psllw, INS_pslld, INS_pslld, INS_psllq, INS_psllq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX2, ShiftLeftLogical128BitLane, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pslldq, INS_pslldq, INS_pslldq, INS_pslldq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX2, ShiftLeftLogicalVariable, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpsllvd, INS_vpsllvd, INS_vpsllvq, INS_vpsllvq, INS_invalid, INS_invalid, INS_invalid, 2, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX2, ShiftRightArithmetic, 32, 2, INS_invalid, INS_invalid, INS_psraw, INS_invalid, INS_psrad, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX2, ShiftRightArithmeticVariable, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpsravd, INS_vpsravd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 2, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX2, ShiftRightLogical, 32, 2, INS_invalid, INS_invalid, INS_psrlw, INS_psrlw, INS_psrld, INS_psrld, INS_psrlq, INS_psrlq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX2, ShiftRightLogical128BitLane, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_psrldq, INS_psrldq, INS_psrldq, INS_psrldq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX2, ShiftRightLogicalVariable, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpsrlvd, INS_vpsrlvd, INS_vpsrlvq, INS_vpsrlvq, INS_invalid, INS_invalid, INS_invalid, 2, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX2, Shuffle, 32, 2, INS_pshufb, INS_pshufb, INS_invalid, INS_invalid, INS_pshufd, INS_pshufd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_MaybeIMM) -HARDWARE_INTRINSIC(AVX2, ShuffleHigh, 32, 2, INS_invalid, INS_invalid, INS_pshufhw, INS_pshufhw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX2, ShuffleLow, 32, 2, INS_invalid, INS_invalid, INS_pshuflw, INS_pshuflw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX2, Sign, 32, 2, INS_psignb, INS_invalid, INS_psignw, INS_invalid, INS_psignd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, Subtract, 32, 2, INS_psubb, INS_psubb, INS_psubw, INS_psubw, INS_psubd, INS_psubd, INS_psubq, INS_psubq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX2, SubtractSaturate, 32, 2, INS_psubsb, INS_psubusb, INS_psubsw, INS_psubusw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX2, SumAbsoluteDifferences, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_psadbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX2, TrailingZeroCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_tzcnt, INS_tzcnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX2, UnpackHigh, 32, 2, INS_punpckhbw, INS_punpckhbw, INS_punpckhwd, INS_punpckhwd, INS_punpckhdq, INS_punpckhdq, INS_punpckhqdq, INS_punpckhqdq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX2, UnpackLow, 32, 2, INS_punpcklbw, INS_punpcklbw, INS_punpcklwd, INS_punpcklwd, INS_punpckldq, INS_punpckldq, INS_punpcklqdq, INS_punpcklqdq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX2, Xor, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pxord, INS_pxord, INS_pxord, INS_pxord, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX2, ZeroHighBits, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bzhi, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialImport|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, Abs, 32, 1, INS_pabsb, INS_invalid, INS_pabsw, INS_invalid, INS_pabsd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX2, Add, 32, 2, INS_paddb, INS_paddb, INS_paddw, INS_paddw, INS_paddd, INS_paddd, INS_paddq, INS_paddq, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX2, AddSaturate, 32, 2, INS_paddsb, INS_paddusb, INS_paddsw, INS_paddusw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX2, AlignRight, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_palignr, INS_palignr, INS_palignr, INS_palignr, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX2, And, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pandd, INS_pandd, INS_pandd, INS_pandd, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX2, AndNot, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_InvalidNodeId|HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(AVX2, Average, 32, 2, INS_invalid, INS_pavgb, INS_invalid, INS_pavgw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX2, BitFieldExtract, 0, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bextr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 2, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, Blend, -1, 3, INS_invalid, INS_invalid, INS_pblendw, INS_pblendw, INS_vpblendd, INS_vpblendd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, BlendVariable, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpblendvb, INS_vpblendvb, INS_vpblendvb, INS_vpblendvb, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics|HW_Flag_NormalizeSmallTypeToInt|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX2, BroadcastScalarToVector128, 16, 1, INS_vpbroadcastb, INS_vpbroadcastb, INS_vpbroadcastw, INS_vpbroadcastw, INS_vpbroadcastd, INS_vpbroadcastd, INS_vpbroadcastq, INS_vpbroadcastq, INS_vbroadcastss, INS_movddup, 1, 1, HW_Category_SIMDScalar, HW_Flag_MaybeMemoryLoad) +HARDWARE_INTRINSIC(AVX2, BroadcastScalarToVector256, 32, 1, INS_vpbroadcastb, INS_vpbroadcastb, INS_vpbroadcastw, INS_vpbroadcastw, INS_vpbroadcastd, INS_vpbroadcastd, INS_vpbroadcastq, INS_vpbroadcastq, INS_vbroadcastss, INS_vbroadcastsd, 3, 3, HW_Category_SIMDScalar, HW_Flag_MaybeMemoryLoad) +HARDWARE_INTRINSIC(AVX2, BroadcastVector128ToVector256, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcasti32x4, INS_vbroadcasti32x4, INS_vbroadcasti32x4, INS_vbroadcasti32x4, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX2, CompareEqual, 32, 2, INS_pcmpeqb, INS_pcmpeqb, INS_pcmpeqw, INS_pcmpeqw, INS_pcmpeqd, INS_pcmpeqd, INS_pcmpeqq, INS_pcmpeqq, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX2, CompareGreaterThan, 32, 2, INS_pcmpgtb, INS_invalid, INS_pcmpgtw, INS_invalid, INS_pcmpgtd, INS_invalid, INS_pcmpgtq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX2, CompareLessThan, 32, 2, INS_pcmpgtb, INS_invalid, INS_pcmpgtw, INS_invalid, INS_pcmpgtd, INS_invalid, INS_pcmpgtq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_NoEvexSemantics|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX2, ConvertToInt32, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX2, ConvertToUInt32, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movd32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX2, ConvertToVector128Half, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2ph, INS_invalid, -1, 5, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX2, ConvertToVector128Single, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_vcvtph2ps, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX2, ConvertToVector256Half, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2ph, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX2, ConvertToVector256Int16, 32, 1, INS_pmovsxbw, INS_pmovzxbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_MaybeMemoryLoad) +HARDWARE_INTRINSIC(AVX2, ConvertToVector256Int32, 32, 1, INS_pmovsxbd, INS_pmovzxbd, INS_pmovsxwd, INS_pmovzxwd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_MaybeMemoryLoad) +HARDWARE_INTRINSIC(AVX2, ConvertToVector256Int64, 32, 1, INS_pmovsxbq, INS_pmovzxbq, INS_pmovsxwq, INS_pmovzxwq, INS_pmovsxdq, INS_pmovzxdq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_MaybeMemoryLoad) +HARDWARE_INTRINSIC(AVX2, ConvertToVector256Single, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_vcvtph2ps, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 7, 7, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX2, ExtractLowestSetBit, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blsi, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, ExtractVector128, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vextracti32x4, INS_vextracti32x4, INS_vextracti32x4, INS_vextracti32x4, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX2, GatherMaskVector128, 16, 5, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpgatherdd, INS_vpgatherdd, INS_vpgatherdq, INS_vpgatherdq, INS_vgatherdps, INS_vgatherdpd, -1, -1, HW_Category_IMM, HW_Flag_MaybeMemoryLoad|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_NoContainment|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, GatherMaskVector256, 32, 5, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpgatherdd, INS_vpgatherdd, INS_vpgatherdq, INS_vpgatherdq, INS_vgatherdps, INS_vgatherdpd, -1, -1, HW_Category_IMM, HW_Flag_MaybeMemoryLoad|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_NoContainment|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, GatherVector128, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpgatherdd, INS_vpgatherdd, INS_vpgatherdq, INS_vpgatherdq, INS_vgatherdps, INS_vgatherdpd, -1, -1, HW_Category_IMM, HW_Flag_MaybeMemoryLoad|HW_Flag_SpecialCodeGen|HW_Flag_NoContainment|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, GatherVector256, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpgatherdd, INS_vpgatherdd, INS_vpgatherdq, INS_vpgatherdq, INS_vgatherdps, INS_vgatherdpd, -1, -1, HW_Category_IMM, HW_Flag_MaybeMemoryLoad|HW_Flag_SpecialCodeGen|HW_Flag_NoContainment|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, GetMaskUpToLowestSetBit, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blsmsk, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(AVX2, HorizontalAdd, 32, 2, INS_invalid, INS_invalid, INS_phaddw, INS_phaddw, INS_phaddd, INS_phaddd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, HorizontalAddSaturate, 32, 2, INS_invalid, INS_invalid, INS_phaddsw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, HorizontalSubtract, 32, 2, INS_invalid, INS_invalid, INS_phsubw, INS_invalid, INS_phsubd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, HorizontalSubtractSaturate, 32, 2, INS_invalid, INS_invalid, INS_phsubsw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, InsertVector128, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vinserti32x4, INS_vinserti32x4, INS_vinserti32x4, INS_vinserti32x4, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX2, LeadingZeroCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_lzcnt, INS_lzcnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX2, LoadAlignedVector256NonTemporal, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movntdqa, INS_movntdqa, INS_movntdqa, INS_movntdqa, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX2, MaskLoad, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmaskmovd, INS_vpmaskmovd, INS_vpmaskmovq, INS_vpmaskmovq, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, MaskStore, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmaskmovd, INS_vpmaskmovd, INS_vpmaskmovq, INS_vpmaskmovq, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoContainment|HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, Max, 32, 2, INS_pmaxsb, INS_pmaxub, INS_pmaxsw, INS_pmaxuw, INS_pmaxsd, INS_pmaxud, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX2, Min, 32, 2, INS_pminsb, INS_pminub, INS_pminsw, INS_pminuw, INS_pminsd, INS_pminud, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX2, MoveMask, 32, 1, INS_pmovmskb, INS_pmovmskb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, MultipleSumAbsoluteDifferences, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_mpsadbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, Multiply, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pmuldq, INS_pmuludq, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX2, MultiplyAdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmadd213ps, INS_vfmadd213pd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX2, MultiplyAddAdjacent, 32, 2, INS_invalid, INS_invalid, INS_pmaddubsw, INS_invalid, INS_pmaddwd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX2, MultiplyAddNegated, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmadd213ps, INS_vfnmadd213pd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX2, MultiplyAddNegatedScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmadd213ss, INS_vfnmadd213sd, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX2, MultiplyAddScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmadd213ss, INS_vfmadd213sd, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX2, MultiplyAddSubtract, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmaddsub213ps, INS_vfmaddsub213pd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX2, MultiplyHigh, 32, 2, INS_invalid, INS_invalid, INS_pmulhw, INS_pmulhuw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX2, MultiplyHighRoundScale, 32, 2, INS_invalid, INS_invalid, INS_pmulhrsw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX2, MultiplyLow, 32, 2, INS_invalid, INS_invalid, INS_pmullw, INS_pmullw, INS_pmulld, INS_pmulld, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX2, MultiplyNoFlags, 0, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mulx, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_Scalar, HW_Flag_NoContainment|HW_Flag_MaybeMemoryStore|HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics|HW_Flag_MaybeCommutative) +HARDWARE_INTRINSIC(AVX2, MultiplySubtract, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmsub213ps, INS_vfmsub213pd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX2, MultiplySubtractAdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmsubadd213ps, INS_vfmsubadd213pd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX2, MultiplySubtractNegated, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmsub213ps, INS_vfnmsub213pd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX2, MultiplySubtractNegatedScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmsub213ss, INS_vfnmsub213sd, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX2, MultiplySubtractScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmsub213ss, INS_vfmsub213sd, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX2, Or, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pord, INS_pord, INS_pord, INS_pord, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX2, PackSignedSaturate, 32, 2, INS_packsswb, INS_invalid, INS_packssdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX2, PackUnsignedSaturate, 32, 2, INS_invalid, INS_packuswb, INS_invalid, INS_packusdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX2, ParallelBitDeposit, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pdep, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, ParallelBitExtract, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pext, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, Permute2x128, 32, 3, INS_vperm2i128, INS_vperm2i128, INS_vperm2i128, INS_vperm2i128, INS_vperm2i128, INS_vperm2i128, INS_vperm2i128, INS_vperm2i128, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, Permute4x64, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermq, INS_vpermq, INS_invalid, INS_vpermpd, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX2, PermuteVar8x32, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermd, INS_vpermd, INS_invalid, INS_invalid, INS_vpermps, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX2, ResetLowestSetBit, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blsr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(AVX2, ShiftLeftLogical, 32, 2, INS_invalid, INS_invalid, INS_psllw, INS_psllw, INS_pslld, INS_pslld, INS_psllq, INS_psllq, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX2, ShiftLeftLogical128BitLane, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pslldq, INS_pslldq, INS_pslldq, INS_pslldq, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX2, ShiftLeftLogicalVariable, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpsllvd, INS_vpsllvd, INS_vpsllvq, INS_vpsllvq, INS_invalid, INS_invalid, 2, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX2, ShiftRightArithmetic, 32, 2, INS_invalid, INS_invalid, INS_psraw, INS_invalid, INS_psrad, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX2, ShiftRightArithmeticVariable, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpsravd, INS_vpsravd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 2, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX2, ShiftRightLogical, 32, 2, INS_invalid, INS_invalid, INS_psrlw, INS_psrlw, INS_psrld, INS_psrld, INS_psrlq, INS_psrlq, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX2, ShiftRightLogical128BitLane, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_psrldq, INS_psrldq, INS_psrldq, INS_psrldq, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX2, ShiftRightLogicalVariable, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpsrlvd, INS_vpsrlvd, INS_vpsrlvq, INS_vpsrlvq, INS_invalid, INS_invalid, 2, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX2, Shuffle, 32, 2, INS_pshufb, INS_pshufb, INS_invalid, INS_invalid, INS_pshufd, INS_pshufd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_MaybeIMM) +HARDWARE_INTRINSIC(AVX2, ShuffleHigh, 32, 2, INS_invalid, INS_invalid, INS_pshufhw, INS_pshufhw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX2, ShuffleLow, 32, 2, INS_invalid, INS_invalid, INS_pshuflw, INS_pshuflw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX2, Sign, 32, 2, INS_psignb, INS_invalid, INS_psignw, INS_invalid, INS_psignd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, Subtract, 32, 2, INS_psubb, INS_psubb, INS_psubw, INS_psubw, INS_psubd, INS_psubd, INS_psubq, INS_psubq, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX2, SubtractSaturate, 32, 2, INS_psubsb, INS_psubusb, INS_psubsw, INS_psubusw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX2, SumAbsoluteDifferences, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_psadbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX2, TrailingZeroCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_tzcnt, INS_tzcnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX2, UnpackHigh, 32, 2, INS_punpckhbw, INS_punpckhbw, INS_punpckhwd, INS_punpckhwd, INS_punpckhdq, INS_punpckhdq, INS_punpckhqdq, INS_punpckhqdq, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX2, UnpackLow, 32, 2, INS_punpcklbw, INS_punpcklbw, INS_punpcklwd, INS_punpcklwd, INS_punpckldq, INS_punpckldq, INS_punpcklqdq, INS_punpcklqdq, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX2, Xor, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pxord, INS_pxord, INS_pxord, INS_pxord, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX2, ZeroHighBits, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bzhi, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialImport|HW_Flag_NoEvexSemantics) #define LAST_NI_AVX2 NI_AVX2_ZeroHighBits // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -393,17 +393,17 @@ HARDWARE_INTRINSIC(AVX2, ZeroHighBits, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // 64-bit only Intrinsics for AVX2, BMI1, BMI2, F16C, FMA, LZCNT, MOVBE #define FIRST_NI_AVX2_X64 NI_AVX2_X64_AndNot -HARDWARE_INTRINSIC(AVX2_X64, AndNot, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_andn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_SpecialImport|HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2_X64, BitFieldExtract, 0, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bextr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 2, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2_X64, ExtractLowestSetBit, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blsi, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2_X64, GetMaskUpToLowestSetBit, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blsmsk, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(AVX2_X64, LeadingZeroCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_lzcnt, INS_lzcnt, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX2_X64, MultiplyNoFlags, 0, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mulx, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_Scalar, HW_Flag_NoContainment|HW_Flag_MaybeMemoryStore|HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics|HW_Flag_MaybeCommutative) -HARDWARE_INTRINSIC(AVX2_X64, ParallelBitDeposit, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pdep, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2_X64, ParallelBitExtract, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pext, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2_X64, ResetLowestSetBit, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blsr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) -HARDWARE_INTRINSIC(AVX2_X64, TrailingZeroCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_tzcnt, INS_tzcnt, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX2_X64, ZeroHighBits, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bzhi, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialImport|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2_X64, AndNot, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_andn, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_SpecialImport|HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2_X64, BitFieldExtract, 0, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bextr, INS_invalid, INS_invalid, INS_invalid, 2, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2_X64, ExtractLowestSetBit, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blsi, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2_X64, GetMaskUpToLowestSetBit, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blsmsk, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(AVX2_X64, LeadingZeroCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_lzcnt, INS_lzcnt, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX2_X64, MultiplyNoFlags, 0, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mulx, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_Scalar, HW_Flag_NoContainment|HW_Flag_MaybeMemoryStore|HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics|HW_Flag_MaybeCommutative) +HARDWARE_INTRINSIC(AVX2_X64, ParallelBitDeposit, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pdep, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2_X64, ParallelBitExtract, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pext, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2_X64, ResetLowestSetBit, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_blsr, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) +HARDWARE_INTRINSIC(AVX2_X64, TrailingZeroCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_tzcnt, INS_tzcnt, INS_invalid, INS_invalid, 3, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX2_X64, ZeroHighBits, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_bzhi, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialImport|HW_Flag_NoEvexSemantics) #define LAST_NI_AVX2_X64 NI_AVX2_X64_ZeroHighBits // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -412,206 +412,206 @@ HARDWARE_INTRINSIC(AVX2_X64, ZeroHighBits, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Intrinsics for AVX512F, AVX512BW, AVX512CD, AVX512DQ, AVX512VL #define FIRST_NI_AVX512 NI_AVX512_Abs -HARDWARE_INTRINSIC(AVX512, Abs, -1, 1, INS_pabsb, INS_invalid, INS_pabsw, INS_invalid, INS_pabsd, INS_invalid, INS_vpabsq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, Add, 64, -1, INS_paddb, INS_paddb, INS_paddw, INS_paddw, INS_paddd, INS_paddd, INS_paddq, INS_paddq, INS_addps, INS_addpd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, AddSaturate, 64, 2, INS_paddsb, INS_paddusb, INS_paddsw, INS_paddusw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX512, AddScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_addss, INS_addsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, AlignRight, 64, 3, INS_palignr, INS_palignr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, AlignRight32, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_valignd, INS_valignd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, AlignRight64, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_valignq, INS_valignq, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, And, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pandd, INS_pandd, INS_vpandq, INS_vpandq, INS_andps, INS_andpd, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX512, AndNot, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pandnd, INS_pandnd, INS_vpandnq, INS_vpandnq, INS_andnps, INS_andnpd, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX512, Average, 64, 2, INS_invalid, INS_pavgb, INS_invalid, INS_pavgw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX512, BlendVariable, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, BroadcastPairScalarToVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcasti32x2, INS_vbroadcasti32x2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, BroadcastPairScalarToVector256, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcasti32x2, INS_vbroadcasti32x2, INS_invalid, INS_invalid, INS_vbroadcastf32x2, INS_invalid, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, BroadcastPairScalarToVector512, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcasti32x2, INS_vbroadcasti32x2, INS_invalid, INS_invalid, INS_vbroadcastf32x2, INS_invalid, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, BroadcastScalarToVector512, 64, 1, INS_vpbroadcastb, INS_vpbroadcastb, INS_vpbroadcastw, INS_vpbroadcastw, INS_vpbroadcastd, INS_vpbroadcastd, INS_vpbroadcastq, INS_vpbroadcastq, INS_vbroadcastss, INS_vbroadcastsd, INS_invalid, 3, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, BroadcastVector128ToVector512, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcasti32x4, INS_vbroadcasti32x4, INS_vbroadcasti64x2, INS_vbroadcasti64x2, INS_vbroadcastf32x4, INS_vbroadcastf64x2, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, BroadcastVector256ToVector512, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcasti32x8, INS_vbroadcasti32x8, INS_vbroadcasti64x4, INS_vbroadcasti64x4, INS_vbroadcastf32x8, INS_vbroadcastf64x4, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, Classify, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_InvalidNodeId|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, ClassifyScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_InvalidNodeId|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, Compare, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, CompareEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, CompareGreaterThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, CompareGreaterThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, CompareLessThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, CompareLessThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, CompareNotEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, CompareNotGreaterThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, CompareNotGreaterThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, CompareNotLessThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, CompareNotLessThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, CompareOrdered, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, CompareUnordered, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, Compress, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, CompressStore, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(AVX512, ConvertScalarToVector128Double, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2sd32, INS_vcvtusi2sd32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX512, ConvertScalarToVector128Single, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2ss32, INS_vcvtusi2ss32, INS_invalid, INS_invalid, INS_invalid, INS_cvtsd2ss, INS_invalid, 5, 4, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, ConvertToInt32, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtss2si32, INS_cvtsd2si32, INS_invalid, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, ConvertToUInt32, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtss2usi32, INS_vcvtsd2usi32, INS_invalid, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, ConvertToUInt32WithTruncation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttss2usi32, INS_vcvttsd2usi32, INS_invalid, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128Byte, -1, 1, INS_invalid, INS_invalid, INS_vpmovwb, INS_vpmovwb, INS_vpmovdb, INS_vpmovdb, INS_vpmovqb, INS_vpmovqb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128ByteWithSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_vpmovuswb, INS_invalid, INS_vpmovusdb, INS_invalid, INS_vpmovusqb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128Double, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtudq2pd, INS_vcvtqq2pd, INS_vcvtuqq2pd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128Int16, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovdw, INS_vpmovdw, INS_vpmovqw, INS_vpmovqw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128Int16WithSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovsdw, INS_invalid, INS_vpmovsqw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128Int32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovqd, INS_vpmovqd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128Int32WithSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovsqd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128Int64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2qq, INS_vcvtpd2qq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128Int64WithTruncation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2qq, INS_vcvttpd2qq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128SByte, -1, 1, INS_invalid, INS_invalid, INS_vpmovwb, INS_vpmovwb, INS_vpmovdb, INS_vpmovdb, INS_vpmovqb, INS_vpmovqb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128SByteWithSaturation, -1, 1, INS_invalid, INS_invalid, INS_vpmovswb, INS_invalid, INS_vpmovsdb, INS_invalid, INS_vpmovsqb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128Single, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtudq2ps, INS_vcvtqq2ps, INS_vcvtuqq2ps, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128UInt16, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovdw, INS_vpmovdw, INS_vpmovqw, INS_vpmovqw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128UInt16WithSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovusdw, INS_invalid, INS_vpmovusqw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128UInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovqd, INS_vpmovqd, INS_vcvtps2udq, INS_vcvtpd2udq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128UInt32WithSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovusqd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128UInt32WithTruncation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2udq, INS_vcvttpd2udq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128UInt64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2uqq, INS_vcvtpd2uqq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector128UInt64WithTruncation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2uqq, INS_vcvttpd2uqq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256Byte, 64, 1, INS_invalid, INS_invalid, INS_vpmovwb, INS_vpmovwb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256ByteWithSaturation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_vpmovuswb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256Double, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtudq2pd, INS_vcvtqq2pd, INS_vcvtuqq2pd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256Int16, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovdw, INS_vpmovdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256Int16WithSaturation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovsdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256Int32, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovqd, INS_vpmovqd, INS_invalid, INS_cvtpd2dq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256Int32WithSaturation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovsqd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256Int32WithTruncation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvttpd2dq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256Int64, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2qq, INS_vcvtpd2qq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256Int64WithTruncation, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2qq, INS_vcvttpd2qq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256SByte, 64, 1, INS_invalid, INS_invalid, INS_vpmovwb, INS_vpmovwb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256SByteWithSaturation, 64, 1, INS_invalid, INS_invalid, INS_vpmovswb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256Single, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtudq2ps, INS_vcvtqq2ps, INS_vcvtuqq2ps, INS_invalid, INS_cvtpd2ps, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256UInt16, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovdw, INS_vpmovdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256UInt16WithSaturation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovusdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256UInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovqd, INS_vpmovqd, INS_vcvtps2udq, INS_vcvtpd2udq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256UInt32WithSaturation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovusqd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256UInt32WithTruncation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2udq, INS_vcvttpd2udq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256UInt64, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2uqq, INS_vcvtpd2uqq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector256UInt64WithTruncation, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2uqq, INS_vcvttpd2uqq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector512Double, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtdq2pd, INS_vcvtudq2pd, INS_vcvtqq2pd, INS_vcvtuqq2pd, INS_cvtps2pd, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, ConvertToVector512Int16, 64, 1, INS_pmovsxbw, INS_pmovzxbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector512Int32, 64, -1, INS_pmovsxbd, INS_pmovzxbd, INS_pmovsxwd, INS_pmovzxwd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtps2dq, INS_invalid, INS_invalid, 3, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, ConvertToVector512Int32WithTruncation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvttps2dq, INS_invalid, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector512Int64, 64, -1, INS_pmovsxbq, INS_pmovzxbq, INS_pmovsxwq, INS_pmovzxwq, INS_pmovsxdq, INS_pmovzxdq, INS_invalid, INS_invalid, INS_vcvtps2qq, INS_vcvtpd2qq, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, ConvertToVector512Int64WithTruncation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2qq, INS_vcvttpd2qq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector512Single, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtdq2ps, INS_vcvtudq2ps, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, ConvertToVector512UInt16, 64, 1, INS_pmovsxbw, INS_pmovzxbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector512UInt32, 64, -1, INS_pmovsxbd, INS_pmovzxbd, INS_pmovsxwd, INS_pmovzxwd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2udq, INS_invalid, INS_invalid, 3, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, ConvertToVector512UInt32WithTruncation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2udq, INS_invalid, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, ConvertToVector512UInt64, 64, -1, INS_pmovsxbq, INS_pmovzxbq, INS_pmovsxwq, INS_pmovzxwq, INS_pmovsxdq, INS_pmovzxdq, INS_invalid, INS_invalid, INS_vcvtps2uqq, INS_vcvtpd2uqq, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, ConvertToVector512UInt64WithTruncation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2uqq, INS_vcvttpd2uqq, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX512, DetectConflicts, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpconflictd, INS_vpconflictd, INS_vpconflictq, INS_vpconflictq, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, Divide, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_divps, INS_divpd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, DivideScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_divss, INS_divsd, INS_invalid, -1, -1, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, DuplicateEvenIndexed, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movsldup, INS_movddup, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, DuplicateOddIndexed, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movshdup, INS_invalid, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, Expand, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, ExpandLoad, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, ExtractVector128, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vextracti32x4, INS_vextracti32x4, INS_vextracti64x2, INS_vextracti64x2, INS_vextractf32x4, INS_vextractf64x2, INS_invalid, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX512, ExtractVector256, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vextracti32x8, INS_vextracti32x8, INS_vextracti64x4, INS_vextracti64x4, INS_vextractf32x8, INS_vextractf64x4, INS_invalid, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX512, Fixup, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfixupimmps, INS_vfixupimmpd, INS_invalid, -1, 4, HW_Category_IMM, HW_Flag_SpecialImport|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, FixupScalar, 16, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfixupimmss, INS_vfixupimmsd, INS_invalid, -1, 4, HW_Category_IMM, HW_Flag_SpecialImport|HW_Flag_FullRangeIMM|HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX512, FusedMultiplyAdd, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmadd213ps, INS_vfmadd213pd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, FusedMultiplyAddNegated, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmadd213ps, INS_vfnmadd213pd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, FusedMultiplyAddNegatedScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmadd213ss, INS_vfnmadd213sd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible|HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX512, FusedMultiplyAddScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmadd213ss, INS_vfmadd213sd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible|HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX512, FusedMultiplyAddSubtract, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmaddsub213ps, INS_vfmaddsub213pd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, FusedMultiplySubtract, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmsub213ps, INS_vfmsub213pd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, FusedMultiplySubtractAdd, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmsubadd213ps, INS_vfmsubadd213pd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, FusedMultiplySubtractNegated, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmsub213ps, INS_vfnmsub213pd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, FusedMultiplySubtractNegatedScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmsub213ss, INS_vfnmsub213sd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible|HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX512, FusedMultiplySubtractScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmsub213ss, INS_vfmsub213sd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible|HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX512, GetExponent, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vgetexpps, INS_vgetexppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, GetExponentScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vgetexpss, INS_vgetexpsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX512, GetMantissa, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vgetmantps, INS_vgetmantpd, INS_invalid, -1, 4, HW_Category_IMM, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, GetMantissaScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vgetmantss, INS_vgetmantsd, INS_invalid, -1, 4, HW_Category_IMM, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX512, InsertVector128, 64, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vinserti32x4, INS_vinserti32x4, INS_vinserti64x2, INS_vinserti64x2, INS_vinsertf32x4, INS_vinsertf64x2, INS_invalid, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX512, InsertVector256, 64, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vinserti32x8, INS_vinserti32x8, INS_vinserti64x4, INS_vinserti64x4, INS_vinsertf32x8, INS_vinsertf64x4, INS_invalid, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX512, LeadingZeroCount, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vplzcntd, INS_vplzcntd, INS_vplzcntq, INS_vplzcntq, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, LoadAlignedVector512, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_vmovdqa64, INS_vmovdqa64, INS_movaps, INS_movapd, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX512, LoadAlignedVector512NonTemporal, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movntdqa, INS_movntdqa, INS_movntdqa, INS_movntdqa, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX512, LoadVector512, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, MaskLoad, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, MaskLoadAligned, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512, MaskStore, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(AVX512, MaskStoreAligned, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(AVX512, Max, -1, 2, INS_pmaxsb, INS_pmaxub, INS_pmaxsw, INS_pmaxuw, INS_pmaxsd, INS_pmaxud, INS_vpmaxsq, INS_vpmaxuq, INS_maxps, INS_maxpd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative) -HARDWARE_INTRINSIC(AVX512, Min, -1, 2, INS_pminsb, INS_pminub, INS_pminsw, INS_pminuw, INS_pminsd, INS_pminud, INS_vpminsq, INS_vpminuq, INS_minps, INS_minpd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative) -HARDWARE_INTRINSIC(AVX512, MoveMask, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoContainment|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, Multiply, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pmuldq, INS_pmuludq, INS_mulps, INS_mulpd, INS_invalid, 5, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, MultiplyAddAdjacent, 64, 2, INS_invalid, INS_invalid, INS_pmaddubsw, INS_invalid, INS_pmaddwd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, MultiplyHigh, 64, 2, INS_invalid, INS_invalid, INS_pmulhw, INS_pmulhuw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX512, MultiplyHighRoundScale, 64, 2, INS_invalid, INS_invalid, INS_pmulhrsw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, MultiplyLow, -1, 2, INS_invalid, INS_invalid, INS_pmullw, INS_pmullw, INS_pmulld, INS_pmulld, INS_vpmullq, INS_vpmullq, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX512, MultiplyScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mulss, INS_mulsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, Or, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pord, INS_pord, INS_vporq, INS_vporq, INS_orps, INS_orpd, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX512, PackSignedSaturate, 64, 2, INS_packsswb, INS_invalid, INS_packssdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, PackUnsignedSaturate, 64, 2, INS_invalid, INS_packuswb, INS_invalid, INS_packusdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, Permute2x64, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermilpd, INS_invalid, -1, 1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, Permute4x32, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermilps, INS_invalid, INS_invalid, -1, 1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, Permute4x64, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermq, INS_vpermq, INS_invalid, INS_vpermpd, INS_invalid, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, PermuteVar16x16, 32, 2, INS_invalid, INS_invalid, INS_vpermw, INS_vpermw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 6, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX512, PermuteVar16x16x2, 32, 3, INS_invalid, INS_invalid, INS_vpermt2w, INS_vpermt2w, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 6, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX512, PermuteVar16x32, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermd, INS_vpermd, INS_invalid, INS_invalid, INS_vpermps, INS_invalid, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX512, PermuteVar16x32x2, 64, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermt2d, INS_vpermt2d, INS_invalid, INS_invalid, INS_vpermt2ps, INS_invalid, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX512, PermuteVar2x64, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermilpdvar, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, PermuteVar2x64x2, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermt2q, INS_vpermt2q, INS_invalid, INS_vpermt2pd, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX512, PermuteVar32x16, 64, 2, INS_invalid, INS_invalid, INS_vpermw, INS_vpermw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 6, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX512, PermuteVar32x16x2, 64, 3, INS_invalid, INS_invalid, INS_vpermt2w, INS_vpermt2w, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 6, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX512, PermuteVar4x32, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermilpsvar, INS_invalid, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, PermuteVar4x32x2, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermt2d, INS_vpermt2d, INS_invalid, INS_invalid, INS_vpermt2ps, INS_invalid, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX512, PermuteVar4x64, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermq_reg, INS_vpermq_reg, INS_invalid, INS_vpermpd_reg, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX512, PermuteVar4x64x2, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermt2q, INS_vpermt2q, INS_invalid, INS_vpermt2pd, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX512, PermuteVar8x16 , 16, 2, INS_invalid, INS_invalid, INS_vpermw, INS_vpermw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 6, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX512, PermuteVar8x16x2, 16, 3, INS_invalid, INS_invalid, INS_vpermt2w, INS_vpermt2w, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 6, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX512, PermuteVar8x32x2, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermt2d, INS_vpermt2d, INS_invalid, INS_invalid, INS_vpermt2ps, INS_invalid, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX512, PermuteVar8x64, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermq_reg, INS_vpermq_reg, INS_invalid, INS_vpermpd_reg, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX512, PermuteVar8x64x2, 64, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermt2q, INS_vpermt2q, INS_invalid, INS_vpermt2pd, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX512, Range, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrangeps, INS_vrangepd, INS_invalid, -1, 4, HW_Category_IMM, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, RangeScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrangess, INS_vrangesd, INS_invalid, -1, 4, HW_Category_IMM, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX512, Reciprocal14, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrcp14ps, INS_vrcp14pd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, Reciprocal14Scalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrcp14ss, INS_vrcp14sd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX512, ReciprocalSqrt14, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrsqrt14ps, INS_vrsqrt14pd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, ReciprocalSqrt14Scalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrsqrt14ss, INS_vrsqrt14sd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX512, Reduce, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vreduceps, INS_vreducepd, INS_invalid, -1, 4, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, ReduceScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vreducess, INS_vreducesd, INS_invalid, -1, 4, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX512, RotateLeft, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vprold, INS_vprold, INS_vprolq, INS_vprolq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_MaybeNoJmpTableIMM|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, RotateLeftVariable, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vprolvd, INS_vprolvd, INS_vprolvq, INS_vprolvq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, RotateRight, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vprord, INS_vprord, INS_vprorq, INS_vprorq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_MaybeNoJmpTableIMM|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, RotateRightVariable, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vprorvd, INS_vprorvd, INS_vprorvq, INS_vprorvq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, RoundScale, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrndscaleps, INS_vrndscalepd, INS_invalid, -1, 8, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, RoundScaleScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrndscaless, INS_vrndscalesd, INS_invalid, -1, 8, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX512, Scale, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vscalefps, INS_vscalefpd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, ScaleScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vscalefss, INS_vscalefsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, ShiftLeftLogical, 64, 2, INS_invalid, INS_invalid, INS_psllw, INS_psllw, INS_pslld, INS_pslld, INS_psllq, INS_psllq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, ShiftLeftLogical128BitLane, 64, 2, INS_pslldq, INS_pslldq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, ShiftLeftLogicalVariable, -1, 2, INS_invalid, INS_invalid, INS_vpsllvw, INS_vpsllvw, INS_vpsllvd, INS_vpsllvd, INS_vpsllvq, INS_vpsllvq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, ShiftRightArithmetic, -1, 2, INS_invalid, INS_invalid, INS_psraw, INS_invalid, INS_psrad, INS_invalid, INS_vpsraq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, ShiftRightArithmeticVariable, -1, 2, INS_invalid, INS_invalid, INS_vpsravw, INS_invalid, INS_vpsravd, INS_invalid, INS_vpsravq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, ShiftRightLogical, 64, 2, INS_invalid, INS_invalid, INS_psrlw, INS_psrlw, INS_psrld, INS_psrld, INS_psrlq, INS_psrlq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, ShiftRightLogical128BitLane, 64, 2, INS_psrldq, INS_psrldq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, ShiftRightLogicalVariable, -1, 2, INS_invalid, INS_invalid, INS_vpsrlvw, INS_vpsrlvw, INS_vpsrlvd, INS_vpsrlvd, INS_vpsrlvq, INS_vpsrlvq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, Shuffle, 64, -1, INS_pshufb, INS_pshufb, INS_invalid, INS_invalid, INS_pshufd, INS_pshufd, INS_invalid, INS_invalid, INS_shufps, INS_shufpd, INS_invalid, 1, 1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, Shuffle2x128, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vshufi32x4, INS_vshufi32x4, INS_vshufi64x2, INS_vshufi64x2, INS_vshuff32x4, INS_vshuff64x2, INS_invalid, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, Shuffle4x128, 64, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vshufi32x4, INS_vshufi32x4, INS_vshufi64x2, INS_vshufi64x2, INS_vshuff32x4, INS_vshuff64x2, INS_invalid, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, ShuffleHigh, 64, 2, INS_invalid, INS_invalid, INS_pshufhw, INS_pshufhw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, ShuffleLow, 64, 2, INS_invalid, INS_invalid, INS_pshuflw, INS_pshuflw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, Sqrt, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqrtps, INS_sqrtpd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, SqrtScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqrtss, INS_sqrtsd, INS_invalid, -1, -1, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, Store, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(AVX512, StoreAligned, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_vmovdqa64, INS_vmovdqa64, INS_movaps, INS_movapd, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX512, StoreAlignedNonTemporal, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movntdq, INS_movntdq, INS_movntdq, INS_movntdq, INS_movntps, INS_movntpd, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX512, Subtract, 64, -1, INS_psubb, INS_psubb, INS_psubw, INS_psubw, INS_psubd, INS_psubd, INS_psubq, INS_psubq, INS_subps, INS_subpd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, SubtractSaturate, 64, 2, INS_psubsb, INS_psubusb, INS_psubsw, INS_psubusw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, SubtractScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_subss, INS_subsd, INS_invalid, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512, SumAbsoluteDifferences, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_psadbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, SumAbsoluteDifferencesInBlock32, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_vdbpsadbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, TernaryLogic, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpternlogd, INS_vpternlogd, INS_vpternlogq, INS_vpternlogq, INS_vpternlogd, INS_vpternlogq, INS_invalid, 1, 1, HW_Category_IMM, HW_Flag_SpecialImport|HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX512, UnpackHigh, 64, 2, INS_punpckhbw, INS_punpckhbw, INS_punpckhwd, INS_punpckhwd, INS_punpckhdq, INS_punpckhdq, INS_punpckhqdq, INS_punpckhqdq, INS_unpckhps, INS_unpckhpd, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, UnpackLow, 64, 2, INS_punpcklbw, INS_punpcklbw, INS_punpcklwd, INS_punpcklwd, INS_punpckldq, INS_punpckldq, INS_punpcklqdq, INS_punpcklqdq, INS_unpcklps, INS_unpcklpd, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, Xor, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pxord, INS_pxord, INS_vpxorq, INS_vpxorq, INS_xorps, INS_xorpd, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX512, Abs, -1, 1, INS_pabsb, INS_invalid, INS_pabsw, INS_invalid, INS_pabsd, INS_invalid, INS_vpabsq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, Add, 64, -1, INS_paddb, INS_paddb, INS_paddw, INS_paddw, INS_paddd, INS_paddd, INS_paddq, INS_paddq, INS_addps, INS_addpd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, AddSaturate, 64, 2, INS_paddsb, INS_paddusb, INS_paddsw, INS_paddusw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX512, AddScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_addss, INS_addsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, AlignRight, 64, 3, INS_palignr, INS_palignr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, AlignRight32, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_valignd, INS_valignd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, AlignRight64, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_valignq, INS_valignq, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, And, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pandd, INS_pandd, INS_vpandq, INS_vpandq, INS_andps, INS_andpd, 1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX512, AndNot, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pandnd, INS_pandnd, INS_vpandnq, INS_vpandnq, INS_andnps, INS_andnpd, 1, 1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX512, Average, 64, 2, INS_invalid, INS_pavgb, INS_invalid, INS_pavgw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX512, BlendVariable, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, BroadcastPairScalarToVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcasti32x2, INS_vbroadcasti32x2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, BroadcastPairScalarToVector256, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcasti32x2, INS_vbroadcasti32x2, INS_invalid, INS_invalid, INS_vbroadcastf32x2, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, BroadcastPairScalarToVector512, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcasti32x2, INS_vbroadcasti32x2, INS_invalid, INS_invalid, INS_vbroadcastf32x2, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, BroadcastScalarToVector512, 64, 1, INS_vpbroadcastb, INS_vpbroadcastb, INS_vpbroadcastw, INS_vpbroadcastw, INS_vpbroadcastd, INS_vpbroadcastd, INS_vpbroadcastq, INS_vpbroadcastq, INS_vbroadcastss, INS_vbroadcastsd, 3, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, BroadcastVector128ToVector512, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcasti32x4, INS_vbroadcasti32x4, INS_vbroadcasti64x2, INS_vbroadcasti64x2, INS_vbroadcastf32x4, INS_vbroadcastf64x2, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, BroadcastVector256ToVector512, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vbroadcasti32x8, INS_vbroadcasti32x8, INS_vbroadcasti64x4, INS_vbroadcasti64x4, INS_vbroadcastf32x8, INS_vbroadcastf64x4, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, Classify, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_InvalidNodeId|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, ClassifyScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_InvalidNodeId|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, Compare, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, CompareEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, CompareGreaterThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, CompareGreaterThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, CompareLessThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, CompareLessThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, CompareNotEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, CompareNotGreaterThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, CompareNotGreaterThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, CompareNotLessThan, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, CompareNotLessThanOrEqual, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, CompareOrdered, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, CompareUnordered, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, Compress, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, CompressStore, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(AVX512, ConvertScalarToVector128Double, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2sd32, INS_vcvtusi2sd32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX512, ConvertScalarToVector128Single, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2ss32, INS_vcvtusi2ss32, INS_invalid, INS_invalid, INS_invalid, INS_cvtsd2ss, 5, 4, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, ConvertToInt32, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtss2si32, INS_cvtsd2si32, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, ConvertToUInt32, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtss2usi32, INS_vcvtsd2usi32, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, ConvertToUInt32WithTruncation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttss2usi32, INS_vcvttsd2usi32, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128Byte, -1, 1, INS_invalid, INS_invalid, INS_vpmovwb, INS_vpmovwb, INS_vpmovdb, INS_vpmovdb, INS_vpmovqb, INS_vpmovqb, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128ByteWithSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_vpmovuswb, INS_invalid, INS_vpmovusdb, INS_invalid, INS_vpmovusqb, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128Double, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtudq2pd, INS_vcvtqq2pd, INS_vcvtuqq2pd, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128Int16, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovdw, INS_vpmovdw, INS_vpmovqw, INS_vpmovqw, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128Int16WithSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovsdw, INS_invalid, INS_vpmovsqw, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128Int32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovqd, INS_vpmovqd, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128Int32WithSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovsqd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128Int64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2qq, INS_vcvtpd2qq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128Int64WithTruncation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2qq, INS_vcvttpd2qq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128SByte, -1, 1, INS_invalid, INS_invalid, INS_vpmovwb, INS_vpmovwb, INS_vpmovdb, INS_vpmovdb, INS_vpmovqb, INS_vpmovqb, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128SByteWithSaturation, -1, 1, INS_invalid, INS_invalid, INS_vpmovswb, INS_invalid, INS_vpmovsdb, INS_invalid, INS_vpmovsqb, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128Single, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtudq2ps, INS_vcvtqq2ps, INS_vcvtuqq2ps, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128UInt16, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovdw, INS_vpmovdw, INS_vpmovqw, INS_vpmovqw, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128UInt16WithSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovusdw, INS_invalid, INS_vpmovusqw, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128UInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovqd, INS_vpmovqd, INS_vcvtps2udq, INS_vcvtpd2udq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128UInt32WithSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovusqd, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128UInt32WithTruncation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2udq, INS_vcvttpd2udq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128UInt64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2uqq, INS_vcvtpd2uqq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector128UInt64WithTruncation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2uqq, INS_vcvttpd2uqq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256Byte, 64, 1, INS_invalid, INS_invalid, INS_vpmovwb, INS_vpmovwb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256ByteWithSaturation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_vpmovuswb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256Double, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtudq2pd, INS_vcvtqq2pd, INS_vcvtuqq2pd, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256Int16, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovdw, INS_vpmovdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256Int16WithSaturation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovsdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256Int32, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovqd, INS_vpmovqd, INS_invalid, INS_cvtpd2dq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256Int32WithSaturation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovsqd, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256Int32WithTruncation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvttpd2dq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256Int64, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2qq, INS_vcvtpd2qq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256Int64WithTruncation, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2qq, INS_vcvttpd2qq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256SByte, 64, 1, INS_invalid, INS_invalid, INS_vpmovwb, INS_vpmovwb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256SByteWithSaturation, 64, 1, INS_invalid, INS_invalid, INS_vpmovswb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256Single, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtudq2ps, INS_vcvtqq2ps, INS_vcvtuqq2ps, INS_invalid, INS_cvtpd2ps, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256UInt16, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovdw, INS_vpmovdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256UInt16WithSaturation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovusdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256UInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovqd, INS_vpmovqd, INS_vcvtps2udq, INS_vcvtpd2udq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256UInt32WithSaturation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmovusqd, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256UInt32WithTruncation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2udq, INS_vcvttpd2udq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256UInt64, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2uqq, INS_vcvtpd2uqq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector256UInt64WithTruncation, 32, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2uqq, INS_vcvttpd2uqq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector512Double, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtdq2pd, INS_vcvtudq2pd, INS_vcvtqq2pd, INS_vcvtuqq2pd, INS_cvtps2pd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, ConvertToVector512Int16, 64, 1, INS_pmovsxbw, INS_pmovzxbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector512Int32, 64, -1, INS_pmovsxbd, INS_pmovzxbd, INS_pmovsxwd, INS_pmovzxwd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtps2dq, INS_invalid, 3, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, ConvertToVector512Int32WithTruncation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvttps2dq, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector512Int64, 64, -1, INS_pmovsxbq, INS_pmovzxbq, INS_pmovsxwq, INS_pmovzxwq, INS_pmovsxdq, INS_pmovzxdq, INS_invalid, INS_invalid, INS_vcvtps2qq, INS_vcvtpd2qq, 3, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, ConvertToVector512Int64WithTruncation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2qq, INS_vcvttpd2qq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector512Single, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtdq2ps, INS_vcvtudq2ps, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, ConvertToVector512UInt16, 64, 1, INS_pmovsxbw, INS_pmovzxbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector512UInt32, 64, -1, INS_pmovsxbd, INS_pmovzxbd, INS_pmovsxwd, INS_pmovzxwd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2udq, INS_invalid, 3, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, ConvertToVector512UInt32WithTruncation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2udq, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, ConvertToVector512UInt64, 64, -1, INS_pmovsxbq, INS_pmovzxbq, INS_pmovsxwq, INS_pmovzxwq, INS_pmovsxdq, INS_pmovzxdq, INS_invalid, INS_invalid, INS_vcvtps2uqq, INS_vcvtpd2uqq, 3, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, ConvertToVector512UInt64WithTruncation, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2uqq, INS_vcvttpd2uqq, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512, DetectConflicts, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpconflictd, INS_vpconflictd, INS_vpconflictq, INS_vpconflictq, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, Divide, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_divps, INS_divpd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, DivideScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_divss, INS_divsd, -1, -1, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, DuplicateEvenIndexed, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movsldup, INS_movddup, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, DuplicateOddIndexed, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movshdup, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, Expand, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, ExpandLoad, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, ExtractVector128, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vextracti32x4, INS_vextracti32x4, INS_vextracti64x2, INS_vextracti64x2, INS_vextractf32x4, INS_vextractf64x2, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX512, ExtractVector256, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vextracti32x8, INS_vextracti32x8, INS_vextracti64x4, INS_vextracti64x4, INS_vextractf32x8, INS_vextractf64x4, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX512, Fixup, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfixupimmps, INS_vfixupimmpd, -1, 4, HW_Category_IMM, HW_Flag_SpecialImport|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, FixupScalar, 16, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfixupimmss, INS_vfixupimmsd, -1, 4, HW_Category_IMM, HW_Flag_SpecialImport|HW_Flag_FullRangeIMM|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX512, FusedMultiplyAdd, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmadd213ps, INS_vfmadd213pd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, FusedMultiplyAddNegated, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmadd213ps, INS_vfnmadd213pd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, FusedMultiplyAddNegatedScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmadd213ss, INS_vfnmadd213sd, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX512, FusedMultiplyAddScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmadd213ss, INS_vfmadd213sd, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX512, FusedMultiplyAddSubtract, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmaddsub213ps, INS_vfmaddsub213pd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, FusedMultiplySubtract, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmsub213ps, INS_vfmsub213pd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, FusedMultiplySubtractAdd, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmsubadd213ps, INS_vfmsubadd213pd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, FusedMultiplySubtractNegated, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmsub213ps, INS_vfnmsub213pd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, FusedMultiplySubtractNegatedScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfnmsub213ss, INS_vfnmsub213sd, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX512, FusedMultiplySubtractScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmsub213ss, INS_vfmsub213sd, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX512, GetExponent, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vgetexpps, INS_vgetexppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, GetExponentScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vgetexpss, INS_vgetexpsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX512, GetMantissa, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vgetmantps, INS_vgetmantpd, -1, 4, HW_Category_IMM, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, GetMantissaScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vgetmantss, INS_vgetmantsd, -1, 4, HW_Category_IMM, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX512, InsertVector128, 64, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vinserti32x4, INS_vinserti32x4, INS_vinserti64x2, INS_vinserti64x2, INS_vinsertf32x4, INS_vinsertf64x2, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX512, InsertVector256, 64, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vinserti32x8, INS_vinserti32x8, INS_vinserti64x4, INS_vinserti64x4, INS_vinsertf32x8, INS_vinsertf64x4, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX512, LeadingZeroCount, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vplzcntd, INS_vplzcntd, INS_vplzcntq, INS_vplzcntq, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, LoadAlignedVector512, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_vmovdqa64, INS_vmovdqa64, INS_movaps, INS_movapd, -1, -1, HW_Category_MemoryLoad, HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX512, LoadAlignedVector512NonTemporal, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movntdqa, INS_movntdqa, INS_movntdqa, INS_movntdqa, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX512, LoadVector512, 64, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, MaskLoad, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, MaskLoadAligned, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512, MaskStore, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(AVX512, MaskStoreAligned, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(AVX512, Max, -1, 2, INS_pmaxsb, INS_pmaxub, INS_pmaxsw, INS_pmaxuw, INS_pmaxsd, INS_pmaxud, INS_vpmaxsq, INS_vpmaxuq, INS_maxps, INS_maxpd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative) +HARDWARE_INTRINSIC(AVX512, Min, -1, 2, INS_pminsb, INS_pminub, INS_pminsw, INS_pminuw, INS_pminsd, INS_pminud, INS_vpminsq, INS_vpminuq, INS_minps, INS_minpd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative) +HARDWARE_INTRINSIC(AVX512, MoveMask, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoContainment|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, Multiply, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pmuldq, INS_pmuludq, INS_mulps, INS_mulpd, 5, 4, HW_Category_SimpleSIMD, HW_Flag_MaybeCommutative|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, MultiplyAddAdjacent, 64, 2, INS_invalid, INS_invalid, INS_pmaddubsw, INS_invalid, INS_pmaddwd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, MultiplyHigh, 64, 2, INS_invalid, INS_invalid, INS_pmulhw, INS_pmulhuw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX512, MultiplyHighRoundScale, 64, 2, INS_invalid, INS_invalid, INS_pmulhrsw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, MultiplyLow, -1, 2, INS_invalid, INS_invalid, INS_pmullw, INS_pmullw, INS_pmulld, INS_pmulld, INS_vpmullq, INS_vpmullq, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX512, MultiplyScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mulss, INS_mulsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, Or, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pord, INS_pord, INS_vporq, INS_vporq, INS_orps, INS_orpd, 1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX512, PackSignedSaturate, 64, 2, INS_packsswb, INS_invalid, INS_packssdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, PackUnsignedSaturate, 64, 2, INS_invalid, INS_packuswb, INS_invalid, INS_packusdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, Permute2x64, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermilpd, -1, 1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, Permute4x32, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermilps, INS_invalid, -1, 1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, Permute4x64, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermq, INS_vpermq, INS_invalid, INS_vpermpd, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, PermuteVar16x16, 32, 2, INS_invalid, INS_invalid, INS_vpermw, INS_vpermw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 6, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX512, PermuteVar16x16x2, 32, 3, INS_invalid, INS_invalid, INS_vpermt2w, INS_vpermt2w, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 6, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX512, PermuteVar16x32, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermd, INS_vpermd, INS_invalid, INS_invalid, INS_vpermps, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX512, PermuteVar16x32x2, 64, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermt2d, INS_vpermt2d, INS_invalid, INS_invalid, INS_vpermt2ps, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX512, PermuteVar2x64, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermilpdvar, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, PermuteVar2x64x2, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermt2q, INS_vpermt2q, INS_invalid, INS_vpermt2pd, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX512, PermuteVar32x16, 64, 2, INS_invalid, INS_invalid, INS_vpermw, INS_vpermw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 6, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX512, PermuteVar32x16x2, 64, 3, INS_invalid, INS_invalid, INS_vpermt2w, INS_vpermt2w, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 6, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX512, PermuteVar4x32, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermilpsvar, INS_invalid, -1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, PermuteVar4x32x2, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermt2d, INS_vpermt2d, INS_invalid, INS_invalid, INS_vpermt2ps, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX512, PermuteVar4x64, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermq_reg, INS_vpermq_reg, INS_invalid, INS_vpermpd_reg, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX512, PermuteVar4x64x2, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermt2q, INS_vpermt2q, INS_invalid, INS_vpermt2pd, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX512, PermuteVar8x16 , 16, 2, INS_invalid, INS_invalid, INS_vpermw, INS_vpermw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 6, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX512, PermuteVar8x16x2, 16, 3, INS_invalid, INS_invalid, INS_vpermt2w, INS_vpermt2w, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 6, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX512, PermuteVar8x32x2, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermt2d, INS_vpermt2d, INS_invalid, INS_invalid, INS_vpermt2ps, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX512, PermuteVar8x64, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermq_reg, INS_vpermq_reg, INS_invalid, INS_vpermpd_reg, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX512, PermuteVar8x64x2, 64, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermt2q, INS_vpermt2q, INS_invalid, INS_vpermt2pd, 3, 3, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX512, Range, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrangeps, INS_vrangepd, -1, 4, HW_Category_IMM, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, RangeScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrangess, INS_vrangesd, -1, 4, HW_Category_IMM, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX512, Reciprocal14, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrcp14ps, INS_vrcp14pd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, Reciprocal14Scalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrcp14ss, INS_vrcp14sd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX512, ReciprocalSqrt14, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrsqrt14ps, INS_vrsqrt14pd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, ReciprocalSqrt14Scalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrsqrt14ss, INS_vrsqrt14sd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX512, Reduce, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vreduceps, INS_vreducepd, -1, 4, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, ReduceScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vreducess, INS_vreducesd, -1, 4, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX512, RotateLeft, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vprold, INS_vprold, INS_vprolq, INS_vprolq, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_MaybeNoJmpTableIMM|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, RotateLeftVariable, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vprolvd, INS_vprolvd, INS_vprolvq, INS_vprolvq, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, RotateRight, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vprord, INS_vprord, INS_vprorq, INS_vprorq, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_MaybeNoJmpTableIMM|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, RotateRightVariable, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vprorvd, INS_vprorvd, INS_vprorvq, INS_vprorvq, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, RoundScale, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrndscaleps, INS_vrndscalepd, -1, 8, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, RoundScaleScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrndscaless, INS_vrndscalesd, -1, 8, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX512, Scale, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vscalefps, INS_vscalefpd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, ScaleScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vscalefss, INS_vscalefsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, ShiftLeftLogical, 64, 2, INS_invalid, INS_invalid, INS_psllw, INS_psllw, INS_pslld, INS_pslld, INS_psllq, INS_psllq, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, ShiftLeftLogical128BitLane, 64, 2, INS_pslldq, INS_pslldq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, ShiftLeftLogicalVariable, -1, 2, INS_invalid, INS_invalid, INS_vpsllvw, INS_vpsllvw, INS_vpsllvd, INS_vpsllvd, INS_vpsllvq, INS_vpsllvq, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, ShiftRightArithmetic, -1, 2, INS_invalid, INS_invalid, INS_psraw, INS_invalid, INS_psrad, INS_invalid, INS_vpsraq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, ShiftRightArithmeticVariable, -1, 2, INS_invalid, INS_invalid, INS_vpsravw, INS_invalid, INS_vpsravd, INS_invalid, INS_vpsravq, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, ShiftRightLogical, 64, 2, INS_invalid, INS_invalid, INS_psrlw, INS_psrlw, INS_psrld, INS_psrld, INS_psrlq, INS_psrlq, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, ShiftRightLogical128BitLane, 64, 2, INS_psrldq, INS_psrldq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, ShiftRightLogicalVariable, -1, 2, INS_invalid, INS_invalid, INS_vpsrlvw, INS_vpsrlvw, INS_vpsrlvd, INS_vpsrlvd, INS_vpsrlvq, INS_vpsrlvq, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, Shuffle, 64, -1, INS_pshufb, INS_pshufb, INS_invalid, INS_invalid, INS_pshufd, INS_pshufd, INS_invalid, INS_invalid, INS_shufps, INS_shufpd, 1, 1, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, Shuffle2x128, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vshufi32x4, INS_vshufi32x4, INS_vshufi64x2, INS_vshufi64x2, INS_vshuff32x4, INS_vshuff64x2, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, Shuffle4x128, 64, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vshufi32x4, INS_vshufi32x4, INS_vshufi64x2, INS_vshufi64x2, INS_vshuff32x4, INS_vshuff64x2, 3, 3, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, ShuffleHigh, 64, 2, INS_invalid, INS_invalid, INS_pshufhw, INS_pshufhw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, ShuffleLow, 64, 2, INS_invalid, INS_invalid, INS_pshuflw, INS_pshuflw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, Sqrt, 64, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqrtps, INS_sqrtpd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, SqrtScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqrtss, INS_sqrtsd, -1, -1, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, Store, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(AVX512, StoreAligned, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_vmovdqa64, INS_vmovdqa64, INS_movaps, INS_movapd, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX512, StoreAlignedNonTemporal, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movntdq, INS_movntdq, INS_movntdq, INS_movntdq, INS_movntps, INS_movntpd, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX512, Subtract, 64, -1, INS_psubb, INS_psubb, INS_psubw, INS_psubw, INS_psubd, INS_psubd, INS_psubq, INS_psubq, INS_subps, INS_subpd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, SubtractSaturate, 64, 2, INS_psubsb, INS_psubusb, INS_psubsw, INS_psubusw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, SubtractScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_subss, INS_subsd, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512, SumAbsoluteDifferences, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_psadbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, SumAbsoluteDifferencesInBlock32, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_vdbpsadbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, TernaryLogic, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpternlogd, INS_vpternlogd, INS_vpternlogq, INS_vpternlogq, INS_vpternlogd, INS_vpternlogq, 1, 1, HW_Category_IMM, HW_Flag_SpecialImport|HW_Flag_FullRangeIMM|HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX512, UnpackHigh, 64, 2, INS_punpckhbw, INS_punpckhbw, INS_punpckhwd, INS_punpckhwd, INS_punpckhdq, INS_punpckhdq, INS_punpckhqdq, INS_punpckhqdq, INS_unpckhps, INS_unpckhpd, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, UnpackLow, 64, 2, INS_punpcklbw, INS_punpcklbw, INS_punpcklwd, INS_punpcklwd, INS_punpckldq, INS_punpckldq, INS_punpcklqdq, INS_punpcklqdq, INS_unpcklps, INS_unpcklpd, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, Xor, 64, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pxord, INS_pxord, INS_vpxorq, INS_vpxorq, INS_xorps, INS_xorpd, 1, 1, HW_Category_SimpleSIMD, HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp|HW_Flag_NormalizeSmallTypeToInt) #define LAST_NI_AVX512 NI_AVX512_Xor // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -620,11 +620,11 @@ HARDWARE_INTRINSIC(AVX512, Xor, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // 64-bit only Intrinsics for AVX512F, AVX512BW, AVX512CD, AVX512DQ, AVX512VL #define FIRST_NI_AVX512_X64 NI_AVX512_X64_ConvertScalarToVector128Double -HARDWARE_INTRINSIC(AVX512_X64, ConvertScalarToVector128Double, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2sd64, INS_vcvtusi2sd64, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_CopyUpperBits|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512_X64, ConvertScalarToVector128Single, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2ss64, INS_vcvtusi2ss64, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_CopyUpperBits|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512_X64, ConvertToInt64, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtss2si64, INS_cvtsd2si64, INS_invalid, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512_X64, ConvertToUInt64, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtss2usi64, INS_vcvtsd2usi64, INS_invalid, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX512_X64, ConvertToUInt64WithTruncation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttss2usi64, INS_vcvttsd2usi64, INS_invalid, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512_X64, ConvertScalarToVector128Double, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2sd64, INS_vcvtusi2sd64, INS_invalid, INS_invalid, 5, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_CopyUpperBits|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512_X64, ConvertScalarToVector128Single, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtsi2ss64, INS_vcvtusi2ss64, INS_invalid, INS_invalid, 5, -1, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromSecondArg|HW_Flag_CopyUpperBits|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512_X64, ConvertToInt64, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cvtss2si64, INS_cvtsd2si64, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512_X64, ConvertToUInt64, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtss2usi64, INS_vcvtsd2usi64, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX512_X64, ConvertToUInt64WithTruncation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttss2usi64, INS_vcvttsd2usi64, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) #define LAST_NI_AVX512_X64 NI_AVX512_X64_ConvertToUInt64WithTruncation // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -633,13 +633,13 @@ HARDWARE_INTRINSIC(AVX512_X64, ConvertToUInt64WithTruncation, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Intrinsics for AVX512-IFMA, AVX512-VBMI #define FIRST_NI_AVX512v2 NI_AVX512v2_MultiShift -HARDWARE_INTRINSIC(AVX512v2, MultiShift, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmultishiftqb, INS_vpmultishiftqb, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(AVX512v2, PermuteVar16x8, 16, 2, INS_vpermb, INS_vpermb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX512v2, PermuteVar16x8x2, 16, 3, INS_vpermt2b, INS_vpermt2b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX512v2, PermuteVar32x8, 32, 2, INS_vpermb, INS_vpermb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX512v2, PermuteVar32x8x2, 32, 3, INS_vpermt2b, INS_vpermt2b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX512v2, PermuteVar64x8, 64, 2, INS_vpermb, INS_vpermb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVX512v2, PermuteVar64x8x2, 64, 3, INS_vpermt2b, INS_vpermt2b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX512v2, MultiShift, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmultishiftqb, INS_vpmultishiftqb, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(AVX512v2, PermuteVar16x8, 16, 2, INS_vpermb, INS_vpermb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX512v2, PermuteVar16x8x2, 16, 3, INS_vpermt2b, INS_vpermt2b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX512v2, PermuteVar32x8, 32, 2, INS_vpermb, INS_vpermb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX512v2, PermuteVar32x8x2, 32, 3, INS_vpermt2b, INS_vpermt2b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX512v2, PermuteVar64x8, 64, 2, INS_vpermb, INS_vpermb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVX512v2, PermuteVar64x8x2, 64, 3, INS_vpermt2b, INS_vpermt2b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) #define LAST_NI_AVX512v2 NI_AVX512v2_PermuteVar64x8x2 // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -648,12 +648,12 @@ HARDWARE_INTRINSIC(AVX512v2, PermuteVar64x8x2, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Intrinsics for AVX512-BITALG, AVX512-VBMI2, AVX512-VPOPCNTDQ, AVX512-VNNI #define FIRST_NI_AVX512v3 NI_AVX512v3_Compress -HARDWARE_INTRINSIC(AVX512v3, Compress, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512v3, CompressStore, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(AVX512v3, Expand, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512v3, ExpandLoad, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_InvalidNodeId) -HARDWARE_INTRINSIC(AVX512v3, MultiplyWideningAndAdd, 64, 3, INS_invalid, INS_vpdpbusd, INS_vpdpwssd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(AVX512v3, MultiplyWideningAndAddSaturate, 64, 3, INS_invalid, INS_vpdpbusds, INS_vpdpwssds, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(AVX512v3, Compress, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512v3, CompressStore, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(AVX512v3, Expand, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512v3, ExpandLoad, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(AVX512v3, MultiplyWideningAndAdd, 64, 3, INS_invalid, INS_vpdpbusd, INS_vpdpwssd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(AVX512v3, MultiplyWideningAndAddSaturate, 64, 3, INS_invalid, INS_vpdpbusds, INS_vpdpwssds, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg) #define LAST_NI_AVX512v3 NI_AVX512v3_MultiplyWideningAndAddSaturate // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -662,34 +662,34 @@ HARDWARE_INTRINSIC(AVX512v3, MultiplyWideningAndAddSaturate, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Intrinsics for AVX10v1 #define FIRST_NI_AVX10v1 NI_AVX10v1_AddScalar -HARDWARE_INTRINSIC(AVX10v1, AddScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vaddsh, -1, 4, HW_Category_SIMDScalar, HW_Flag_EmbRoundingCompatible|HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX10v1, CompareScalarOrderedEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcomish, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX10v1, CompareScalarOrderedGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcomish, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX10v1, CompareScalarOrderedGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcomish, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX10v1, CompareScalarOrderedLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcomish, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX10v1, CompareScalarOrderedLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcomish, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX10v1, CompareScalarOrderedNotEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcomish, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX10v1, CompareScalarUnorderedEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vucomish, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX10v1, CompareScalarUnorderedGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vucomish, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX10v1, CompareScalarUnorderedGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vucomish, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX10v1, CompareScalarUnorderedLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vucomish, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX10v1, CompareScalarUnorderedLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vucomish, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX10v1, CompareScalarUnorderedNotEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vucomish, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX10v1, ConvertScalarToVector128Double, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtsh2sd, 5, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX10v1, ConvertScalarToVector128Half, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtsi2sh32, INS_vcvtusi2sh32, INS_vcvtsi2sh64, INS_vcvtusi2sh64, INS_vcvtss2sh, INS_vcvtsd2sh, INS_invalid, 5, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX10v1, ConvertScalarToVector128Single, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtsh2ss, 5, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX10v1, ConvertToInt32, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtsh2si32, -1, 7, HW_Category_SIMDScalar, HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX10v1, ConvertToInt64, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtsh2si64, -1, 7, HW_Category_SIMDScalar, HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX10v1, ConvertToUInt32, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtsh2usi32, -1, 7, HW_Category_SIMDScalar, HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX10v1, ConvertToUInt64, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtsh2usi64, -1, 7, HW_Category_SIMDScalar, HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX10v1, DivideScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vdivsh, -1, -1, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX10v1, FusedMultiplyAddScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfmadd213sh, -1, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible|HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX10v1, MultiplyScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vmulsh, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX10v1, ReciprocalScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrcpsh, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX10v1, ReciprocalSqrtScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrsqrtsh, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX10v1, RoundScaleScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vrndscalesh, -1, 8, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX10v1, SqrtScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vsqrtsh, -1, -1, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) -HARDWARE_INTRINSIC(AVX10v1, SubtractScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vsubsh, -1, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX10v1, AddScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vaddsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_SIMDScalar, HW_Flag_EmbRoundingCompatible|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarOrderedEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vcomish, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarOrderedGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vcomish, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarOrderedGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vcomish, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarOrderedLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vcomish, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarOrderedLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vcomish, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarOrderedNotEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vcomish, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarUnorderedEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vucomish, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarUnorderedGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vucomish, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarUnorderedGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vucomish, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarUnorderedLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vucomish, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarUnorderedLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vucomish, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarUnorderedNotEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vucomish, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, ConvertScalarToVector128Double, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vcvtsh2sd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX10v1, ConvertScalarToVector128Half, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtsi2sh32, INS_vcvtusi2sh32, INS_vcvtsi2sh64, INS_vcvtusi2sh64, INS_vcvtss2sh, INS_vcvtsd2sh, 5, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX10v1, ConvertScalarToVector128Single, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vcvtsh2ss, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX10v1, ConvertToInt32WithTruncation, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vcvttsh2si32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 7, 7, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, ConvertToInt64WithTruncation, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vcvttsh2si64, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 7, 7, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, ConvertToUInt32WithTruncation, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vcvttsh2usi32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 7, 7, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, ConvertToUInt64WithTruncation, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vcvttsh2usi64, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 7, 7, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, DivideScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vdivsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX10v1, FusedMultiplyAddScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_vfmadd213sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX10v1, MultiplyScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vmulsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX10v1, ReciprocalScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vrcpsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX10v1, ReciprocalSqrtScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vrsqrtsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX10v1, RoundScaleScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_vrndscalesh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 8, 8, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX10v1, SqrtScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vsqrtsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX10v1, SubtractScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vsubsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) #define LAST_NI_AVX10v1 NI_AVX10v1_SubtractScalar // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -698,21 +698,21 @@ HARDWARE_INTRINSIC(AVX10v1, SubtractScalar, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Intrinsics for AVX10v2 #define FIRST_NI_AVX10v2 NI_AVX10v2_ConvertToByteWithSaturationAndZeroExtendToInt32 -HARDWARE_INTRINSIC(AVX10v2, ConvertToByteWithSaturationAndZeroExtendToInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2iubs, INS_invalid, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX10v2, ConvertToByteWithTruncatedSaturationAndZeroExtendToInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2iubs, INS_invalid, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX10v2, ConvertToInt32WithTruncatedSaturation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttss2sis32, INS_vcvttsd2sis32, INS_invalid, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX10v2, ConvertToSByteWithSaturationAndZeroExtendToInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2ibs, INS_invalid, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX10v2, ConvertToSByteWithTruncatedSaturationAndZeroExtendToInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2ibs, INS_invalid, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX10v2, ConvertToUInt32WithTruncatedSaturation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttss2usis32, INS_vcvttsd2usis32, INS_invalid, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX10v2, ConvertToVectorInt32WithTruncatedSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2dqs, INS_vcvttpd2dqs, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX10v2, ConvertToVectorInt64WithTruncatedSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2qqs, INS_vcvttpd2qqs, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX10v2, ConvertToVectorUInt32WithTruncatedSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2udqs, INS_vcvttpd2udqs, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX10v2, ConvertToVectorUInt64WithTruncatedSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2uqqs, INS_vcvttpd2uqqs, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX10v2, MinMax, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vminmaxps, INS_vminmaxpd, INS_invalid, -1, 4, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX10v2, MinMaxScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vminmaxss, INS_vminmaxsd, INS_invalid, -1, 4, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(AVX10v2, MoveScalar, 16, -1, INS_invalid, INS_invalid, INS_vmovw_simd, INS_vmovw_simd, INS_vmovd_simd, INS_vmovd_simd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_NoContainment) -HARDWARE_INTRINSIC(AVX10v2, MultipleSumAbsoluteDifferences, 64, 3, INS_invalid, INS_invalid, INS_invalid, INS_vmpsadbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX10v2, StoreScalar, 16, 2, INS_invalid, INS_invalid, INS_vmovw_simd, INS_vmovw_simd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(AVX10v2, ConvertToByteWithSaturationAndZeroExtendToInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2iubs, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX10v2, ConvertToByteWithTruncatedSaturationAndZeroExtendToInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2iubs, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX10v2, ConvertToInt32WithTruncatedSaturation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttss2sis32, INS_vcvttsd2sis32, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX10v2, ConvertToSByteWithSaturationAndZeroExtendToInt32, -1, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2ibs, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX10v2, ConvertToSByteWithTruncatedSaturationAndZeroExtendToInt32, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2ibs, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX10v2, ConvertToUInt32WithTruncatedSaturation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttss2usis32, INS_vcvttsd2usis32, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX10v2, ConvertToVectorInt32WithTruncatedSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2dqs, INS_vcvttpd2dqs, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX10v2, ConvertToVectorInt64WithTruncatedSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2qqs, INS_vcvttpd2qqs, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX10v2, ConvertToVectorUInt32WithTruncatedSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2udqs, INS_vcvttpd2udqs, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX10v2, ConvertToVectorUInt64WithTruncatedSaturation, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2uqqs, INS_vcvttpd2uqqs, -1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX10v2, MinMax, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vminmaxps, INS_vminmaxpd, -1, 4, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX10v2, MinMaxScalar, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vminmaxss, INS_vminmaxsd, -1, 4, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX10v2, MoveScalar, 16, -1, INS_invalid, INS_invalid, INS_vmovw_simd, INS_vmovw_simd, INS_vmovd_simd, INS_vmovd_simd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SIMDScalar, HW_Flag_NoContainment) +HARDWARE_INTRINSIC(AVX10v2, MultipleSumAbsoluteDifferences, 64, 3, INS_invalid, INS_invalid, INS_invalid, INS_vmpsadbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX10v2, StoreScalar, 16, 2, INS_invalid, INS_invalid, INS_vmovw_simd, INS_vmovw_simd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromSecondArg) #define LAST_NI_AVX10v2 NI_AVX10v2_StoreScalar // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -721,14 +721,14 @@ HARDWARE_INTRINSIC(AVX10v2, StoreScalar, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // 64-bit only intrinsics for AVX10v2 #define FIRST_NI_AVX10v2_X64 NI_AVX10v2_X64_ConvertToInt64WithTruncatedSaturation -HARDWARE_INTRINSIC(AVX10v2_X64, ConvertToInt64WithTruncatedSaturation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttss2sis64, INS_vcvttsd2sis64, INS_invalid, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX10v2_X64, ConvertToUInt64WithTruncatedSaturation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttss2usis64, INS_vcvttsd2usis64, INS_invalid, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX10v2_X64, ConvertToInt64WithTruncatedSaturation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttss2sis64, INS_vcvttsd2sis64, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX10v2_X64, ConvertToUInt64WithTruncatedSaturation, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttss2usis64, INS_vcvttsd2usis64, -1, 7, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) #define LAST_NI_AVX10v2_X64 NI_AVX10v2_X64_ConvertToUInt64WithTruncatedSaturation #define FIRST_NI_AVX512BMM NI_AVX512BMM_BitMultiplyMatrix16x16WithOrReduction -HARDWARE_INTRINSIC(AVX512BMM, BitMultiplyMatrix16x16WithOrReduction, -1, -1, INS_invalid, INS_invalid, INS_vbmacor16x16x16, INS_vbmacor16x16x16, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX512BMM, BitMultiplyMatrix16x16WithXorReduction, -1, -1, INS_invalid, INS_invalid, INS_vbmacxor16x16x16, INS_vbmacxor16x16x16, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX512BMM, ReverseBits, -1, -1, INS_invalid, INS_vbitrev, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(AVX512BMM, BitMultiplyMatrix16x16WithOrReduction, -1, -1, INS_invalid, INS_invalid, INS_vbmacor16x16x16, INS_vbmacor16x16x16, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX512BMM, BitMultiplyMatrix16x16WithXorReduction, -1, -1, INS_invalid, INS_invalid, INS_vbmacxor16x16x16, INS_vbmacxor16x16x16, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_RmwIntrinsic) +HARDWARE_INTRINSIC(AVX512BMM, ReverseBits, -1, -1, INS_invalid, INS_vbitrev, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg) #define LAST_NI_AVX512BMM NI_AVX512BMM_ReverseBits // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -737,8 +737,8 @@ HARDWARE_INTRINSIC(AVX512BMM, ReverseBits, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Intrinsics for AVXVNNI #define FIRST_NI_AVXVNNI NI_AVXVNNI_MultiplyWideningAndAdd -HARDWARE_INTRINSIC(AVXVNNI, MultiplyWideningAndAdd, -1, 3, INS_invalid, INS_vpdpbusd, INS_vpdpwssd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVXVNNI, MultiplyWideningAndAddSaturate, -1, 3, INS_invalid, INS_vpdpbusds, INS_vpdpwssds, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVXVNNI, MultiplyWideningAndAdd, -1, 3, INS_invalid, INS_vpdpbusd, INS_vpdpwssd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVXVNNI, MultiplyWideningAndAddSaturate, -1, 3, INS_invalid, INS_vpdpbusds, INS_vpdpwssds, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoEvexSemantics) #define LAST_NI_AVXVNNI NI_AVXVNNI_MultiplyWideningAndAddSaturate // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -747,8 +747,8 @@ HARDWARE_INTRINSIC(AVXVNNI, MultiplyWideningAndAddSaturate, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // AVXVNNIINT Intrinsics #define FIRST_NI_AVXVNNIINT NI_AVXVNNIINT_MultiplyWideningAndAdd -HARDWARE_INTRINSIC(AVXVNNIINT, MultiplyWideningAndAdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVXVNNIINT, MultiplyWideningAndAddSaturate, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVXVNNIINT, MultiplyWideningAndAdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVXVNNIINT, MultiplyWideningAndAddSaturate, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) #define LAST_NI_AVXVNNIINT NI_AVXVNNIINT_MultiplyWideningAndAddSaturate @@ -758,8 +758,8 @@ HARDWARE_INTRINSIC(AVXVNNIINT, MultiplyWideningAndAddSaturate, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // AVXVNNIINT_V512 Intrinsics #define FIRST_NI_AVXVNNIINT_V512 NI_AVXVNNIINT_V512_MultiplyWideningAndAdd -HARDWARE_INTRINSIC(AVXVNNIINT_V512, MultiplyWideningAndAdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(AVXVNNIINT_V512, MultiplyWideningAndAddSaturate, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVXVNNIINT_V512, MultiplyWideningAndAdd, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(AVXVNNIINT_V512, MultiplyWideningAndAddSaturate, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) #define LAST_NI_AVXVNNIINT_V512 NI_AVXVNNIINT_V512_MultiplyWideningAndAddSaturate // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -768,13 +768,13 @@ HARDWARE_INTRINSIC(AVXVNNIINT_V512, MultiplyWideningAndAddSaturate, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Intrinsics for AES, PCLMULQDQ #define FIRST_NI_AES NI_AES_CarrylessMultiply -HARDWARE_INTRINSIC(AES, CarrylessMultiply, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pclmulqdq, INS_pclmulqdq, INS_invalid, INS_invalid, INS_invalid, 7, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AES, Decrypt, 16, 2, INS_invalid, INS_aesdec, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AES, DecryptLast, 16, 2, INS_invalid, INS_aesdeclast, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AES, Encrypt, 16, 2, INS_invalid, INS_aesenc, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AES, EncryptLast, 16, 2, INS_invalid, INS_aesenclast, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AES, InverseMixColumns, 16, 1, INS_invalid, INS_aesimc, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 8, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AES, KeygenAssist, 16, 2, INS_invalid, INS_aeskeygenassist, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 6, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AES, CarrylessMultiply, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pclmulqdq, INS_pclmulqdq, INS_invalid, INS_invalid, 7, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AES, Decrypt, 16, 2, INS_invalid, INS_aesdec, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AES, DecryptLast, 16, 2, INS_invalid, INS_aesdeclast, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AES, Encrypt, 16, 2, INS_invalid, INS_aesenc, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AES, EncryptLast, 16, 2, INS_invalid, INS_aesenclast, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AES, InverseMixColumns, 16, 1, INS_invalid, INS_aesimc, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 8, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AES, KeygenAssist, 16, 2, INS_invalid, INS_aeskeygenassist, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 6, -1, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_NoEvexSemantics) #define LAST_NI_AES NI_AES_KeygenAssist // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -783,7 +783,7 @@ HARDWARE_INTRINSIC(AES, KeygenAssist, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Intrinsics for VAES, VPCLMULQDQ #define FIRST_NI_AES_V256 NI_AES_V256_CarrylessMultiply -HARDWARE_INTRINSIC(AES_V256, CarrylessMultiply, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pclmulqdq, INS_pclmulqdq, INS_invalid, INS_invalid, INS_invalid, 7, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AES_V256, CarrylessMultiply, 32, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pclmulqdq, INS_pclmulqdq, INS_invalid, INS_invalid, 7, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) #define LAST_NI_AES_V256 NI_AES_V256_CarrylessMultiply // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -792,7 +792,7 @@ HARDWARE_INTRINSIC(AES_V256, CarrylessMultiply, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // AVX Intrinsics for VAES, VPCLMULQDQ #define FIRST_NI_AES_V512 NI_AES_V512_CarrylessMultiply -HARDWARE_INTRINSIC(AES_V512, CarrylessMultiply, 64, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pclmulqdq, INS_pclmulqdq, INS_invalid, INS_invalid, INS_invalid, 7, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AES_V512, CarrylessMultiply, 64, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pclmulqdq, INS_pclmulqdq, INS_invalid, INS_invalid, 7, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) #define LAST_NI_AES_V512 NI_AES_V512_CarrylessMultiply // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -801,7 +801,7 @@ HARDWARE_INTRINSIC(AES_V512, CarrylessMultiply, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // AVX512 Intrinsics for X86Serialize #define FIRST_NI_X86Serialize NI_X86Serialize_Serialize -HARDWARE_INTRINSIC(X86Serialize, Serialize, 0, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Barrier) +HARDWARE_INTRINSIC(X86Serialize, Serialize, 0, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Barrier) #define LAST_NI_X86Serialize NI_X86Serialize_Serialize // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -810,9 +810,9 @@ HARDWARE_INTRINSIC(X86Serialize, Serialize, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Intrinsics for GFNI #define FIRST_NI_GFNI NI_GFNI_GaloisFieldAffineTransform -HARDWARE_INTRINSIC(GFNI, GaloisFieldAffineTransform, 16, 3, INS_invalid, INS_gf2p8affineqb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_gf2p8affineqb, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(GFNI, GaloisFieldAffineTransformInverse, 16, 3, INS_invalid, INS_gf2p8affineinvqb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_gf2p8affineinvqb, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(GFNI, GaloisFieldMultiply, 16, 2, INS_invalid, INS_gf2p8mulb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(GFNI, GaloisFieldAffineTransform, 16, 3, INS_invalid, INS_gf2p8affineqb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_gf2p8affineqb, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(GFNI, GaloisFieldAffineTransformInverse, 16, 3, INS_invalid, INS_gf2p8affineinvqb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_gf2p8affineinvqb, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(GFNI, GaloisFieldMultiply, 16, 2, INS_invalid, INS_gf2p8mulb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) #define LAST_NI_GFNI NI_GFNI_GaloisFieldMultiply // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -821,9 +821,9 @@ HARDWARE_INTRINSIC(GFNI, GaloisFieldMultiply, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // AVX Intrinsics for GFNI #define FIRST_NI_GFNI_V256 NI_GFNI_V256_GaloisFieldAffineTransform -HARDWARE_INTRINSIC(GFNI_V256, GaloisFieldAffineTransform, 32, 3, INS_invalid, INS_gf2p8affineqb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_gf2p8affineqb, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(GFNI_V256, GaloisFieldAffineTransformInverse, 32, 3, INS_invalid, INS_gf2p8affineinvqb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_gf2p8affineinvqb, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(GFNI_V256, GaloisFieldMultiply, 32, 2, INS_invalid, INS_gf2p8mulb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(GFNI_V256, GaloisFieldAffineTransform, 32, 3, INS_invalid, INS_gf2p8affineqb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_gf2p8affineqb, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(GFNI_V256, GaloisFieldAffineTransformInverse, 32, 3, INS_invalid, INS_gf2p8affineinvqb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_gf2p8affineinvqb, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(GFNI_V256, GaloisFieldMultiply, 32, 2, INS_invalid, INS_gf2p8mulb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) #define LAST_NI_GFNI_V256 NI_GFNI_V256_GaloisFieldMultiply // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -832,9 +832,9 @@ HARDWARE_INTRINSIC(GFNI_V256, GaloisFieldMultiply, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // AVX512 Intrinsics for GFNI #define FIRST_NI_GFNI_V512 NI_GFNI_V512_GaloisFieldAffineTransform -HARDWARE_INTRINSIC(GFNI_V512, GaloisFieldAffineTransform, 64, 3, INS_invalid, INS_gf2p8affineqb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_gf2p8affineqb, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(GFNI_V512, GaloisFieldAffineTransformInverse, 64, 3, INS_invalid, INS_gf2p8affineinvqb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_gf2p8affineinvqb, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(GFNI_V512, GaloisFieldMultiply, 64, 2, INS_invalid, INS_gf2p8mulb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(GFNI_V512, GaloisFieldAffineTransform, 64, 3, INS_invalid, INS_gf2p8affineqb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_gf2p8affineqb, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(GFNI_V512, GaloisFieldAffineTransformInverse, 64, 3, INS_invalid, INS_gf2p8affineinvqb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_gf2p8affineinvqb, INS_invalid, INS_invalid, 3, -1, HW_Category_IMM, HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(GFNI_V512, GaloisFieldMultiply, 64, 2, INS_invalid, INS_gf2p8mulb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) #define LAST_NI_GFNI_V512 NI_GFNI_V512_GaloisFieldMultiply // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** @@ -842,55 +842,55 @@ HARDWARE_INTRINSIC(GFNI_V512, GaloisFieldMultiply, // TYP_BYTE, TYP_UBYTE, TYP_SHORT, TYP_USHORT, TYP_INT, TYP_UINT, TYP_LONG, TYP_ULONG, TYP_FLOAT, TYP_DOUBLE // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Special intrinsics that are generated during lowering -HARDWARE_INTRINSIC(X86Base, COMIS, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_comiss, INS_comisd, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(X86Base, PTEST, 16, 2, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(X86Base, UCOMIS, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ucomiss, INS_ucomisd, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(AVX10v1, VCOMISH, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcomish, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(AVX10v1, VUCOMISH, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vucomish, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics) -HARDWARE_INTRINSIC(AVX, PTEST, -1, 2, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_vtestps, INS_vtestpd, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX2, AndNotVector, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pandnd, INS_pandnd, INS_pandnd, INS_pandnd, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NormalizeSmallTypeToInt) -HARDWARE_INTRINSIC(AVX2, AndNotScalar, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_andn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) -HARDWARE_INTRINSIC(AVX512, KORTEST, 0, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_Special, HW_Flag_NoContainment) -HARDWARE_INTRINSIC(AVX512, KTEST, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_Special, HW_Flag_NoContainment) -HARDWARE_INTRINSIC(AVX512, PTESTM, 0, 2, INS_vptestmb, INS_vptestmb, INS_vptestmw, INS_vptestmw, INS_vptestmd, INS_vptestmd, INS_vptestmq, INS_vptestmq, INS_vptestmd, INS_vptestmq, INS_invalid, 4, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX512, PTESTNM, 0, 2, INS_vptestnmb, INS_vptestnmb, INS_vptestnmw, INS_vptestnmw, INS_vptestnmd, INS_vptestnmd, INS_vptestnmq, INS_vptestnmq, INS_vptestnmd, INS_vptestnmq, INS_invalid, 4, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(X86Base, COMIS, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_comiss, INS_comisd, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(X86Base, PTEST, 16, 2, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(X86Base, UCOMIS, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ucomiss, INS_ucomisd, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(AVX10v1, VCOMISH, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vcomish, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(AVX10v1, VUCOMISH, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vucomish, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(AVX, PTEST, -1, 2, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_vtestps, INS_vtestpd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX2, AndNotVector, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pandnd, INS_pandnd, INS_pandnd, INS_pandnd, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NormalizeSmallTypeToInt) +HARDWARE_INTRINSIC(AVX2, AndNotScalar, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_andn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVX512, KORTEST, 0, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_Special, HW_Flag_NoContainment) +HARDWARE_INTRINSIC(AVX512, KTEST, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_Special, HW_Flag_NoContainment) +HARDWARE_INTRINSIC(AVX512, PTESTM, 0, 2, INS_vptestmb, INS_vptestmb, INS_vptestmw, INS_vptestmw, INS_vptestmd, INS_vptestmd, INS_vptestmq, INS_vptestmq, INS_vptestmd, INS_vptestmq, 4, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative) +HARDWARE_INTRINSIC(AVX512, PTESTNM, 0, 2, INS_vptestnmb, INS_vptestnmb, INS_vptestnmw, INS_vptestnmw, INS_vptestnmd, INS_vptestnmd, INS_vptestnmq, INS_vptestnmq, INS_vptestnmd, INS_vptestnmq, 4, 4, HW_Category_SimpleSIMD, HW_Flag_Commutative) -HARDWARE_INTRINSIC(AVX512, AddMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, AndMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, AndNotMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, BlendVariableMask, -1, 3, INS_vpblendmb, INS_vpblendmb, INS_vpblendmw, INS_vpblendmw, INS_vpblendmd, INS_vpblendmd, INS_vpblendmq, INS_vpblendmq, INS_vblendmps, INS_vblendmpd, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, ClassifyMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfpclassps, INS_vfpclasspd, INS_invalid, -1, 3, HW_Category_IMM, HW_Flag_ReturnsPerElementMask|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, ClassifyScalarMask, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfpclassss, INS_vfpclasssd, INS_invalid, -1, 3, HW_Category_IMM, HW_Flag_ReturnsPerElementMask|HW_Flag_FullRangeIMM) -HARDWARE_INTRINSIC(AVX512, CompareMask, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpps, INS_vcmppd, INS_invalid, -1, 4, HW_Category_IMM, HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(AVX512, CompareEqualMask, -1, 2, INS_vpcmpeqb, INS_vpcmpeqb, INS_vpcmpeqw, INS_vpcmpeqw, INS_vpcmpeqd, INS_vpcmpeqd, INS_vpcmpeqq, INS_vpcmpeqq, INS_vcmpps, INS_vcmppd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AVX512, CompareGreaterThanMask, -1, 2, INS_vpcmpgtb, INS_vpcmpub, INS_vpcmpgtw, INS_vpcmpuw, INS_vpcmpgtd, INS_vpcmpud, INS_vpcmpgtq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(AVX512, CompareGreaterThanOrEqualMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(AVX512, CompareLessThanMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(AVX512, CompareLessThanOrEqualMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(AVX512, CompareNotEqualMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp) -HARDWARE_INTRINSIC(AVX512, CompareNotGreaterThanMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(AVX512, CompareNotGreaterThanOrEqualMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(AVX512, CompareNotLessThanMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(AVX512, CompareNotLessThanOrEqualMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, INS_invalid, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(AVX512, CompareOrderedMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpps, INS_vcmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(AVX512, CompareScalarMask, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpss, INS_vcmpsd, INS_invalid, -1, 4, HW_Category_IMM, HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(AVX512, CompareUnorderedMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpps, INS_vcmppd, INS_invalid, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(AVX512, CompressMask, -1, 3, INS_vpcompressb, INS_vpcompressb, INS_vpcompressw, INS_vpcompressw, INS_vpcompressd, INS_vpcompressd, INS_vpcompressq, INS_vpcompressq, INS_vcompressps, INS_vcompresspd, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(AVX512, CompressStoreMask, -1, 3, INS_vpcompressb, INS_vpcompressb, INS_vpcompressw, INS_vpcompressw, INS_vpcompressd, INS_vpcompressd, INS_vpcompressq, INS_vpcompressq, INS_vcompressps, INS_vcompresspd, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, ConvertMaskToVector, -1, 1, INS_vpmovm2b, INS_vpmovm2b, INS_vpmovm2w, INS_vpmovm2w, INS_vpmovm2d, INS_vpmovm2d, INS_vpmovm2q, INS_vpmovm2q, INS_vpmovm2d, INS_vpmovm2q, INS_invalid, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(AVX512, ConvertVectorToMask, -1, 1, INS_vpmovb2m, INS_vpmovb2m, INS_vpmovw2m, INS_vpmovw2m, INS_vpmovd2m, INS_vpmovd2m, INS_vpmovq2m, INS_vpmovq2m, INS_vpmovd2m, INS_vpmovq2m, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(AVX512, ExpandLoadMask, -1, 3, INS_vpexpandb, INS_vpexpandb, INS_vpexpandw, INS_vpexpandw, INS_vpexpandd, INS_vpexpandd, INS_vpexpandq, INS_vpexpandq, INS_vexpandps, INS_vexpandpd, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, ExpandMask, -1, 3, INS_vpexpandb, INS_vpexpandb, INS_vpexpandw, INS_vpexpandw, INS_vpexpandd, INS_vpexpandd, INS_vpexpandq, INS_vpexpandq, INS_vexpandps, INS_vexpandpd, INS_invalid, 3, 3, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, MaskLoadMask, -1, 3, INS_vmovdqu8, INS_vmovdqu8, INS_vmovdqu16, INS_vmovdqu16, INS_movdqu32, INS_movdqu32, INS_vmovdqu64, INS_vmovdqu64, INS_movups, INS_movupd, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, MaskLoadAlignedMask, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_vmovdqa64, INS_vmovdqa64, INS_movaps, INS_movapd, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512, MaskStoreMask, -1, 3, INS_vmovdqu8, INS_vmovdqu8, INS_vmovdqu16, INS_vmovdqu16, INS_movdqu32, INS_movdqu32, INS_vmovdqu64, INS_vmovdqu64, INS_movups, INS_movupd, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(AVX512, MaskStoreAlignedMask, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_vmovdqa64, INS_vmovdqa64, INS_movaps, INS_movapd, INS_invalid, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(AVX512, NotMask, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, OrMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ShiftLeftMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, ShiftRightMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, XorMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(AVX512, XnorMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, AddMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, AndMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, AndNotMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, BlendVariableMask, -1, 3, INS_vpblendmb, INS_vpblendmb, INS_vpblendmw, INS_vpblendmw, INS_vpblendmd, INS_vpblendmd, INS_vpblendmq, INS_vpblendmq, INS_vblendmps, INS_vblendmpd, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, ClassifyMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfpclassps, INS_vfpclasspd, -1, 3, HW_Category_IMM, HW_Flag_ReturnsPerElementMask|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, ClassifyScalarMask, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vfpclassss, INS_vfpclasssd, -1, 3, HW_Category_IMM, HW_Flag_ReturnsPerElementMask|HW_Flag_FullRangeIMM) +HARDWARE_INTRINSIC(AVX512, CompareMask, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpps, INS_vcmppd, -1, 4, HW_Category_IMM, HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(AVX512, CompareEqualMask, -1, 2, INS_vpcmpeqb, INS_vpcmpeqb, INS_vpcmpeqw, INS_vpcmpeqw, INS_vpcmpeqd, INS_vpcmpeqd, INS_vpcmpeqq, INS_vpcmpeqq, INS_vcmpps, INS_vcmppd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AVX512, CompareGreaterThanMask, -1, 2, INS_vpcmpgtb, INS_vpcmpub, INS_vpcmpgtw, INS_vpcmpuw, INS_vpcmpgtd, INS_vpcmpud, INS_vpcmpgtq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(AVX512, CompareGreaterThanOrEqualMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(AVX512, CompareLessThanMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(AVX512, CompareLessThanOrEqualMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(AVX512, CompareNotEqualMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask|HW_Flag_Commutative|HW_Flag_CanBenefitFromConstantProp) +HARDWARE_INTRINSIC(AVX512, CompareNotGreaterThanMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(AVX512, CompareNotGreaterThanOrEqualMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(AVX512, CompareNotLessThanMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(AVX512, CompareNotLessThanOrEqualMask, -1, 2, INS_vpcmpb, INS_vpcmpub, INS_vpcmpw, INS_vpcmpuw, INS_vpcmpd, INS_vpcmpud, INS_vpcmpq, INS_vpcmpuq, INS_vcmpps, INS_vcmppd, 1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(AVX512, CompareOrderedMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpps, INS_vcmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(AVX512, CompareScalarMask, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpss, INS_vcmpsd, -1, 4, HW_Category_IMM, HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(AVX512, CompareUnorderedMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcmpps, INS_vcmppd, -1, 4, HW_Category_SimpleSIMD, HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(AVX512, CompressMask, -1, 3, INS_vpcompressb, INS_vpcompressb, INS_vpcompressw, INS_vpcompressw, INS_vpcompressd, INS_vpcompressd, INS_vpcompressq, INS_vpcompressq, INS_vcompressps, INS_vcompresspd, 3, 3, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(AVX512, CompressStoreMask, -1, 3, INS_vpcompressb, INS_vpcompressb, INS_vpcompressw, INS_vpcompressw, INS_vpcompressd, INS_vpcompressd, INS_vpcompressq, INS_vpcompressq, INS_vcompressps, INS_vcompresspd, -1, -1, HW_Category_MemoryStore, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, ConvertMaskToVector, -1, 1, INS_vpmovm2b, INS_vpmovm2b, INS_vpmovm2w, INS_vpmovm2w, INS_vpmovm2d, INS_vpmovm2d, INS_vpmovm2q, INS_vpmovm2q, INS_vpmovm2d, INS_vpmovm2q, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(AVX512, ConvertVectorToMask, -1, 1, INS_vpmovb2m, INS_vpmovb2m, INS_vpmovw2m, INS_vpmovw2m, INS_vpmovd2m, INS_vpmovd2m, INS_vpmovq2m, INS_vpmovq2m, INS_vpmovd2m, INS_vpmovq2m, 3, 3, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(AVX512, ExpandLoadMask, -1, 3, INS_vpexpandb, INS_vpexpandb, INS_vpexpandw, INS_vpexpandw, INS_vpexpandd, INS_vpexpandd, INS_vpexpandq, INS_vpexpandq, INS_vexpandps, INS_vexpandpd, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, ExpandMask, -1, 3, INS_vpexpandb, INS_vpexpandb, INS_vpexpandw, INS_vpexpandw, INS_vpexpandd, INS_vpexpandd, INS_vpexpandq, INS_vpexpandq, INS_vexpandps, INS_vexpandpd, 3, 3, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, MaskLoadMask, -1, 3, INS_vmovdqu8, INS_vmovdqu8, INS_vmovdqu16, INS_vmovdqu16, INS_movdqu32, INS_movdqu32, INS_vmovdqu64, INS_vmovdqu64, INS_movups, INS_movupd, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, MaskLoadAlignedMask, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_vmovdqa64, INS_vmovdqa64, INS_movaps, INS_movapd, -1, -1, HW_Category_MemoryLoad, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512, MaskStoreMask, -1, 3, INS_vmovdqu8, INS_vmovdqu8, INS_vmovdqu16, INS_vmovdqu16, INS_movdqu32, INS_movdqu32, INS_vmovdqu64, INS_vmovdqu64, INS_movups, INS_movupd, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(AVX512, MaskStoreAlignedMask, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqa32, INS_movdqa32, INS_vmovdqa64, INS_vmovdqa64, INS_movaps, INS_movapd, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(AVX512, NotMask, -1, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, OrMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ShiftLeftMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, ShiftRightMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, XorMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX512, XnorMask, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, 1, HW_Category_SimpleSIMD, HW_Flag_NoContainment|HW_Flag_Commutative|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) #endif // FEATURE_HW_INTRINSIC #undef HARDWARE_INTRINSIC diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index e957cd2bcd6fd4..e90963fbb8726b 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -4519,61 +4519,132 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, if (retType == TYP_STRUCT) { + // Converting some arithmetic type -> Half assert(isSystemHalfClass(retClsHnd)); assert(varTypeIsArithmetic(op1Type)); - switch (op1Type) +#if defined(TARGET_XARCH) + if (compOpportunisticallyDependsOn(InstructionSet_AVX10v1)) { - case TYP_FLOAT: + bool supported = false; + + switch (op1Type) { -#if defined(TARGET_XARCH) - if (compOpportunisticallyDependsOn(InstructionSet_AVX2)) - { - GenTree* op1 = impPopStack().val; - op1 = gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, op1, TYP_FLOAT, 16); + case TYP_FLOAT: + case TYP_DOUBLE: + case TYP_INT: + case TYP_UINT: + supported = true; + break; +#ifdef TARGET_AMD64 + case TYP_LONG: + case TYP_ULONG: + supported = true; + break; +#endif // TARGET_AMD64 + default: + break; + } - retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, gtNewIconNode(0), - NI_AVX2_ConvertToVector128Half, TYP_FLOAT, 16); - retNode = impSimdToScalarHalf(retNode, retClsHnd); + if (supported) + { + GenTree* op1 = impPopStack().val; + GenTree* zeroVec = gtNewZeroConNode(TYP_SIMD16); + + // The integer scalar convert instructions read the source directly from a general + // purpose register, so only floating-point sources need to be moved into a vector. + if (varTypeIsFloating(op1Type)) + { + op1 = gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, op1, op1Type, 16); } -#endif + + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, zeroVec, op1, + NI_AVX10v1_ConvertScalarToVector128Half, op1Type, 16); + retNode = impSimdToScalarHalf(retNode, retClsHnd); break; } + } - default: - { - unreached(); - } + if ((op1Type == TYP_FLOAT) && compOpportunisticallyDependsOn(InstructionSet_AVX2)) + { + GenTree* op1 = impPopStack().val; + op1 = gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, op1, TYP_FLOAT, 16); + + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, gtNewIconNode(0), + NI_AVX2_ConvertToVector128Half, TYP_FLOAT, 16); + retNode = impSimdToScalarHalf(retNode, retClsHnd); } +#endif // TARGET_XARCH } else { + // Converting Half -> some arithmetic type assert(varTypeIsArithmetic(retType)); assert((op1Type == TYP_STRUCT) && isSystemHalfClass(op1ClsHnd)); - switch (retType) +#if defined(TARGET_XARCH) + if (compOpportunisticallyDependsOn(InstructionSet_AVX10v1)) { - case TYP_FLOAT: + NamedIntrinsic opId = NI_Illegal; + + switch (retType) { -#if defined(TARGET_XARCH) - if (compOpportunisticallyDependsOn(InstructionSet_AVX2)) - { - GenTree* op1 = impPopStack().val; - op1 = impSimdCreateScalarHalf(op1); + case TYP_FLOAT: + opId = NI_AVX10v1_ConvertScalarToVector128Single; + break; + case TYP_DOUBLE: + opId = NI_AVX10v1_ConvertScalarToVector128Double; + break; + case TYP_INT: + opId = NI_AVX10v1_ConvertToInt32WithTruncation; + break; + case TYP_UINT: + opId = NI_AVX10v1_ConvertToUInt32WithTruncation; + break; +#ifdef TARGET_AMD64 + case TYP_LONG: + opId = NI_AVX10v1_ConvertToInt64WithTruncation; + break; + case TYP_ULONG: + opId = NI_AVX10v1_ConvertToUInt64WithTruncation; + break; +#endif // TARGET_AMD64 + default: + break; + } + + if (opId != NI_Illegal) + { + GenTree* op1 = impPopStack().val; + op1 = impSimdCreateScalarHalf(op1); - retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, NI_AVX2_ConvertToVector128Single, - TYP_USHORT, 16); - retNode = gtNewSimdToScalarNode(TYP_FLOAT, retNode, TYP_FLOAT, 16); + if (varTypeIsFloating(retType)) + { + GenTree* zeroVec = gtNewZeroConNode(TYP_SIMD16); + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, zeroVec, op1, opId, TYP_USHORT, 16); + retNode = gtNewSimdToScalarNode(retType, retNode, retType, 16); + } + else + { + // The result is produced directly in a general purpose register. The node + // type must be the actual machine type (TYP_INT/TYP_LONG); signedness is + // encoded by the selected instruction (vcvttsh2si vs vcvttsh2usi). + retNode = gtNewSimdHWIntrinsicNode(genActualType(retType), op1, opId, TYP_USHORT, 16); } -#endif break; } + } - default: - { - unreached(); - } + if ((retType == TYP_FLOAT) && compOpportunisticallyDependsOn(InstructionSet_AVX2)) + { + GenTree* op1 = impPopStack().val; + op1 = impSimdCreateScalarHalf(op1); + + retNode = + gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, NI_AVX2_ConvertToVector128Single, TYP_USHORT, 16); + retNode = gtNewSimdToScalarNode(TYP_FLOAT, retNode, TYP_FLOAT, 16); } +#endif // TARGET_XARCH } break; } @@ -4594,7 +4665,7 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, op2 = impSimdCreateScalarHalf(op2); op1 = impSimdCreateScalarHalf(op1); - retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, opId, TYP_HALF, 16); + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, opId, TYP_USHORT, 16); retNode = impSimdToScalarHalf(retNode, clsHnd); } #endif // TARGET_XARCH @@ -4617,7 +4688,7 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, // upper bits from the first. We only consume lane 0, so a zeroed upper-bits source is fine. GenTree* op2 = gtNewZeroConNode(TYP_SIMD16); op1 = impSimdCreateScalarHalf(op1); - retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op2, op1, opId, TYP_HALF, 16); + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op2, op1, opId, TYP_USHORT, 16); retNode = impSimdToScalarHalf(retNode, clsHnd); } #endif // TARGET_XARCH @@ -4637,7 +4708,7 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, op2 = impSimdCreateScalarHalf(op2); op1 = impSimdCreateScalarHalf(op1); retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, op3, NI_AVX10v1_FusedMultiplyAddScalar, - TYP_HALF, 16); + TYP_USHORT, 16); retNode = impSimdToScalarHalf(retNode, clsHnd); } #endif // TARGET_XARCH @@ -4660,7 +4731,7 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, GenTree* op2 = gtNewZeroConNode(TYP_SIMD16); op1 = impSimdCreateScalarHalf(op1); retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op2, op1, gtNewIconNode(halfRoundingMode, TYP_INT), - NI_AVX10v1_RoundScaleScalar, TYP_HALF, 16); + NI_AVX10v1_RoundScaleScalar, TYP_USHORT, 16); retNode = impSimdToScalarHalf(retNode, clsHnd); } #endif // TARGET_XARCH @@ -4685,7 +4756,32 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, op2 = impSimdCreateScalarHalf(op2); op1 = impSimdCreateScalarHalf(op1); - retNode = gtNewSimdHWIntrinsicNode(TYP_INT, op1, op2, opId, TYP_HALF, 16); + retNode = gtNewSimdHWIntrinsicNode(TYP_INT, op1, op2, opId, TYP_USHORT, 16); + } +#endif // TARGET_XARCH + break; + } + + case NI_System_Half_op_Increment: + case NI_System_Half_op_Decrement: + { +#if defined(TARGET_XARCH) + if (compOpportunisticallyDependsOn(InstructionSet_AVX10v1)) + { + NamedIntrinsic opId = lookupHalfIntrinsic(ni); + assert(opId != NI_Illegal); + + GenTree* op1 = impPopStack().val; + + // Materialize 1.0 as a Half in lane 0 of a vector by converting 1.0f -> half. + GenTree* oneVec = gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, gtNewDconNodeF(1.0f), TYP_FLOAT, 16); + GenTree* zeroVec = gtNewZeroConNode(TYP_SIMD16); + oneVec = gtNewSimdHWIntrinsicNode(TYP_SIMD16, zeroVec, oneVec, + NI_AVX10v1_ConvertScalarToVector128Half, TYP_FLOAT, 16); + + op1 = impSimdCreateScalarHalf(op1); + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, oneVec, opId, TYP_USHORT, 16); + retNode = impSimdToScalarHalf(retNode, clsHnd); } #endif // TARGET_XARCH break; @@ -12441,6 +12537,14 @@ NamedIntrinsic Compiler::lookupHalfNamedIntrinsic(CORINFO_METHOD_HANDLE method, { result = NI_System_Half_Truncate; } + else if (strcmp(methodName, "op_Increment") == 0) + { + result = NI_System_Half_op_Increment; + } + else if (strcmp(methodName, "op_Decrement") == 0) + { + result = NI_System_Half_op_Decrement; + } return result; } @@ -12464,8 +12568,12 @@ NamedIntrinsic Compiler::lookupHalfIntrinsic(NamedIntrinsic ni) { case NI_System_Half_op_Addition: return NI_AVX10v1_AddScalar; + case NI_System_Half_op_Increment: + return NI_AVX10v1_AddScalar; case NI_System_Half_op_Subtraction: return NI_AVX10v1_SubtractScalar; + case NI_System_Half_op_Decrement: + return NI_AVX10v1_SubtractScalar; case NI_System_Half_op_Multiply: return NI_AVX10v1_MultiplyScalar; case NI_System_Half_op_Division: diff --git a/src/coreclr/jit/namedintrinsiclist.h b/src/coreclr/jit/namedintrinsiclist.h index 49b78e7d26c12a..1ab2b2e14ab05f 100644 --- a/src/coreclr/jit/namedintrinsiclist.h +++ b/src/coreclr/jit/namedintrinsiclist.h @@ -215,7 +215,7 @@ enum NamedIntrinsic : unsigned short NI_HW_INTRINSIC_START, // clang-format off -#define HARDWARE_INTRINSIC(isa, name, simdSize, numArgs, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, intCost, fltCost, category, flag) \ +#define HARDWARE_INTRINSIC(isa, name, simdSize, numArgs, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, intCost, fltCost, category, flag) \ NI_##isa##_##name, #include "hwintrinsiclist.h" // clang-format on diff --git a/src/coreclr/jit/registeropswasm.cpp b/src/coreclr/jit/registeropswasm.cpp index 87d6f1b7463883..112520dd9928b1 100644 --- a/src/coreclr/jit/registeropswasm.cpp +++ b/src/coreclr/jit/registeropswasm.cpp @@ -79,7 +79,6 @@ WasmValueType TypeToWasmValueType(var_types type) WasmValueType::Invalid, // TYP_ULONG, WasmValueType::F32, // TYP_FLOAT, WasmValueType::F64, // TYP_DOUBLE, - WasmValueType::Invalid, // TYP_HALF, WasmValueType::I, // TYP_REF, WasmValueType::I, // TYP_BYREF, WasmValueType::Invalid, // TYP_STRUCT, @@ -123,7 +122,6 @@ WasmValueType ActualTypeToWasmValueType(var_types type) WasmValueType::Invalid, // TYP_ULONG, WasmValueType::F32, // TYP_FLOAT, WasmValueType::F64, // TYP_DOUBLE, - WasmValueType::Invalid, // TYP_HALF, WasmValueType::I, // TYP_REF, WasmValueType::I, // TYP_BYREF, WasmValueType::Invalid, // TYP_STRUCT, diff --git a/src/coreclr/jit/typelist.h b/src/coreclr/jit/typelist.h index 3b374c1cdb7feb..b2cf2ffa1390f4 100644 --- a/src/coreclr/jit/typelist.h +++ b/src/coreclr/jit/typelist.h @@ -54,8 +54,6 @@ DEF_TP(ULONG ,"ulong" , TYP_LONG, 8,EPS,EPS, 2, 8, VTR_INT, available DEF_TP(FLOAT ,"float" , TYP_FLOAT, 4, 4, 4, 1, 4, VTR_FLOAT, availableFloatRegs, RBM_FLT_CALLEE_SAVED, RBM_FLT_CALLEE_TRASH, VTF_FLT) DEF_TP(DOUBLE ,"double" , TYP_DOUBLE, 8, 8, 8, 2, 8, VTR_FLOAT, availableDoubleRegs, RBM_FLT_CALLEE_SAVED, RBM_FLT_CALLEE_TRASH, VTF_FLT) -DEF_TP(HALF ,"half" , TYP_HALF, 2, 2, 2, 1, 2, VTR_FLOAT, availableFloatRegs, RBM_FLT_CALLEE_SAVED, RBM_FLT_CALLEE_TRASH, VTF_S) - DEF_TP(REF ,"ref" , TYP_REF, PS,GCS,GCS, PST,PS,VTR_INT, availableIntRegs, RBM_INT_CALLEE_SAVED, RBM_INT_CALLEE_TRASH, VTF_ANY|VTF_GCR|VTF_I) DEF_TP(BYREF ,"byref" , TYP_BYREF, PS,BRS,BRS, PST,PS,VTR_INT, availableIntRegs, RBM_INT_CALLEE_SAVED, RBM_INT_CALLEE_TRASH, VTF_ANY|VTF_BYR|VTF_I) DEF_TP(STRUCT ,"struct" , TYP_STRUCT, 0, 0, 0, 1, 4, VTR_INT, availableIntRegs, RBM_INT_CALLEE_SAVED, RBM_INT_CALLEE_TRASH, VTF_S) diff --git a/src/coreclr/jit/valuenumfuncs.h b/src/coreclr/jit/valuenumfuncs.h index 84bd04b8a89b6d..956c460fc13282 100644 --- a/src/coreclr/jit/valuenumfuncs.h +++ b/src/coreclr/jit/valuenumfuncs.h @@ -192,7 +192,7 @@ ValueNumFuncDef(SimdType, 2, false, false) // A value number function to compos #if defined(FEATURE_HW_INTRINSICS) ValueNumFuncDef(HWI_INTRINSIC_START, -1, false, false) -#define HARDWARE_INTRINSIC(isa, name, simdSize, numArgs, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, intCost, fltCost, category, flag) \ +#define HARDWARE_INTRINSIC(isa, name, simdSize, numArgs, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, intCost, fltCost, category, flag) \ ValueNumFuncDef(HWI_##isa##_##name, ((numArgs == -1) ? -1 : (numArgs + 1)), ((flag) & HW_Flag_Commutative) >> 0, false) // All of the HARDWARE_INTRINSICS #include "hwintrinsiclist.h" diff --git a/src/libraries/System.Private.CoreLib/src/System/Half.cs b/src/libraries/System.Private.CoreLib/src/System/Half.cs index a1777ad249cd57..7f51823623ce46 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Half.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Half.cs @@ -575,6 +575,7 @@ public bool TryFormat(Span utf8Destination, out int bytesWritten, [StringS /// Explicitly converts a value to its nearest representable half-precision floating-point value. /// The value to convert. /// converted to its nearest representable half-precision floating-point value. + [Intrinsic] public static explicit operator Half(double value) { const int DoubleMaxExponent = 0x7FF; @@ -609,11 +610,13 @@ public static explicit operator Half(double value) /// Explicitly converts a value to its nearest representable half-precision floating-point value. /// The value to convert. /// converted to its nearest representable half-precision floating-point value. + [Intrinsic] public static explicit operator Half(int value) => (Half)(float)value; /// Explicitly converts a value to its nearest representable half-precision floating-point value. /// The value to convert. /// converted to its nearest representable half-precision floating-point value. + [Intrinsic] public static explicit operator Half(long value) => (Half)(float)value; /// Explicitly converts a value to its nearest representable half-precision floating-point value. @@ -798,12 +801,14 @@ public static explicit operator Half(float value) /// The value to convert. /// converted to its nearest representable half-precision floating-point value. [CLSCompliant(false)] + [Intrinsic] public static explicit operator Half(uint value) => (Half)(float)value; /// Explicitly converts a value to its nearest representable half-precision floating-point value. /// The value to convert. /// converted to its nearest representable half-precision floating-point value. [CLSCompliant(false)] + [Intrinsic] public static explicit operator Half(ulong value) => (Half)(float)value; /// Explicitly converts a value to its nearest representable half-precision floating-point value. @@ -857,6 +862,7 @@ public static explicit operator Half(float value) /// Explicitly converts a half-precision floating-point value to its nearest representable value. /// The value to convert. /// converted to its nearest representable value. + [Intrinsic] public static explicit operator int(Half value) => (int)(float)value; /// Explicitly converts a half-precision floating-point value to its nearest representable value, throwing an overflow exception for any values that fall outside the representable range. @@ -868,6 +874,7 @@ public static explicit operator Half(float value) /// Explicitly converts a half-precision floating-point value to its nearest representable value. /// The value to convert. /// converted to its nearest representable value. + [Intrinsic] public static explicit operator long(Half value) => (long)(float)value; /// Explicitly converts a half-precision floating-point value to its nearest representable value, throwing an overflow exception for any values that fall outside the representable range. @@ -928,6 +935,7 @@ public static explicit operator Half(float value) /// The value to convert. /// converted to its nearest representable value. [CLSCompliant(false)] + [Intrinsic] public static explicit operator uint(Half value) => (uint)(float)value; /// Explicitly converts a half-precision floating-point value to its nearest representable value, throwing an overflow exception for any values that fall outside the representable range. @@ -941,6 +949,7 @@ public static explicit operator Half(float value) /// The value to convert. /// converted to its nearest representable value. [CLSCompliant(false)] + [Intrinsic] public static explicit operator ulong(Half value) => (ulong)(float)value; /// Explicitly converts a half-precision floating-point value to its nearest representable value, throwing an overflow exception for any values that fall outside the representable range. @@ -998,6 +1007,7 @@ public static explicit operator Half(float value) /// Explicitly converts a half-precision floating-point value to its nearest representable value. /// The value to convert. /// converted to its nearest representable value. + [Intrinsic] public static explicit operator double(Half value) { bool sign = IsNegative(value); @@ -1275,6 +1285,7 @@ public static bool IsPow2(Half value) // /// + [Intrinsic] public static Half operator --(Half value) { var tmp = (float)value; @@ -1598,6 +1609,7 @@ public static int ILogB(Half x) // /// + [Intrinsic] public static Half operator ++(Half value) { var tmp = (float)value; From aa11831e3916eb4b4fc33114c55e1f19437307ce Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Fri, 10 Jul 2026 08:13:32 -0700 Subject: [PATCH 13/22] Fix register width display for AVX10v1 Half convert intrinsics The scalar Half<->integer converts (vcvttsh2si/vcvttsh2usi and vcvtsi2sh/vcvtusi2sh) were table-driven and emitted with the hardcoded EA_16BYTE SIMD size, so their general purpose operand rendered the 64-bit register name for the 32-bit forms (e.g. `rax`/`rcx` instead of `eax`/`ecx`). The instruction encoding was already correct since the width is baked into the selected instruction, only the disassembly display was wrong. Mark the four ConvertTo*WithTruncation and ConvertScalarToVector128Half intrinsics as HW_Flag_SpecialCodeGen and emit them with the appropriate operand size: emitTypeSize(targetType) for the integer result and emitActualTypeSize(baseType) for an integer source, leaving floating-point sources at the full 128-bit size. Also handle IsXMMReg in emitRegName's EA_2BYTE case for parity with the EA_4BYTE/EA_8BYTE/EA_16BYTE cases. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/emitxarch.cpp | 5 ++++ src/coreclr/jit/hwintrinsiccodegenxarch.cpp | 29 +++++++++++++++++++++ src/coreclr/jit/hwintrinsiclistxarch.h | 10 +++---- 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index 24414afa3c02a9..7bf1d19bcd56b3 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -11986,6 +11986,11 @@ const char* emitter::emitRegName(regNumber reg, emitAttr attr, bool varName) con case EA_2BYTE: { + if (IsXMMReg(reg)) + { + return emitXMMregName(reg); + } + #if defined(TARGET_AMD64) if (reg > REG_RDI) { diff --git a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp index c18175fd8f0dc1..45ffb0fa06eb4d 100644 --- a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp @@ -3703,6 +3703,35 @@ void CodeGen::genAvxFamilyIntrinsic(GenTreeHWIntrinsic* node, insOpts instOption break; } + case NI_AVX10v1_ConvertToInt32WithTruncation: + case NI_AVX10v1_ConvertToUInt32WithTruncation: +#if defined(TARGET_AMD64) + case NI_AVX10v1_ConvertToInt64WithTruncation: + case NI_AVX10v1_ConvertToUInt64WithTruncation: +#endif // TARGET_AMD64 + { + // The source is a scalar `Half` (bound as `TYP_USHORT`) held in a vector register while the + // result is produced directly into a general purpose register. Use the size of the target + // integer type so the correct register width is displayed (e.g. `eax` rather than `rax`). + assert(baseType == TYP_USHORT); + attr = emitTypeSize(targetType); + genHWIntrinsic_R_RM(node, ins, attr, targetReg, op1, instOptions); + break; + } + + case NI_AVX10v1_ConvertScalarToVector128Half: + { + // For integer sources the value is read directly from a general purpose register, so the + // operand size must reflect the source type (e.g. `ecx` rather than `rcx`). Floating-point + // sources come from a vector register and use the full 128-bit size. + if (varTypeIsIntegral(baseType)) + { + attr = emitActualTypeSize(baseType); + } + genHWIntrinsic_R_R_RM(node, ins, attr, instOptions); + break; + } + case NI_AVXVNNIINT_MultiplyWideningAndAddSaturate: case NI_AVXVNNIINT_V512_MultiplyWideningAndAddSaturate: { diff --git a/src/coreclr/jit/hwintrinsiclistxarch.h b/src/coreclr/jit/hwintrinsiclistxarch.h index 195a0de7b6c232..6de21e5056368c 100644 --- a/src/coreclr/jit/hwintrinsiclistxarch.h +++ b/src/coreclr/jit/hwintrinsiclistxarch.h @@ -676,12 +676,12 @@ HARDWARE_INTRINSIC(AVX10v1, CompareScalarUnorderedLessThan, HARDWARE_INTRINSIC(AVX10v1, CompareScalarUnorderedLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vucomish, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) HARDWARE_INTRINSIC(AVX10v1, CompareScalarUnorderedNotEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vucomish, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) HARDWARE_INTRINSIC(AVX10v1, ConvertScalarToVector128Double, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vcvtsh2sd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX10v1, ConvertScalarToVector128Half, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtsi2sh32, INS_vcvtusi2sh32, INS_vcvtsi2sh64, INS_vcvtusi2sh64, INS_vcvtss2sh, INS_vcvtsd2sh, 5, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX10v1, ConvertScalarToVector128Half, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtsi2sh32, INS_vcvtusi2sh32, INS_vcvtsi2sh64, INS_vcvtusi2sh64, INS_vcvtss2sh, INS_vcvtsd2sh, 5, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) HARDWARE_INTRINSIC(AVX10v1, ConvertScalarToVector128Single, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vcvtsh2ss, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) -HARDWARE_INTRINSIC(AVX10v1, ConvertToInt32WithTruncation, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vcvttsh2si32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 7, 7, HW_Category_SIMDScalar, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX10v1, ConvertToInt64WithTruncation, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vcvttsh2si64, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 7, 7, HW_Category_SIMDScalar, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX10v1, ConvertToUInt32WithTruncation, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vcvttsh2usi32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 7, 7, HW_Category_SIMDScalar, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX10v1, ConvertToUInt64WithTruncation, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vcvttsh2usi64, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 7, 7, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, ConvertToInt32WithTruncation, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vcvttsh2si32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 7, 7, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX10v1, ConvertToInt64WithTruncation, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vcvttsh2si64, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 7, 7, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX10v1, ConvertToUInt32WithTruncation, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vcvttsh2usi32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 7, 7, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX10v1, ConvertToUInt64WithTruncation, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vcvttsh2usi64, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 7, 7, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(AVX10v1, DivideScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vdivsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) HARDWARE_INTRINSIC(AVX10v1, FusedMultiplyAddScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_vfmadd213sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible|HW_Flag_CopyUpperBits) HARDWARE_INTRINSIC(AVX10v1, MultiplyScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vmulsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) From 6ed0cf74a30d669ad6b7484db474de83b257b297 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Fri, 10 Jul 2026 08:37:09 -0700 Subject: [PATCH 14/22] Add internal Arm64 Fp16 instruction set for Half acceleration Declare an internal-only Fp16 ISA (no public managed API) mirroring the x86 F16C/Evex internal-ISA pattern, giving the JIT an opportunistic FEAT_FP16 gate for accelerating System.Half. Regenerated the JIT/EE thunk files and bumped the interface GUID. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/inc/corinfoinstructionset.h | 79 ++++++++++-------- src/coreclr/inc/jiteeversionguid.h | 10 +-- src/coreclr/inc/readytoruninstructionset.h | 1 + .../Runtime/ReadyToRunInstructionSet.cs | 1 + .../Runtime/ReadyToRunInstructionSetHelper.cs | 2 + .../JitInterface/CorInfoInstructionSet.cs | 82 +++++++++++-------- .../ThunkGenerator/InstructionSetDesc.txt | 5 +- 7 files changed, 110 insertions(+), 70 deletions(-) diff --git a/src/coreclr/inc/corinfoinstructionset.h b/src/coreclr/inc/corinfoinstructionset.h index e4910526fdd92d..af0d9c342ec918 100644 --- a/src/coreclr/inc/corinfoinstructionset.h +++ b/src/coreclr/inc/corinfoinstructionset.h @@ -23,38 +23,40 @@ enum CORINFO_InstructionSet InstructionSet_Crc32=4, InstructionSet_Dp=5, InstructionSet_Rdm=6, - InstructionSet_Sha1=7, - InstructionSet_Sha256=8, - InstructionSet_Atomics=9, - InstructionSet_Vector64=10, - InstructionSet_Vector128=11, - InstructionSet_VectorT=12, - InstructionSet_Dczva=13, - InstructionSet_Rcpc=14, - InstructionSet_VectorT128=15, - InstructionSet_Rcpc2=16, - InstructionSet_Sve=17, - InstructionSet_Sve2=18, - InstructionSet_Sha3=19, - InstructionSet_Sm4=20, - InstructionSet_SveAes=21, - InstructionSet_SveSha3=22, - InstructionSet_SveSm4=23, - InstructionSet_ArmBase_Arm64=24, - InstructionSet_AdvSimd_Arm64=25, - InstructionSet_Aes_Arm64=26, - InstructionSet_Crc32_Arm64=27, - InstructionSet_Dp_Arm64=28, - InstructionSet_Rdm_Arm64=29, - InstructionSet_Sha1_Arm64=30, - InstructionSet_Sha256_Arm64=31, - InstructionSet_Sve_Arm64=32, - InstructionSet_Sve2_Arm64=33, - InstructionSet_Sha3_Arm64=34, - InstructionSet_Sm4_Arm64=35, - InstructionSet_SveAes_Arm64=36, - InstructionSet_SveSha3_Arm64=37, - InstructionSet_SveSm4_Arm64=38, + InstructionSet_Fp16=7, + InstructionSet_Sha1=8, + InstructionSet_Sha256=9, + InstructionSet_Atomics=10, + InstructionSet_Vector64=11, + InstructionSet_Vector128=12, + InstructionSet_VectorT=13, + InstructionSet_Dczva=14, + InstructionSet_Rcpc=15, + InstructionSet_VectorT128=16, + InstructionSet_Rcpc2=17, + InstructionSet_Sve=18, + InstructionSet_Sve2=19, + InstructionSet_Sha3=20, + InstructionSet_Sm4=21, + InstructionSet_SveAes=22, + InstructionSet_SveSha3=23, + InstructionSet_SveSm4=24, + InstructionSet_ArmBase_Arm64=25, + InstructionSet_AdvSimd_Arm64=26, + InstructionSet_Aes_Arm64=27, + InstructionSet_Crc32_Arm64=28, + InstructionSet_Dp_Arm64=29, + InstructionSet_Rdm_Arm64=30, + InstructionSet_Fp16_Arm64=31, + InstructionSet_Sha1_Arm64=32, + InstructionSet_Sha256_Arm64=33, + InstructionSet_Sve_Arm64=34, + InstructionSet_Sve2_Arm64=35, + InstructionSet_Sha3_Arm64=36, + InstructionSet_Sm4_Arm64=37, + InstructionSet_SveAes_Arm64=38, + InstructionSet_SveSha3_Arm64=39, + InstructionSet_SveSm4_Arm64=40, #endif // TARGET_ARM64 #ifdef TARGET_RISCV64 InstructionSet_RiscV64Base=1, @@ -269,6 +271,8 @@ struct CORINFO_InstructionSetFlags AddInstructionSet(InstructionSet_Dp_Arm64); if (HasInstructionSet(InstructionSet_Rdm)) AddInstructionSet(InstructionSet_Rdm_Arm64); + if (HasInstructionSet(InstructionSet_Fp16)) + AddInstructionSet(InstructionSet_Fp16_Arm64); if (HasInstructionSet(InstructionSet_Sha1)) AddInstructionSet(InstructionSet_Sha1_Arm64); if (HasInstructionSet(InstructionSet_Sha256)) @@ -369,6 +373,10 @@ inline CORINFO_InstructionSetFlags EnsureInstructionSetFlagsAreValid(CORINFO_Ins resultflags.RemoveInstructionSet(InstructionSet_Rdm); if (resultflags.HasInstructionSet(InstructionSet_Rdm_Arm64) && !resultflags.HasInstructionSet(InstructionSet_Rdm)) resultflags.RemoveInstructionSet(InstructionSet_Rdm_Arm64); + if (resultflags.HasInstructionSet(InstructionSet_Fp16) && !resultflags.HasInstructionSet(InstructionSet_Fp16_Arm64)) + resultflags.RemoveInstructionSet(InstructionSet_Fp16); + if (resultflags.HasInstructionSet(InstructionSet_Fp16_Arm64) && !resultflags.HasInstructionSet(InstructionSet_Fp16)) + resultflags.RemoveInstructionSet(InstructionSet_Fp16_Arm64); if (resultflags.HasInstructionSet(InstructionSet_Sha1) && !resultflags.HasInstructionSet(InstructionSet_Sha1_Arm64)) resultflags.RemoveInstructionSet(InstructionSet_Sha1); if (resultflags.HasInstructionSet(InstructionSet_Sha1_Arm64) && !resultflags.HasInstructionSet(InstructionSet_Sha1)) @@ -415,6 +423,8 @@ inline CORINFO_InstructionSetFlags EnsureInstructionSetFlagsAreValid(CORINFO_Ins resultflags.RemoveInstructionSet(InstructionSet_Dp); if (resultflags.HasInstructionSet(InstructionSet_Rdm) && !resultflags.HasInstructionSet(InstructionSet_AdvSimd)) resultflags.RemoveInstructionSet(InstructionSet_Rdm); + if (resultflags.HasInstructionSet(InstructionSet_Fp16) && !resultflags.HasInstructionSet(InstructionSet_AdvSimd)) + resultflags.RemoveInstructionSet(InstructionSet_Fp16); if (resultflags.HasInstructionSet(InstructionSet_Sha1) && !resultflags.HasInstructionSet(InstructionSet_ArmBase)) resultflags.RemoveInstructionSet(InstructionSet_Sha1); if (resultflags.HasInstructionSet(InstructionSet_Sha256) && !resultflags.HasInstructionSet(InstructionSet_ArmBase)) @@ -693,6 +703,10 @@ inline const char *InstructionSetToString(CORINFO_InstructionSet instructionSet) return "Rdm"; case InstructionSet_Rdm_Arm64 : return "Rdm_Arm64"; + case InstructionSet_Fp16 : + return "Fp16"; + case InstructionSet_Fp16_Arm64 : + return "Fp16_Arm64"; case InstructionSet_Sha1 : return "Sha1"; case InstructionSet_Sha1_Arm64 : @@ -945,6 +959,7 @@ inline CORINFO_InstructionSet InstructionSetFromR2RInstructionSet(ReadyToRunInst case READYTORUN_INSTRUCTION_Crc32: return InstructionSet_Crc32; case READYTORUN_INSTRUCTION_Dp: return InstructionSet_Dp; case READYTORUN_INSTRUCTION_Rdm: return InstructionSet_Rdm; + case READYTORUN_INSTRUCTION_Fp16: return InstructionSet_Fp16; case READYTORUN_INSTRUCTION_Sha1: return InstructionSet_Sha1; case READYTORUN_INSTRUCTION_Sha256: return InstructionSet_Sha256; case READYTORUN_INSTRUCTION_Atomics: return InstructionSet_Atomics; diff --git a/src/coreclr/inc/jiteeversionguid.h b/src/coreclr/inc/jiteeversionguid.h index 6e64f7d1e4a1a8..b0f9131452ba6e 100644 --- a/src/coreclr/inc/jiteeversionguid.h +++ b/src/coreclr/inc/jiteeversionguid.h @@ -37,11 +37,11 @@ #include -constexpr GUID JITEEVersionIdentifier = { /* 58dd28fc-38f0-4782-989a-25e2d721a98b */ - 0x58dd28fc, - 0x38f0, - 0x4782, - {0x98, 0x9a, 0x25, 0xe2, 0xd7, 0x21, 0xa9, 0x8b} +constexpr GUID JITEEVersionIdentifier = { /* cb8178b1-ea25-490e-80fd-a320f9e1f5b8 */ + 0xcb8178b1, + 0xea25, + 0x490e, + {0x80, 0xfd, 0xa3, 0x20, 0xf9, 0xe1, 0xf5, 0xb8} }; #endif // JIT_EE_VERSIONING_GUID_H diff --git a/src/coreclr/inc/readytoruninstructionset.h b/src/coreclr/inc/readytoruninstructionset.h index 6e491f85381089..99f61a1035b9c1 100644 --- a/src/coreclr/inc/readytoruninstructionset.h +++ b/src/coreclr/inc/readytoruninstructionset.h @@ -100,6 +100,7 @@ enum ReadyToRunInstructionSet READYTORUN_INSTRUCTION_SveSm4=90, READYTORUN_INSTRUCTION_WasmBase=91, READYTORUN_INSTRUCTION_PackedSimd=92, + READYTORUN_INSTRUCTION_Fp16=93, }; diff --git a/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSet.cs b/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSet.cs index 14e8c9585a0495..c0c0b53008603d 100644 --- a/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSet.cs +++ b/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSet.cs @@ -103,5 +103,6 @@ public enum ReadyToRunInstructionSet SveSm4 = 90, WasmBase = 91, PackedSimd = 92, + Fp16 = 93, } } diff --git a/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSetHelper.cs b/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSetHelper.cs index 196200fc55434f..c3a74601719dac 100644 --- a/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSetHelper.cs +++ b/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSetHelper.cs @@ -34,6 +34,8 @@ public static class ReadyToRunInstructionSetHelper case InstructionSet.ARM64_Dp_Arm64: return ReadyToRunInstructionSet.Dp; case InstructionSet.ARM64_Rdm: return ReadyToRunInstructionSet.Rdm; case InstructionSet.ARM64_Rdm_Arm64: return ReadyToRunInstructionSet.Rdm; + case InstructionSet.ARM64_Fp16: return ReadyToRunInstructionSet.Fp16; + case InstructionSet.ARM64_Fp16_Arm64: return ReadyToRunInstructionSet.Fp16; case InstructionSet.ARM64_Sha1: return ReadyToRunInstructionSet.Sha1; case InstructionSet.ARM64_Sha1_Arm64: return ReadyToRunInstructionSet.Sha1; case InstructionSet.ARM64_Sha256: return ReadyToRunInstructionSet.Sha256; diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs b/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs index 264d2cd0bdbcd6..c83e4ae21ea413 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs @@ -23,6 +23,7 @@ public enum InstructionSet ARM64_Crc32 = InstructionSet_ARM64.Crc32, ARM64_Dp = InstructionSet_ARM64.Dp, ARM64_Rdm = InstructionSet_ARM64.Rdm, + ARM64_Fp16 = InstructionSet_ARM64.Fp16, ARM64_Sha1 = InstructionSet_ARM64.Sha1, ARM64_Sha256 = InstructionSet_ARM64.Sha256, ARM64_Atomics = InstructionSet_ARM64.Atomics, @@ -46,6 +47,7 @@ public enum InstructionSet ARM64_Crc32_Arm64 = InstructionSet_ARM64.Crc32_Arm64, ARM64_Dp_Arm64 = InstructionSet_ARM64.Dp_Arm64, ARM64_Rdm_Arm64 = InstructionSet_ARM64.Rdm_Arm64, + ARM64_Fp16_Arm64 = InstructionSet_ARM64.Fp16_Arm64, ARM64_Sha1_Arm64 = InstructionSet_ARM64.Sha1_Arm64, ARM64_Sha256_Arm64 = InstructionSet_ARM64.Sha256_Arm64, ARM64_Sve_Arm64 = InstructionSet_ARM64.Sve_Arm64, @@ -165,38 +167,40 @@ public enum InstructionSet_ARM64 Crc32 = 4, Dp = 5, Rdm = 6, - Sha1 = 7, - Sha256 = 8, - Atomics = 9, - Vector64 = 10, - Vector128 = 11, - VectorT = 12, - Dczva = 13, - Rcpc = 14, - VectorT128 = 15, - Rcpc2 = 16, - Sve = 17, - Sve2 = 18, - Sha3 = 19, - Sm4 = 20, - SveAes = 21, - SveSha3 = 22, - SveSm4 = 23, - ArmBase_Arm64 = 24, - AdvSimd_Arm64 = 25, - Aes_Arm64 = 26, - Crc32_Arm64 = 27, - Dp_Arm64 = 28, - Rdm_Arm64 = 29, - Sha1_Arm64 = 30, - Sha256_Arm64 = 31, - Sve_Arm64 = 32, - Sve2_Arm64 = 33, - Sha3_Arm64 = 34, - Sm4_Arm64 = 35, - SveAes_Arm64 = 36, - SveSha3_Arm64 = 37, - SveSm4_Arm64 = 38, + Fp16 = 7, + Sha1 = 8, + Sha256 = 9, + Atomics = 10, + Vector64 = 11, + Vector128 = 12, + VectorT = 13, + Dczva = 14, + Rcpc = 15, + VectorT128 = 16, + Rcpc2 = 17, + Sve = 18, + Sve2 = 19, + Sha3 = 20, + Sm4 = 21, + SveAes = 22, + SveSha3 = 23, + SveSm4 = 24, + ArmBase_Arm64 = 25, + AdvSimd_Arm64 = 26, + Aes_Arm64 = 27, + Crc32_Arm64 = 28, + Dp_Arm64 = 29, + Rdm_Arm64 = 30, + Fp16_Arm64 = 31, + Sha1_Arm64 = 32, + Sha256_Arm64 = 33, + Sve_Arm64 = 34, + Sve2_Arm64 = 35, + Sha3_Arm64 = 36, + Sm4_Arm64 = 37, + SveAes_Arm64 = 38, + SveSha3_Arm64 = 39, + SveSm4_Arm64 = 40, } public enum InstructionSet_RiscV64 @@ -512,6 +516,10 @@ public static InstructionSetFlags ExpandInstructionSetByImplicationHelper(Target resultflags.AddInstructionSet(InstructionSet.ARM64_Rdm_Arm64); if (resultflags.HasInstructionSet(InstructionSet.ARM64_Rdm_Arm64)) resultflags.AddInstructionSet(InstructionSet.ARM64_Rdm); + if (resultflags.HasInstructionSet(InstructionSet.ARM64_Fp16)) + resultflags.AddInstructionSet(InstructionSet.ARM64_Fp16_Arm64); + if (resultflags.HasInstructionSet(InstructionSet.ARM64_Fp16_Arm64)) + resultflags.AddInstructionSet(InstructionSet.ARM64_Fp16); if (resultflags.HasInstructionSet(InstructionSet.ARM64_Sha1)) resultflags.AddInstructionSet(InstructionSet.ARM64_Sha1_Arm64); if (resultflags.HasInstructionSet(InstructionSet.ARM64_Sha1_Arm64)) @@ -558,6 +566,8 @@ public static InstructionSetFlags ExpandInstructionSetByImplicationHelper(Target resultflags.AddInstructionSet(InstructionSet.ARM64_AdvSimd); if (resultflags.HasInstructionSet(InstructionSet.ARM64_Rdm)) resultflags.AddInstructionSet(InstructionSet.ARM64_AdvSimd); + if (resultflags.HasInstructionSet(InstructionSet.ARM64_Fp16)) + resultflags.AddInstructionSet(InstructionSet.ARM64_AdvSimd); if (resultflags.HasInstructionSet(InstructionSet.ARM64_Sha1)) resultflags.AddInstructionSet(InstructionSet.ARM64_ArmBase); if (resultflags.HasInstructionSet(InstructionSet.ARM64_Sha256)) @@ -835,6 +845,8 @@ private static InstructionSetFlags ExpandInstructionSetByReverseImplicationHelpe resultflags.AddInstructionSet(InstructionSet.ARM64_Dp); if (resultflags.HasInstructionSet(InstructionSet.ARM64_Rdm_Arm64)) resultflags.AddInstructionSet(InstructionSet.ARM64_Rdm); + if (resultflags.HasInstructionSet(InstructionSet.ARM64_Fp16_Arm64)) + resultflags.AddInstructionSet(InstructionSet.ARM64_Fp16); if (resultflags.HasInstructionSet(InstructionSet.ARM64_Sha1_Arm64)) resultflags.AddInstructionSet(InstructionSet.ARM64_Sha1); if (resultflags.HasInstructionSet(InstructionSet.ARM64_Sha256_Arm64)) @@ -863,6 +875,8 @@ private static InstructionSetFlags ExpandInstructionSetByReverseImplicationHelpe resultflags.AddInstructionSet(InstructionSet.ARM64_Dp); if (resultflags.HasInstructionSet(InstructionSet.ARM64_AdvSimd)) resultflags.AddInstructionSet(InstructionSet.ARM64_Rdm); + if (resultflags.HasInstructionSet(InstructionSet.ARM64_AdvSimd)) + resultflags.AddInstructionSet(InstructionSet.ARM64_Fp16); if (resultflags.HasInstructionSet(InstructionSet.ARM64_ArmBase)) resultflags.AddInstructionSet(InstructionSet.ARM64_Sha1); if (resultflags.HasInstructionSet(InstructionSet.ARM64_ArmBase)) @@ -1132,6 +1146,7 @@ public static IEnumerable ArchitectureToValidInstructionSets yield return new InstructionSetInfo("crc", "Crc32", InstructionSet.ARM64_Crc32, true); yield return new InstructionSetInfo("dotprod", "Dp", InstructionSet.ARM64_Dp, true); yield return new InstructionSetInfo("rdma", "Rdm", InstructionSet.ARM64_Rdm, true); + yield return new InstructionSetInfo("fp16", "", InstructionSet.ARM64_Fp16, true); yield return new InstructionSetInfo("sha1", "Sha1", InstructionSet.ARM64_Sha1, true); yield return new InstructionSetInfo("sha2", "Sha256", InstructionSet.ARM64_Sha256, true); yield return new InstructionSetInfo("lse", "", InstructionSet.ARM64_Atomics, true); @@ -1332,6 +1347,8 @@ public void Set64BitInstructionSetVariants(TargetArchitecture architecture) AddInstructionSet(InstructionSet.ARM64_Dp_Arm64); if (HasInstructionSet(InstructionSet.ARM64_Rdm)) AddInstructionSet(InstructionSet.ARM64_Rdm_Arm64); + if (HasInstructionSet(InstructionSet.ARM64_Fp16)) + AddInstructionSet(InstructionSet.ARM64_Fp16_Arm64); if (HasInstructionSet(InstructionSet.ARM64_Sha1)) AddInstructionSet(InstructionSet.ARM64_Sha1_Arm64); if (HasInstructionSet(InstructionSet.ARM64_Sha256)) @@ -1410,6 +1427,7 @@ public void Set64BitInstructionSetVariantsUnconditionally(TargetArchitecture arc AddInstructionSet(InstructionSet.ARM64_Crc32_Arm64); AddInstructionSet(InstructionSet.ARM64_Dp_Arm64); AddInstructionSet(InstructionSet.ARM64_Rdm_Arm64); + AddInstructionSet(InstructionSet.ARM64_Fp16_Arm64); AddInstructionSet(InstructionSet.ARM64_Sha1_Arm64); AddInstructionSet(InstructionSet.ARM64_Sha256_Arm64); AddInstructionSet(InstructionSet.ARM64_Sve_Arm64); diff --git a/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt b/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt index b961d2accdb9e3..411786b5186ac2 100644 --- a/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt +++ b/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt @@ -26,7 +26,7 @@ ; DO NOT CHANGE R2R NUMERIC VALUES OF THE EXISTING SETS. Changing R2R numeric values definitions would be R2R format breaking change. ; The ISA definitions should also be mapped to `hwintrinsicIsaRangeArray` in hwintrinsic.cpp. -; NEXT_AVAILABLE_R2R_BIT = 93 +; NEXT_AVAILABLE_R2R_BIT = 94 ; Definition of X86 instruction sets definearch ,X86 ,32Bit ,X64, X64, X86 @@ -215,6 +215,7 @@ instructionset ,ARM64 ,Aes , ,9 ,Aes instructionset ,ARM64 ,Crc32 , ,18 ,Crc32 ,crc instructionset ,ARM64 ,Dp , ,23 ,Dp ,dotprod instructionset ,ARM64 ,Rdm , ,24 ,Rdm ,rdma +instructionset ,ARM64 , ,Fp16 ,93 ,Fp16 ,fp16 instructionset ,ARM64 ,Sha1 , ,19 ,Sha1 ,sha1 instructionset ,ARM64 ,Sha256 , ,20 ,Sha256 ,sha2 instructionset ,ARM64 , ,Atomics ,21 ,Atomics ,lse @@ -239,6 +240,7 @@ instructionset64bit,ARM64 ,Aes instructionset64bit,ARM64 ,Crc32 instructionset64bit,ARM64 ,Dp instructionset64bit,ARM64 ,Rdm +instructionset64bit,ARM64 ,Fp16 instructionset64bit,ARM64 ,Sha1 instructionset64bit,ARM64 ,Sha256 instructionset64bit,ARM64 ,Sve @@ -258,6 +260,7 @@ implication ,ARM64 ,Aes ,ArmBase implication ,ARM64 ,Crc32 ,ArmBase implication ,ARM64 ,Dp ,AdvSimd implication ,ARM64 ,Rdm ,AdvSimd +implication ,ARM64 ,Fp16 ,AdvSimd implication ,ARM64 ,Sha1 ,ArmBase implication ,ARM64 ,Sha256 ,ArmBase implication ,ARM64 ,Vector64 ,AdvSimd From 9336392271a4f2958214f3485d998268ec7170e6 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Fri, 10 Jul 2026 08:44:17 -0700 Subject: [PATCH 15/22] Detect Arm64 FEAT_FP16 and expose the internal Fp16 ISA to the JIT Add ARM64IntrinsicConstants_Fp16 with per-OS detection (Linux HWCAP_FPHP+ASIMDHP, macOS sysctl hw.optional.arm.FEAT_FP16; Windows has no IsProcessorFeaturePresent flag yet, left as a TODO), an EnableArm64Fp16 config knob, and wire it through the VM (codeman), the JIT default flags, and the shared ILC/crossgen2 mapping. Insert the Fp16 placeholder rows in hwintrinsicIsaRangeArray to keep the positional array aligned with the shifted InstructionSet enum. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/inc/clrconfigvalues.h | 1 + src/coreclr/jit/compiler.cpp | 5 +++++ src/coreclr/jit/hwintrinsic.cpp | 2 ++ src/coreclr/jit/jitconfigvalues.h | 1 + .../Common/Compiler/HardwareIntrinsicHelpers.cs | 5 +++++ src/coreclr/vm/codeman.cpp | 5 +++++ src/native/minipal/cpufeatures.c | 17 +++++++++++++++++ src/native/minipal/cpufeatures.h | 1 + 8 files changed, 37 insertions(+) diff --git a/src/coreclr/inc/clrconfigvalues.h b/src/coreclr/inc/clrconfigvalues.h index e923170681b432..2954839a3ef851 100644 --- a/src/coreclr/inc/clrconfigvalues.h +++ b/src/coreclr/inc/clrconfigvalues.h @@ -700,6 +700,7 @@ RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Atomics, W("EnableArm64At RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Crc32, W("EnableArm64Crc32"), 1, "Allows Arm64 Crc32+ hardware intrinsics to be disabled") RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Dczva, W("EnableArm64Dczva"), 1, "Allows Arm64 Dczva+ hardware intrinsics to be disabled") RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Dp, W("EnableArm64Dp"), 1, "Allows Arm64 Dp+ hardware intrinsics to be disabled") +RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Fp16, W("EnableArm64Fp16"), 1, "Allows Arm64 Fp16+ hardware intrinsics to be disabled") RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Rdm, W("EnableArm64Rdm"), 1, "Allows Arm64 Rdm+ hardware intrinsics to be disabled") RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Sha1, W("EnableArm64Sha1"), 1, "Allows Arm64 Sha1+ hardware intrinsics to be disabled") RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Sha256, W("EnableArm64Sha256"), 1, "Allows Arm64 Sha256+ hardware intrinsics to be disabled") diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index 105ed99fe35427..c11a22e6b66422 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -6080,6 +6080,11 @@ int Compiler::compCompileAfterInit(CORINFO_MODULE_HANDLE classPtr, instructionSetFlags.AddInstructionSet(InstructionSet_Rdm); } + if (JitConfig.EnableArm64Fp16() != 0) + { + instructionSetFlags.AddInstructionSet(InstructionSet_Fp16); + } + if (JitConfig.EnableArm64Sha1() != 0) { instructionSetFlags.AddInstructionSet(InstructionSet_Sha1); diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index 26288d849e2a88..cb312268e4bce4 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -980,6 +980,7 @@ static const HWIntrinsicIsaRange hwintrinsicIsaRangeArray[] = { { FIRST_NI_Crc32, LAST_NI_Crc32 }, // Crc32 { FIRST_NI_Dp, LAST_NI_Dp }, // Dp { FIRST_NI_Rdm, LAST_NI_Rdm }, // Rdm + { NI_Illegal, NI_Illegal }, // Fp16 { FIRST_NI_Sha1, LAST_NI_Sha1 }, // Sha1 { FIRST_NI_Sha256, LAST_NI_Sha256 }, // Sha256 { NI_Illegal, NI_Illegal }, // Atomics @@ -1003,6 +1004,7 @@ static const HWIntrinsicIsaRange hwintrinsicIsaRangeArray[] = { { FIRST_NI_Crc32_Arm64, LAST_NI_Crc32_Arm64 }, // Crc32_Arm64 { NI_Illegal, NI_Illegal }, // Dp_Arm64 { FIRST_NI_Rdm_Arm64, LAST_NI_Rdm_Arm64 }, // Rdm_Arm64 + { NI_Illegal, NI_Illegal }, // Fp16_Arm64 { NI_Illegal, NI_Illegal }, // Sha1_Arm64 { NI_Illegal, NI_Illegal }, // Sha256_Arm64 { NI_Illegal, NI_Illegal }, // Sve_Arm64 diff --git a/src/coreclr/jit/jitconfigvalues.h b/src/coreclr/jit/jitconfigvalues.h index bedf9e217c71c5..80e9c0647c8273 100644 --- a/src/coreclr/jit/jitconfigvalues.h +++ b/src/coreclr/jit/jitconfigvalues.h @@ -441,6 +441,7 @@ RELEASE_CONFIG_INTEGER(EnableArm64Atomics, "EnableArm64Atomics", RELEASE_CONFIG_INTEGER(EnableArm64Crc32, "EnableArm64Crc32", 1) // Allows Arm64 Crc32+ hardware intrinsics to be disabled RELEASE_CONFIG_INTEGER(EnableArm64Dczva, "EnableArm64Dczva", 1) // Allows Arm64 Dczva+ hardware intrinsics to be disabled RELEASE_CONFIG_INTEGER(EnableArm64Dp, "EnableArm64Dp", 1) // Allows Arm64 Dp+ hardware intrinsics to be disabled +RELEASE_CONFIG_INTEGER(EnableArm64Fp16, "EnableArm64Fp16", 1) // Allows Arm64 Fp16+ hardware intrinsics to be disabled RELEASE_CONFIG_INTEGER(EnableArm64Rdm, "EnableArm64Rdm", 1) // Allows Arm64 Rdm+ hardware intrinsics to be disabled RELEASE_CONFIG_INTEGER(EnableArm64Sha1, "EnableArm64Sha1", 1) // Allows Arm64 Sha1+ hardware intrinsics to be disabled RELEASE_CONFIG_INTEGER(EnableArm64Sha256, "EnableArm64Sha256", 1) // Allows Arm64 Sha256+ hardware intrinsics to be disabled diff --git a/src/coreclr/tools/Common/Compiler/HardwareIntrinsicHelpers.cs b/src/coreclr/tools/Common/Compiler/HardwareIntrinsicHelpers.cs index e40bc64c73a289..2044f35875349f 100644 --- a/src/coreclr/tools/Common/Compiler/HardwareIntrinsicHelpers.cs +++ b/src/coreclr/tools/Common/Compiler/HardwareIntrinsicHelpers.cs @@ -232,6 +232,7 @@ private static class Arm64IntrinsicConstants public const int SveAes = (1 << 13); public const int SveSha3 = (1 << 14); public const int SveSm4 = (1 << 15); + public const int Fp16 = (1 << 16); public static void AddToBuilder(InstructionSetSupportBuilder builder, int flags) { @@ -267,6 +268,8 @@ public static void AddToBuilder(InstructionSetSupportBuilder builder, int flags) builder.AddSupportedInstructionSet("sve_sha3"); if ((flags & SveSm4) != 0) builder.AddSupportedInstructionSet("sve_sm4"); + if ((flags & Fp16) != 0) + builder.AddSupportedInstructionSet("fp16"); } public static int FromInstructionSet(InstructionSet instructionSet) @@ -289,6 +292,8 @@ public static int FromInstructionSet(InstructionSet instructionSet) InstructionSet.ARM64_Dp_Arm64 => Dp, InstructionSet.ARM64_Rdm => Rdm, InstructionSet.ARM64_Rdm_Arm64 => Rdm, + InstructionSet.ARM64_Fp16 => Fp16, + InstructionSet.ARM64_Fp16_Arm64 => Fp16, InstructionSet.ARM64_Sha1 => Sha1, InstructionSet.ARM64_Sha1_Arm64 => Sha1, InstructionSet.ARM64_Sha256 => Sha256, diff --git a/src/coreclr/vm/codeman.cpp b/src/coreclr/vm/codeman.cpp index d3a640139b1a2e..da8442feecd8db 100644 --- a/src/coreclr/vm/codeman.cpp +++ b/src/coreclr/vm/codeman.cpp @@ -1702,6 +1702,11 @@ void EEJitManager::SetCpuInfo() CPUCompileFlags.Set(InstructionSet_Rdm); } + if (((cpuFeatures & ARM64IntrinsicConstants_Fp16) != 0) && CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_EnableArm64Fp16)) + { + CPUCompileFlags.Set(InstructionSet_Fp16); + } + if (((cpuFeatures & ARM64IntrinsicConstants_Sha1) != 0) && CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_EnableArm64Sha1)) { CPUCompileFlags.Set(InstructionSet_Sha1); diff --git a/src/native/minipal/cpufeatures.c b/src/native/minipal/cpufeatures.c index 9456932b62d2b2..b7e9dcc8125ecc 100644 --- a/src/native/minipal/cpufeatures.c +++ b/src/native/minipal/cpufeatures.c @@ -52,6 +52,14 @@ #ifndef HWCAP_ASIMDRDM #define HWCAP_ASIMDRDM (1 << 12) #endif + +#ifndef HWCAP_FPHP +#define HWCAP_FPHP (1 << 9) +#endif + +#ifndef HWCAP_ASIMDHP +#define HWCAP_ASIMDHP (1 << 10) +#endif #ifndef HWCAP_LRCPC #define HWCAP_LRCPC (1 << 15) #endif @@ -549,6 +557,10 @@ int minipal_getcpufeatures(void) if (hwCap & HWCAP_ASIMDRDM) result |= ARM64IntrinsicConstants_Rdm; + // FEAT_FP16 provides both scalar (FPHP) and Advanced SIMD (ASIMDHP) half-precision arithmetic. + if ((hwCap & HWCAP_FPHP) && (hwCap & HWCAP_ASIMDHP)) + result |= ARM64IntrinsicConstants_Fp16; + if (hwCap & HWCAP_SVE) result |= ARM64IntrinsicConstants_Sve; @@ -604,6 +616,9 @@ int minipal_getcpufeatures(void) if ((sysctlbyname("hw.optional.arm.FEAT_RDM", &valueFromSysctl, &sz, NULL, 0) == 0) && (valueFromSysctl != 0)) result |= ARM64IntrinsicConstants_Rdm; + if ((sysctlbyname("hw.optional.arm.FEAT_FP16", &valueFromSysctl, &sz, NULL, 0) == 0) && (valueFromSysctl != 0)) + result |= ARM64IntrinsicConstants_Fp16; + if ((sysctlbyname("hw.optional.arm.FEAT_SHA1", &valueFromSysctl, &sz, NULL, 0) == 0) && (valueFromSysctl != 0)) result |= ARM64IntrinsicConstants_Sha1; @@ -687,6 +702,8 @@ int minipal_getcpufeatures(void) // TODO: IsProcessorFeaturePresent doesn't support LRCPC2 yet. + // TODO: IsProcessorFeaturePresent doesn't support FEAT_FP16 (half-precision) yet. + if (IsProcessorFeaturePresent(PF_ARM_SVE_INSTRUCTIONS_AVAILABLE)) { result |= ARM64IntrinsicConstants_Sve; diff --git a/src/native/minipal/cpufeatures.h b/src/native/minipal/cpufeatures.h index 53d9308f921247..77b9aa17fed992 100644 --- a/src/native/minipal/cpufeatures.h +++ b/src/native/minipal/cpufeatures.h @@ -50,6 +50,7 @@ #define ARM64IntrinsicConstants_SveAes (1 << 13) #define ARM64IntrinsicConstants_SveSha3 (1 << 14) #define ARM64IntrinsicConstants_SveSm4 (1 << 15) +#define ARM64IntrinsicConstants_Fp16 (1 << 16) #include From 739d2f30db069aed26a9e9f2202a53a85f0baed0 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Fri, 10 Jul 2026 09:48:20 -0700 Subject: [PATCH 16/22] Add Arm64 emitter support for scalar half-precision floating-point Teach the Arm64 emitter to encode scalar half-precision (FEAT_FP16) FP instructions used to accelerate System.Half: ftype=0b11 (EA_2BYTE) for DV_3D/DV_2G/DV_2K forms, plus four new INS_OPTS values for the half<->int conversions (DV_2H fcvtzs/fcvtzu and DV_2I scvtf/ucvtf). Relax the relevant sanity, size, and perfscore paths to accept EA_2BYTE, and add a genArm64EmitterUnitTestsFp16 section covering the new forms. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/codegen.h | 1 + src/coreclr/jit/codegenarm64test.cpp | 62 ++++++++++++++++++++++++++++ src/coreclr/jit/codegenlinear.cpp | 4 ++ src/coreclr/jit/emitarm64.cpp | 54 +++++++++++++++++++----- src/coreclr/jit/emitarm64.h | 6 +-- src/coreclr/jit/instr.h | 6 +++ 6 files changed, 119 insertions(+), 14 deletions(-) diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index 1fc303b4162b3d..07518288ced19e 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -629,6 +629,7 @@ class CodeGen final : public CodeGenInterface #if defined(TARGET_ARM64) void genArm64EmitterUnitTestsGeneral(); void genArm64EmitterUnitTestsAdvSimd(); + void genArm64EmitterUnitTestsFp16(); void genArm64EmitterUnitTestsSve(); void genArm64EmitterUnitTestsPac(); #endif diff --git a/src/coreclr/jit/codegenarm64test.cpp b/src/coreclr/jit/codegenarm64test.cpp index 0bf47c1aa50a0a..c4ff7f7566b716 100644 --- a/src/coreclr/jit/codegenarm64test.cpp +++ b/src/coreclr/jit/codegenarm64test.cpp @@ -9173,4 +9173,66 @@ void CodeGen::genArm64EmitterUnitTestsPac() theEmitter->emitIns_R_R(INS_pacia, EA_8BYTE, REG_R27, REG_SP); // PACIA , theEmitter->emitIns_R_R(INS_pacib, EA_8BYTE, REG_R28, REG_SP); // PACIB , } + +//----------------------------------------------------------------------------- +// genArm64EmitterUnitTestsFp16: Emit the scalar half-precision (FEAT_FP16) +// floating-point instructions used to accelerate `System.Half`, so their +// encodings and disassembly can be verified. +// +void CodeGen::genArm64EmitterUnitTestsFp16() +{ + emitter* theEmitter = GetEmitter(); + + genDefineTempLabel(genCreateTempLabel()); + + // IF_DV_3D: scalar half-precision, two source registers + theEmitter->emitIns_R_R_R(INS_fadd, EA_2BYTE, REG_V0, REG_V1, REG_V2); // FADD Hd, Hn, Hm + theEmitter->emitIns_R_R_R(INS_fsub, EA_2BYTE, REG_V3, REG_V4, REG_V5); // FSUB Hd, Hn, Hm + theEmitter->emitIns_R_R_R(INS_fmul, EA_2BYTE, REG_V6, REG_V7, REG_V8); // FMUL Hd, Hn, Hm + theEmitter->emitIns_R_R_R(INS_fdiv, EA_2BYTE, REG_V9, REG_V10, REG_V11); // FDIV Hd, Hn, Hm + theEmitter->emitIns_R_R_R(INS_fnmul, EA_2BYTE, REG_V12, REG_V13, REG_V14); // FNMUL Hd, Hn, Hm + theEmitter->emitIns_R_R_R(INS_fmax, EA_2BYTE, REG_V15, REG_V16, REG_V17); // FMAX Hd, Hn, Hm + theEmitter->emitIns_R_R_R(INS_fmin, EA_2BYTE, REG_V18, REG_V19, REG_V20); // FMIN Hd, Hn, Hm + theEmitter->emitIns_R_R_R(INS_fmaxnm, EA_2BYTE, REG_V21, REG_V22, REG_V23); // FMAXNM Hd, Hn, Hm + theEmitter->emitIns_R_R_R(INS_fminnm, EA_2BYTE, REG_V24, REG_V25, REG_V26); // FMINNM Hd, Hn, Hm + theEmitter->emitIns_R_R_R(INS_fabd, EA_2BYTE, REG_V27, REG_V28, REG_V29); // FABD Hd, Hn, Hm + + // IF_DV_2G: scalar half-precision, one source register + theEmitter->emitIns_R_R(INS_fsqrt, EA_2BYTE, REG_V0, REG_V1); // FSQRT Hd, Hn + theEmitter->emitIns_R_R(INS_frinta, EA_2BYTE, REG_V2, REG_V3); // FRINTA Hd, Hn + theEmitter->emitIns_R_R(INS_frinti, EA_2BYTE, REG_V4, REG_V5); // FRINTI Hd, Hn + theEmitter->emitIns_R_R(INS_frintm, EA_2BYTE, REG_V6, REG_V7); // FRINTM Hd, Hn + theEmitter->emitIns_R_R(INS_frintn, EA_2BYTE, REG_V8, REG_V9); // FRINTN Hd, Hn + theEmitter->emitIns_R_R(INS_frintp, EA_2BYTE, REG_V10, REG_V11); // FRINTP Hd, Hn + theEmitter->emitIns_R_R(INS_frintx, EA_2BYTE, REG_V12, REG_V13); // FRINTX Hd, Hn + theEmitter->emitIns_R_R(INS_frintz, EA_2BYTE, REG_V14, REG_V15); // FRINTZ Hd, Hn + theEmitter->emitIns_R_R(INS_frecpe, EA_2BYTE, REG_V16, REG_V17); // FRECPE Hd, Hn + theEmitter->emitIns_R_R(INS_frsqrte, EA_2BYTE, REG_V18, REG_V19); // FRSQRTE Hd, Hn + + // IF_DV_2K: scalar half-precision compare + theEmitter->emitIns_R_R(INS_fcmp, EA_2BYTE, REG_V0, REG_V1); // FCMP Hn, Hm + theEmitter->emitIns_R_R(INS_fcmpe, EA_2BYTE, REG_V2, REG_V3); // FCMPE Hn, Hm + + // IF_DV_4A: scalar half-precision fused multiply-add + theEmitter->emitIns_R_R_R_R(INS_fmadd, EA_2BYTE, REG_V0, REG_V1, REG_V2, REG_V3); // FMADD Hd, Hn, Hm, Ha + theEmitter->emitIns_R_R_R_R(INS_fnmadd, EA_2BYTE, REG_V4, REG_V5, REG_V6, REG_V7); // FNMADD Hd, Hn, Hm, Ha + + // IF_DV_2J: convert between half and single/double + theEmitter->emitIns_R_R(INS_fcvt, EA_4BYTE, REG_V0, REG_V1, INS_OPTS_H_TO_S); // FCVT Sd, Hn + theEmitter->emitIns_R_R(INS_fcvt, EA_8BYTE, REG_V2, REG_V3, INS_OPTS_H_TO_D); // FCVT Dd, Hn + theEmitter->emitIns_R_R(INS_fcvt, EA_2BYTE, REG_V4, REG_V5, INS_OPTS_S_TO_H); // FCVT Hd, Sn + theEmitter->emitIns_R_R(INS_fcvt, EA_2BYTE, REG_V6, REG_V7, INS_OPTS_D_TO_H); // FCVT Hd, Dn + + // IF_DV_2H: convert half to integer (truncating toward zero) + theEmitter->emitIns_R_R(INS_fcvtzs, EA_4BYTE, REG_R0, REG_V1, INS_OPTS_H_TO_4BYTE); // FCVTZS Wd, Hn + theEmitter->emitIns_R_R(INS_fcvtzs, EA_8BYTE, REG_R2, REG_V3, INS_OPTS_H_TO_8BYTE); // FCVTZS Xd, Hn + theEmitter->emitIns_R_R(INS_fcvtzu, EA_4BYTE, REG_R4, REG_V5, INS_OPTS_H_TO_4BYTE); // FCVTZU Wd, Hn + theEmitter->emitIns_R_R(INS_fcvtzu, EA_8BYTE, REG_R6, REG_V7, INS_OPTS_H_TO_8BYTE); // FCVTZU Xd, Hn + + // IF_DV_2I: convert integer to half + theEmitter->emitIns_R_R(INS_scvtf, EA_2BYTE, REG_V0, REG_R1, INS_OPTS_4BYTE_TO_H); // SCVTF Hd, Wn + theEmitter->emitIns_R_R(INS_scvtf, EA_2BYTE, REG_V2, REG_R3, INS_OPTS_8BYTE_TO_H); // SCVTF Hd, Xn + theEmitter->emitIns_R_R(INS_ucvtf, EA_2BYTE, REG_V4, REG_R5, INS_OPTS_4BYTE_TO_H); // UCVTF Hd, Wn + theEmitter->emitIns_R_R(INS_ucvtf, EA_2BYTE, REG_V6, REG_R7, INS_OPTS_8BYTE_TO_H); // UCVTF Hd, Xn +} #endif // defined(TARGET_ARM64) && defined(DEBUG) diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index 49f67585225024..0f7d01d9b8d34d 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -2788,6 +2788,10 @@ void CodeGen::genEmitterUnitTests() { genArm64EmitterUnitTestsAdvSimd(); } + if (unitTestSectionAll || (strstr(unitTestSection, "fp16") != nullptr)) + { + genArm64EmitterUnitTestsFp16(); + } if (unitTestSectionAll || (strstr(unitTestSection, "sve") != nullptr)) { genArm64EmitterUnitTestsSve(); diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 49a836ed881ae5..eb3730e46cc5c0 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -725,7 +725,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) case IF_DV_2G: // DV_2G .........X...... ......nnnnnddddd Vd Vn (fmov, fcvtXX - register) case IF_DV_2K: // DV_2K .........X.mmmmm ......nnnnn..... Vn Vm (fcmp) assert(insOptsNone(id->idInsOpt())); - assert(isValidVectorElemsizeFloat(id->idOpSize())); + assert(isValidVectorElemsizeFloat(id->idOpSize()) || (id->idOpSize() == EA_2BYTE)); assert(isVectorRegister(id->idReg1())); assert(isVectorRegister(id->idReg2())); break; @@ -735,7 +735,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) dstsize = optGetDstsize(id->idInsOpt()); srcsize = optGetSrcsize(id->idInsOpt()); assert(isValidGeneralDatasize(dstsize)); - assert(isValidVectorElemsizeFloat(srcsize)); + assert(isValidVectorElemsizeFloat(srcsize) || (srcsize == EA_2BYTE)); assert(dstsize == id->idOpSize()); assert(isGeneralRegister(id->idReg1())); assert(isVectorRegister(id->idReg2())); @@ -746,7 +746,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) dstsize = optGetDstsize(id->idInsOpt()); srcsize = optGetSrcsize(id->idInsOpt()); assert(isValidGeneralDatasize(srcsize)); - assert(isValidVectorElemsizeFloat(dstsize)); + assert(isValidVectorElemsizeFloat(dstsize) || (dstsize == EA_2BYTE)); assert(dstsize == id->idOpSize()); assert(isVectorRegister(id->idReg1())); assert(isGeneralRegister(id->idReg2())); @@ -958,7 +958,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) break; case IF_DV_4A: // DV_4A .........X.mmmmm .aaaaannnnnddddd Rd Rn Rm Ra (scalar) - assert(isValidGeneralDatasize(id->idOpSize())); + assert(isValidScalarDatasize(id->idOpSize())); assert(isVectorRegister(id->idReg1())); assert(isVectorRegister(id->idReg2())); assert(isVectorRegister(id->idReg3())); @@ -3613,6 +3613,7 @@ emitter::code_t emitter::emitInsCode(instruction ins, insFormat fmt) case INS_OPTS_8BYTE_TO_D: case INS_OPTS_S_TO_D: case INS_OPTS_H_TO_D: + case INS_OPTS_H_TO_8BYTE: return EA_8BYTE; @@ -3622,11 +3623,14 @@ emitter::code_t emitter::emitInsCode(instruction ins, insFormat fmt) case INS_OPTS_8BYTE_TO_S: case INS_OPTS_D_TO_S: case INS_OPTS_H_TO_S: + case INS_OPTS_H_TO_4BYTE: return EA_4BYTE; case INS_OPTS_S_TO_H: case INS_OPTS_D_TO_H: + case INS_OPTS_4BYTE_TO_H: + case INS_OPTS_8BYTE_TO_H: return EA_2BYTE; @@ -3647,6 +3651,7 @@ emitter::code_t emitter::emitInsCode(instruction ins, insFormat fmt) case INS_OPTS_8BYTE_TO_S: case INS_OPTS_D_TO_S: case INS_OPTS_D_TO_H: + case INS_OPTS_8BYTE_TO_H: return EA_8BYTE; @@ -3656,11 +3661,14 @@ emitter::code_t emitter::emitInsCode(instruction ins, insFormat fmt) case INS_OPTS_4BYTE_TO_D: case INS_OPTS_S_TO_D: case INS_OPTS_S_TO_H: + case INS_OPTS_4BYTE_TO_H: return EA_4BYTE; case INS_OPTS_H_TO_S: case INS_OPTS_H_TO_D: + case INS_OPTS_H_TO_4BYTE: + case INS_OPTS_H_TO_8BYTE: return EA_2BYTE; @@ -4751,7 +4759,7 @@ void emitter::emitIns_R_R(instruction ins, case INS_fcmp: case INS_fcmpe: assert(insOptsNone(opt)); - assert(isValidVectorElemsizeFloat(size)); + assert(isValidVectorElemsizeFloat(size) || (size == EA_2BYTE)); assert(isVectorRegister(reg1)); assert(isVectorRegister(reg2)); fmt = IF_DV_2K; @@ -4873,7 +4881,7 @@ void emitter::emitIns_R_R(instruction ins, { assert(isGeneralRegister(reg2)); assert(insOptsConvertIntToFloat(opt)); - assert(isValidVectorElemsizeFloat(size)); + assert(isValidVectorElemsizeFloat(size) || (size == EA_2BYTE)); fmt = IF_DV_2I; } } @@ -4905,7 +4913,7 @@ void emitter::emitIns_R_R(instruction ins, { // Scalar operation assert(insOptsNone(opt)); - assert(isValidVectorElemsizeFloat(size)); + assert(isValidVectorElemsizeFloat(size) || (size == EA_2BYTE)); assert(isVectorRegister(reg1)); assert(isVectorRegister(reg2)); fmt = IF_DV_2G; @@ -10118,6 +10126,10 @@ void emitter::emitIns_Call(const EmitCallParams& params) { return 0x00400000; // set the bit at location 22 } + else if (size == EA_2BYTE) + { + return 0x00C00000; // set the bits at location 23 and 22 (ftype=11, half-precision) + } assert(size == EA_4BYTE); return 0x00000000; } @@ -10327,6 +10339,16 @@ void emitter::emitIns_Call(const EmitCallParams& params) result = 0x80400000; // sf=1, type=01 break; + case INS_OPTS_H_TO_4BYTE: // Half to INT32 + assert(fmt == IF_DV_2H); + result = 0x00C00000; // sf=0, type=11 + break; + + case INS_OPTS_H_TO_8BYTE: // Half to INT64 + assert(fmt == IF_DV_2H); + result = 0x80C00000; // sf=1, type=11 + break; + case INS_OPTS_4BYTE_TO_S: // INT32 to Single assert(fmt == IF_DV_2I); result = 0x00000000; // sf=0, type=00 @@ -10347,6 +10369,16 @@ void emitter::emitIns_Call(const EmitCallParams& params) result = 0x80400000; // sf=1, type=01 break; + case INS_OPTS_4BYTE_TO_H: // INT32 to Half + assert(fmt == IF_DV_2I); + result = 0x00C00000; // sf=0, type=11 + break; + + case INS_OPTS_8BYTE_TO_H: // INT64 to Half + assert(fmt == IF_DV_2I); + result = 0x80C00000; // sf=1, type=11 + break; + default: assert(!"Invalid 'conversion' value"); break; @@ -16472,8 +16504,8 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins } else { - // S-form - assert(id->idOpSize() == EA_4BYTE); + // S-form or H-form + assert((id->idOpSize() == EA_4BYTE) || (id->idOpSize() == EA_2BYTE)); result.insThroughput = PERFSCORE_THROUGHPUT_9C; result.insLatency = PERFSCORE_LATENCY_12C; } @@ -16633,8 +16665,8 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins } else { - // S-form - assert(id->idOpSize() == EA_4BYTE); + // S-form or H-form + assert((id->idOpSize() == EA_4BYTE) || (id->idOpSize() == EA_2BYTE)); result.insThroughput = PERFSCORE_THROUGHPUT_3C; result.insLatency = PERFSCORE_LATENCY_10C; } diff --git a/src/coreclr/jit/emitarm64.h b/src/coreclr/jit/emitarm64.h index f2a4c9e7f0b652..38b4505d32c118 100644 --- a/src/coreclr/jit/emitarm64.h +++ b/src/coreclr/jit/emitarm64.h @@ -1113,7 +1113,7 @@ inline static bool isValidGeneralDatasize(emitAttr size) inline static bool isValidScalarDatasize(emitAttr size) { - return (size == EA_8BYTE) || (size == EA_4BYTE); + return (size == EA_8BYTE) || (size == EA_4BYTE) || (size == EA_2BYTE); } inline static bool isValidScalableDatasize(emitAttr size) @@ -1331,12 +1331,12 @@ inline static bool insOptsConvertFloatToFloat(insOpts opt) inline static bool insOptsConvertFloatToInt(insOpts opt) { - return ((opt >= INS_OPTS_S_TO_4BYTE) && (opt <= INS_OPTS_D_TO_8BYTE)); + return ((opt >= INS_OPTS_S_TO_4BYTE) && (opt <= INS_OPTS_H_TO_8BYTE)); } inline static bool insOptsConvertIntToFloat(insOpts opt) { - return ((opt >= INS_OPTS_4BYTE_TO_S) && (opt <= INS_OPTS_8BYTE_TO_D)); + return ((opt >= INS_OPTS_4BYTE_TO_S) && (opt <= INS_OPTS_8BYTE_TO_H)); } inline static bool insOptsScalable(insOpts opt) diff --git a/src/coreclr/jit/instr.h b/src/coreclr/jit/instr.h index bb75395bfccdc6..f2b7310d5c641d 100644 --- a/src/coreclr/jit/instr.h +++ b/src/coreclr/jit/instr.h @@ -402,12 +402,18 @@ enum insOpts : unsigned INS_OPTS_S_TO_8BYTE, // Single to INT64 INS_OPTS_D_TO_8BYTE, // Double to INT64 + INS_OPTS_H_TO_4BYTE, // Half to INT32 + INS_OPTS_H_TO_8BYTE, // Half to INT64 + INS_OPTS_4BYTE_TO_S, // INT32 to Single INS_OPTS_4BYTE_TO_D, // INT32 to Double INS_OPTS_8BYTE_TO_S, // INT64 to Single INS_OPTS_8BYTE_TO_D, // INT64 to Double + INS_OPTS_4BYTE_TO_H, // INT32 to Half + INS_OPTS_8BYTE_TO_H, // INT64 to Half + INS_OPTS_S_TO_D, // Single to Double INS_OPTS_D_TO_S, // Double to Single From 90b53ced491c629c576e02cfb572591d2977db39 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Fri, 10 Jul 2026 10:10:38 -0700 Subject: [PATCH 17/22] Add Arm64 internal Fp16 hardware intrinsic table entries Define the JIT-internal NI_Fp16_* hardware intrinsics used to accelerate System.Half on Arm64 (FEAT_FP16). These mirror the scalar half-precision operations, binding Half internally as TYP_USHORT on TYP_SIMD nodes, and are created directly by the importer rather than via a managed API. Wire up the Fp16 ISA range so the debug range validation recognizes the new contiguous block. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/hwintrinsic.cpp | 2 +- src/coreclr/jit/hwintrinsiclistarm64.h | 35 +++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index cb312268e4bce4..74dd91842ea60e 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -980,7 +980,7 @@ static const HWIntrinsicIsaRange hwintrinsicIsaRangeArray[] = { { FIRST_NI_Crc32, LAST_NI_Crc32 }, // Crc32 { FIRST_NI_Dp, LAST_NI_Dp }, // Dp { FIRST_NI_Rdm, LAST_NI_Rdm }, // Rdm - { NI_Illegal, NI_Illegal }, // Fp16 + { FIRST_NI_Fp16, LAST_NI_Fp16 }, // Fp16 { FIRST_NI_Sha1, LAST_NI_Sha1 }, // Sha1 { FIRST_NI_Sha256, LAST_NI_Sha256 }, // Sha256 { NI_Illegal, NI_Illegal }, // Atomics diff --git a/src/coreclr/jit/hwintrinsiclistarm64.h b/src/coreclr/jit/hwintrinsiclistarm64.h index c04f9bf9a8be30..7a61d7b8892b13 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64.h +++ b/src/coreclr/jit/hwintrinsiclistarm64.h @@ -1,4 +1,4 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. /*****************************************************************************/ @@ -549,6 +549,39 @@ HARDWARE_INTRINSIC(Dp, DotProduct, HARDWARE_INTRINSIC(Dp, DotProductBySelectedQuadruplet, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sdot, INS_udot, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) #define LAST_NI_Dp NI_Dp_DotProductBySelectedQuadruplet +// ************************************************************************************************************************************************************************************************************************************************************************************************************ +// ISA Function name SIMD size NumArg Instructions IntCost FltCost Category Flags +// TYP_BYTE, TYP_UBYTE, TYP_SHORT, TYP_USHORT, TYP_INT, TYP_UINT, TYP_LONG, TYP_ULONG, TYP_FLOAT, TYP_DOUBLE +// ************************************************************************************************************************************************************************************************************************************************************************************************************ +// FP16 Intrinsics +#define FIRST_NI_Fp16 NI_Fp16_Add +HARDWARE_INTRINSIC(Fp16, Add, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_fadd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Fp16, Ceiling, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_frintp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Fp16, CompareEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_fcmp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Fp16, CompareGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_fcmp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Fp16, CompareGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_fcmp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Fp16, CompareLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_fcmp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Fp16, CompareLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_fcmp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Fp16, CompareNotEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_fcmp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Fp16, ConvertToDouble, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_fcvt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Fp16, ConvertToHalf, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_scvtf, INS_ucvtf, INS_scvtf, INS_ucvtf, INS_fcvt, INS_fcvt, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Fp16, ConvertToInt32, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzs, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Fp16, ConvertToInt64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzs, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Fp16, ConvertToSingle, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_fcvt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Fp16, ConvertToUInt32, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzu, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Fp16, ConvertToUInt64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzu, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Fp16, Divide, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_fdiv, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Fp16, Floor, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_frintm, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Fp16, FusedMultiplyAdd, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_fmadd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Fp16, Multiply, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_fmul, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Fp16, ReciprocalEstimate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_frecpe, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Fp16, ReciprocalSqrtEstimate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_frsqrte, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Fp16, RoundToNearest, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_frintn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Fp16, Sqrt, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_fsqrt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Fp16, Subtract, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_fsub, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Fp16, Truncate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_frintz, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +#define LAST_NI_Fp16 NI_Fp16_Truncate + // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // ISA Function name SIMD size NumArg Instructions IntCost FltCost Category Flags // TYP_BYTE, TYP_UBYTE, TYP_SHORT, TYP_USHORT, TYP_INT, TYP_UINT, TYP_LONG, TYP_ULONG, TYP_FLOAT, TYP_DOUBLE From 55fcb5ae5e2f15da53a4f8f83e640fb27cda3102 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Fri, 10 Jul 2026 10:17:10 -0700 Subject: [PATCH 18/22] Add Arm64 importer support for accelerating System.Half Import each System.Half operator/method into the internal NI_Fp16_* scalar hardware intrinsics when FEAT_FP16 is opportunistically available, mirroring the xarch AVX10v1 path. Half continues to bind as TYP_USHORT on TYP_SIMD nodes, so there is no ABI impact. Notable Arm64 deviations from xarch: Sqrt/ReciprocalEstimate/RSqrtEstimate and the rounding ops are unary (no dummy upper-bits source, no rounding immediate); Half->arithmetic converts return their scalar result directly in the target register (no vector extraction); and all six integer/floating conversions are available regardless of pointer size. Extend lookupHalfIntrinsic to compile for both xarch and Arm64. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/compiler.h | 6 +- src/coreclr/jit/importercalls.cpp | 196 +++++++++++++++++++++++++++++- 2 files changed, 197 insertions(+), 5 deletions(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 98a2d997695866..ce8f409d7c9986 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -5323,9 +5323,11 @@ class Compiler NamedIntrinsic lookupPrimitiveFloatNamedIntrinsic(CORINFO_METHOD_HANDLE method, const char* methodName); NamedIntrinsic lookupPrimitiveIntNamedIntrinsic(CORINFO_METHOD_HANDLE method, const char* methodName); NamedIntrinsic lookupHalfNamedIntrinsic(CORINFO_METHOD_HANDLE method, const char* methodName); -#if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_XARCH) +#if defined(FEATURE_HW_INTRINSICS) && (defined(TARGET_XARCH) || defined(TARGET_ARM64)) NamedIntrinsic lookupHalfIntrinsic(NamedIntrinsic ni); - int lookupHalfRoundingMode(NamedIntrinsic ni); +#endif // FEATURE_HW_INTRINSICS && (TARGET_XARCH || TARGET_ARM64) +#if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_XARCH) + int lookupHalfRoundingMode(NamedIntrinsic ni); #endif // FEATURE_HW_INTRINSICS && TARGET_XARCH GenTree* impUnsupportedNamedIntrinsic(unsigned helper, CORINFO_METHOD_HANDLE method, diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index e90963fbb8726b..19ad8f33274e91 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -4574,6 +4574,21 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, NI_AVX2_ConvertToVector128Half, TYP_FLOAT, 16); retNode = impSimdToScalarHalf(retNode, retClsHnd); } +#elif defined(TARGET_ARM64) + if (compOpportunisticallyDependsOn(InstructionSet_Fp16)) + { + GenTree* op1 = impPopStack().val; + + // The integer scalar convert instructions read the source directly from a general + // purpose register, so only floating-point sources need to be moved into a vector. + if (varTypeIsFloating(op1Type)) + { + op1 = gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, op1, op1Type, 16); + } + + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, NI_Fp16_ConvertToHalf, op1Type, 16); + retNode = impSimdToScalarHalf(retNode, retClsHnd); + } #endif // TARGET_XARCH } else @@ -4644,6 +4659,46 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, NI_AVX2_ConvertToVector128Single, TYP_USHORT, 16); retNode = gtNewSimdToScalarNode(TYP_FLOAT, retNode, TYP_FLOAT, 16); } +#elif defined(TARGET_ARM64) + if (compOpportunisticallyDependsOn(InstructionSet_Fp16)) + { + NamedIntrinsic opId = NI_Illegal; + + switch (retType) + { + case TYP_FLOAT: + opId = NI_Fp16_ConvertToSingle; + break; + case TYP_DOUBLE: + opId = NI_Fp16_ConvertToDouble; + break; + case TYP_INT: + opId = NI_Fp16_ConvertToInt32; + break; + case TYP_UINT: + opId = NI_Fp16_ConvertToUInt32; + break; + case TYP_LONG: + opId = NI_Fp16_ConvertToInt64; + break; + case TYP_ULONG: + opId = NI_Fp16_ConvertToUInt64; + break; + default: + break; + } + + if (opId != NI_Illegal) + { + GenTree* op1 = impPopStack().val; + op1 = impSimdCreateScalarHalf(op1); + + // The Arm64 scalar convert instructions produce their result directly in the + // target register (an FP register for float/double, a general purpose register + // for integers), so no vector extraction is needed. + retNode = gtNewSimdHWIntrinsicNode(genActualType(retType), op1, opId, TYP_USHORT, 16); + } + } #endif // TARGET_XARCH } break; @@ -4663,6 +4718,20 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, GenTree* op2 = impPopStack().val; GenTree* op1 = impPopStack().val; + op2 = impSimdCreateScalarHalf(op2); + op1 = impSimdCreateScalarHalf(op1); + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, opId, TYP_USHORT, 16); + retNode = impSimdToScalarHalf(retNode, clsHnd); + } +#elif defined(TARGET_ARM64) + if (compOpportunisticallyDependsOn(InstructionSet_Fp16)) + { + NamedIntrinsic opId = lookupHalfIntrinsic(ni); + assert(opId != NI_Illegal); + + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + op2 = impSimdCreateScalarHalf(op2); op1 = impSimdCreateScalarHalf(op1); retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, opId, TYP_USHORT, 16); @@ -4691,6 +4760,17 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op2, op1, opId, TYP_USHORT, 16); retNode = impSimdToScalarHalf(retNode, clsHnd); } +#elif defined(TARGET_ARM64) + if (compOpportunisticallyDependsOn(InstructionSet_Fp16)) + { + NamedIntrinsic opId = lookupHalfIntrinsic(ni); + assert(opId != NI_Illegal); + + GenTree* op1 = impPopStack().val; + op1 = impSimdCreateScalarHalf(op1); + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, opId, TYP_USHORT, 16); + retNode = impSimdToScalarHalf(retNode, clsHnd); + } #endif // TARGET_XARCH break; } @@ -4711,6 +4791,22 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, TYP_USHORT, 16); retNode = impSimdToScalarHalf(retNode, clsHnd); } +#elif defined(TARGET_ARM64) + if (compOpportunisticallyDependsOn(InstructionSet_Fp16)) + { + GenTree* op3 = impPopStack().val; + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + + op3 = impSimdCreateScalarHalf(op3); + op2 = impSimdCreateScalarHalf(op2); + op1 = impSimdCreateScalarHalf(op1); + + // fmadd computes Rd = Rn * Rm + Ra, so (op1 * op2) + op3 == x * y + z. + retNode = + gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, op3, NI_Fp16_FusedMultiplyAdd, TYP_USHORT, 16); + retNode = impSimdToScalarHalf(retNode, clsHnd); + } #endif // TARGET_XARCH break; } @@ -4734,6 +4830,19 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, NI_AVX10v1_RoundScaleScalar, TYP_USHORT, 16); retNode = impSimdToScalarHalf(retNode, clsHnd); } +#elif defined(TARGET_ARM64) + // todo-arm64-half: We only optimize the single-argument overloads for now. + if (compOpportunisticallyDependsOn(InstructionSet_Fp16) && (sig->numArgs == 1)) + { + // Arm64 has a dedicated rounding instruction per mode, so no rounding immediate is needed. + NamedIntrinsic opId = lookupHalfIntrinsic(ni); + assert(opId != NI_Illegal); + + GenTree* op1 = impPopStack().val; + op1 = impSimdCreateScalarHalf(op1); + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, opId, TYP_USHORT, 16); + retNode = impSimdToScalarHalf(retNode, clsHnd); + } #endif // TARGET_XARCH break; } @@ -4754,6 +4863,19 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, GenTree* op2 = impPopStack().val; GenTree* op1 = impPopStack().val; + op2 = impSimdCreateScalarHalf(op2); + op1 = impSimdCreateScalarHalf(op1); + retNode = gtNewSimdHWIntrinsicNode(TYP_INT, op1, op2, opId, TYP_USHORT, 16); + } +#elif defined(TARGET_ARM64) + if (compOpportunisticallyDependsOn(InstructionSet_Fp16)) + { + NamedIntrinsic opId = lookupHalfIntrinsic(ni); + assert(opId != NI_Illegal); + + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + op2 = impSimdCreateScalarHalf(op2); op1 = impSimdCreateScalarHalf(op1); retNode = gtNewSimdHWIntrinsicNode(TYP_INT, op1, op2, opId, TYP_USHORT, 16); @@ -4779,6 +4901,22 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, oneVec = gtNewSimdHWIntrinsicNode(TYP_SIMD16, zeroVec, oneVec, NI_AVX10v1_ConvertScalarToVector128Half, TYP_FLOAT, 16); + op1 = impSimdCreateScalarHalf(op1); + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, oneVec, opId, TYP_USHORT, 16); + retNode = impSimdToScalarHalf(retNode, clsHnd); + } +#elif defined(TARGET_ARM64) + if (compOpportunisticallyDependsOn(InstructionSet_Fp16)) + { + NamedIntrinsic opId = lookupHalfIntrinsic(ni); + assert(opId != NI_Illegal); + + GenTree* op1 = impPopStack().val; + + // Materialize 1.0 as a Half in lane 0 of a vector by converting 1.0f -> half. + GenTree* oneVec = gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, gtNewDconNodeF(1.0f), TYP_FLOAT, 16); + oneVec = gtNewSimdHWIntrinsicNode(TYP_SIMD16, oneVec, NI_Fp16_ConvertToHalf, TYP_FLOAT, 16); + op1 = impSimdCreateScalarHalf(op1); retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, oneVec, opId, TYP_USHORT, 16); retNode = impSimdToScalarHalf(retNode, clsHnd); @@ -12549,19 +12687,20 @@ NamedIntrinsic Compiler::lookupHalfNamedIntrinsic(CORINFO_METHOD_HANDLE method, return result; } -#if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_XARCH) +#if defined(FEATURE_HW_INTRINSICS) && (defined(TARGET_XARCH) || defined(TARGET_ARM64)) //------------------------------------------------------------------------ -// lookupHalfIntrinsic: map a System.Half named intrinsic to the AVX10v1 scalar +// lookupHalfIntrinsic: map a System.Half named intrinsic to the internal scalar // hardware intrinsic that implements it // // Arguments: // ni -- the System.Half named intrinsic // // Return Value: -// The corresponding NI_AVX10v1_* intrinsic, or NI_Illegal if none. +// The corresponding scalar hardware intrinsic, or NI_Illegal if none. // NamedIntrinsic Compiler::lookupHalfIntrinsic(NamedIntrinsic ni) { +#if defined(TARGET_XARCH) assert(compOpportunisticallyDependsOn(InstructionSet_AVX10v1)); switch (ni) @@ -12606,8 +12745,59 @@ NamedIntrinsic Compiler::lookupHalfIntrinsic(NamedIntrinsic ni) default: return NI_Illegal; } +#elif defined(TARGET_ARM64) + assert(compOpportunisticallyDependsOn(InstructionSet_Fp16)); + + switch (ni) + { + case NI_System_Half_op_Addition: + return NI_Fp16_Add; + case NI_System_Half_op_Increment: + return NI_Fp16_Add; + case NI_System_Half_op_Subtraction: + return NI_Fp16_Subtract; + case NI_System_Half_op_Decrement: + return NI_Fp16_Subtract; + case NI_System_Half_op_Multiply: + return NI_Fp16_Multiply; + case NI_System_Half_op_Division: + return NI_Fp16_Divide; + case NI_System_Half_Sqrt: + return NI_Fp16_Sqrt; + case NI_System_Half_ReciprocalEstimate: + return NI_Fp16_ReciprocalEstimate; + case NI_System_Half_ReciprocalSqrtEstimate: + return NI_Fp16_ReciprocalSqrtEstimate; + case NI_System_Half_FusedMultiplyAdd: + return NI_Fp16_FusedMultiplyAdd; + case NI_System_Half_op_GreaterThan: + return NI_Fp16_CompareGreaterThan; + case NI_System_Half_op_GreaterThanOrEqual: + return NI_Fp16_CompareGreaterThanOrEqual; + case NI_System_Half_op_LessThan: + return NI_Fp16_CompareLessThan; + case NI_System_Half_op_LessThanOrEqual: + return NI_Fp16_CompareLessThanOrEqual; + case NI_System_Half_op_Equality: + return NI_Fp16_CompareEqual; + case NI_System_Half_op_Inequality: + return NI_Fp16_CompareNotEqual; + case NI_System_Half_Round: + return NI_Fp16_RoundToNearest; + case NI_System_Half_Ceiling: + return NI_Fp16_Ceiling; + case NI_System_Half_Floor: + return NI_Fp16_Floor; + case NI_System_Half_Truncate: + return NI_Fp16_Truncate; + default: + return NI_Illegal; + } +#endif // TARGET_ARM64 } +#endif // FEATURE_HW_INTRINSICS && (TARGET_XARCH || TARGET_ARM64) +#if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_XARCH) //------------------------------------------------------------------------ // lookupHalfRoundingMode: map a System.Half rounding named intrinsic to the // immediate rounding mode used by RoundScaleScalar From 838b8d40b13807861f0f5837e8f7a813ffd6cabd Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Fri, 10 Jul 2026 10:23:44 -0700 Subject: [PATCH 19/22] Add Arm64 codegen for Fp16 compare and conversion intrinsics Emit the special-codegen sequences for the internal NI_Fp16_* compare and conversion intrinsics used to accelerate System.Half: - Compares lower to fcmp + cset. Less-than and less-than-or-equal use the 'mi'/'ls' conditions rather than 'lt'/'le' so an unordered (NaN) operand yields false, matching C# ordered comparison semantics. - Half<->arithmetic conversions lower to fcvt/fcvtzs/fcvtzu/scvtf/ucvtf with the appropriate conversion option and destination size. Arithmetic, unary and fmadd intrinsics need no additions here since they are scalar and handled by the table-driven path. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 88 +++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index 85d24554e9336c..308ad36390218e 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -1293,6 +1293,94 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) } break; + case NI_Fp16_ConvertToSingle: + GetEmitter()->emitIns_R_R(ins, EA_4BYTE, targetReg, op1Reg, INS_OPTS_H_TO_S); + break; + + case NI_Fp16_ConvertToDouble: + GetEmitter()->emitIns_R_R(ins, EA_8BYTE, targetReg, op1Reg, INS_OPTS_H_TO_D); + break; + + case NI_Fp16_ConvertToInt32: + case NI_Fp16_ConvertToUInt32: + GetEmitter()->emitIns_R_R(ins, EA_4BYTE, targetReg, op1Reg, INS_OPTS_H_TO_4BYTE); + break; + + case NI_Fp16_ConvertToInt64: + case NI_Fp16_ConvertToUInt64: + GetEmitter()->emitIns_R_R(ins, EA_8BYTE, targetReg, op1Reg, INS_OPTS_H_TO_8BYTE); + break; + + case NI_Fp16_ConvertToHalf: + { + // The instruction is already selected by the source (base) type; only the + // conversion option needs to distinguish the source width and signedness. + insOpts cvtOption = INS_OPTS_NONE; + + switch (intrin.baseType) + { + case TYP_FLOAT: + cvtOption = INS_OPTS_S_TO_H; + break; + case TYP_DOUBLE: + cvtOption = INS_OPTS_D_TO_H; + break; + case TYP_INT: + case TYP_UINT: + cvtOption = INS_OPTS_4BYTE_TO_H; + break; + case TYP_LONG: + case TYP_ULONG: + cvtOption = INS_OPTS_8BYTE_TO_H; + break; + default: + unreached(); + } + + GetEmitter()->emitIns_R_R(ins, EA_2BYTE, targetReg, op1Reg, cvtOption); + break; + } + + case NI_Fp16_CompareEqual: + case NI_Fp16_CompareGreaterThan: + case NI_Fp16_CompareGreaterThanOrEqual: + case NI_Fp16_CompareLessThan: + case NI_Fp16_CompareLessThanOrEqual: + case NI_Fp16_CompareNotEqual: + { + insCond cond = INS_COND_EQ; + + switch (intrin.id) + { + case NI_Fp16_CompareEqual: + cond = INS_COND_EQ; + break; + case NI_Fp16_CompareGreaterThan: + cond = INS_COND_GT; + break; + case NI_Fp16_CompareGreaterThanOrEqual: + cond = INS_COND_GE; + break; + case NI_Fp16_CompareLessThan: + // 'mi' rather than 'lt' so an unordered (NaN) comparison yields false. + cond = INS_COND_MI; + break; + case NI_Fp16_CompareLessThanOrEqual: + // 'ls' rather than 'le' so an unordered (NaN) comparison yields false. + cond = INS_COND_LS; + break; + case NI_Fp16_CompareNotEqual: + cond = INS_COND_NE; + break; + default: + unreached(); + } + + GetEmitter()->emitIns_R_R(INS_fcmp, EA_2BYTE, op1Reg, op2Reg); + GetEmitter()->emitIns_R_COND(INS_cset, EA_4BYTE, targetReg, cond); + break; + } + case NI_Crc32_ComputeCrc32: case NI_Crc32_ComputeCrc32C: case NI_Crc32_Arm64_ComputeCrc32: From 75d41fd4333d2addeb952186cd560dc6ce750184 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Fri, 10 Jul 2026 11:05:48 -0700 Subject: [PATCH 20/22] Accelerate Arm64 Half<->float/double conversions at the FP baseline FCVT between half and single/double is part of the Armv8.0 FP baseline and does not require FEAT_FP16, so these precision conversions can always be accelerated -- mirroring how F16C provides float<->half below the AVX10.1 arithmetic bar on x64. Arm64 is even better here since baseline FCVT covers both float<->half and double<->half. Represent them as internal NI_ArmBase_ConvertToSingle/Double/Half intrinsics (ArmBase is guaranteed on Arm64 and scalar FCVT is not strictly SIMD), so the codegen ISA assert on NI_Fp16_* does not block reuse at the baseline. Integer <->half conversions (SCVTF/UCVTF/FCVTZS/FCVTZU with a half operand) still require FEAT_FP16 and remain gated on NI_Fp16_*. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 19 +++-- src/coreclr/jit/hwintrinsiclistarm64.h | 12 ++- src/coreclr/jit/importercalls.cpp | 92 ++++++++++++--------- 3 files changed, 70 insertions(+), 53 deletions(-) diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index 308ad36390218e..fe533351076eb4 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -1293,14 +1293,23 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) } break; - case NI_Fp16_ConvertToSingle: + case NI_ArmBase_ConvertToSingle: GetEmitter()->emitIns_R_R(ins, EA_4BYTE, targetReg, op1Reg, INS_OPTS_H_TO_S); break; - case NI_Fp16_ConvertToDouble: + case NI_ArmBase_ConvertToDouble: GetEmitter()->emitIns_R_R(ins, EA_8BYTE, targetReg, op1Reg, INS_OPTS_H_TO_D); break; + case NI_ArmBase_ConvertToHalf: + { + // Baseline FCVT precision conversion; the source (base) type selects the width. + assert((intrin.baseType == TYP_FLOAT) || (intrin.baseType == TYP_DOUBLE)); + insOpts cvtOption = (intrin.baseType == TYP_FLOAT) ? INS_OPTS_S_TO_H : INS_OPTS_D_TO_H; + GetEmitter()->emitIns_R_R(ins, EA_2BYTE, targetReg, op1Reg, cvtOption); + break; + } + case NI_Fp16_ConvertToInt32: case NI_Fp16_ConvertToUInt32: GetEmitter()->emitIns_R_R(ins, EA_4BYTE, targetReg, op1Reg, INS_OPTS_H_TO_4BYTE); @@ -1319,12 +1328,6 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) switch (intrin.baseType) { - case TYP_FLOAT: - cvtOption = INS_OPTS_S_TO_H; - break; - case TYP_DOUBLE: - cvtOption = INS_OPTS_D_TO_H; - break; case TYP_INT: case TYP_UINT: cvtOption = INS_OPTS_4BYTE_TO_H; diff --git a/src/coreclr/jit/hwintrinsiclistarm64.h b/src/coreclr/jit/hwintrinsiclistarm64.h index 7a61d7b8892b13..51f5f068379d6e 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64.h +++ b/src/coreclr/jit/hwintrinsiclistarm64.h @@ -498,7 +498,13 @@ HARDWARE_INTRINSIC(Aes, PolynomialMultiplyWideningUpper, // TYP_BYTE, TYP_UBYTE, TYP_SHORT, TYP_USHORT, TYP_INT, TYP_UINT, TYP_LONG, TYP_ULONG, TYP_FLOAT, TYP_DOUBLE // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Base Intrinsics -#define FIRST_NI_ArmBase NI_ArmBase_LeadingZeroCount +// NOTE: ConvertToDouble/ConvertToHalf/ConvertToSingle below are JIT-internal only (no managed API). They accelerate +// the baseline half<->single/double FCVT conversions used by System.Half. FCVT between half and single/double is part +// of the base FP/AdvSimd baseline (Armv8.0) and does NOT require FEAT_FP16, so they live in ArmBase rather than Fp16. +#define FIRST_NI_ArmBase NI_ArmBase_ConvertToDouble +HARDWARE_INTRINSIC(ArmBase, ConvertToDouble, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_fcvt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(ArmBase, ConvertToHalf, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvt, INS_fcvt, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(ArmBase, ConvertToSingle, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_fcvt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(ArmBase, LeadingZeroCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_clz, INS_clz, INS_clz, INS_clz, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoFloatingPointUsed) HARDWARE_INTRINSIC(ArmBase, ReverseElementBits, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rbit, INS_rbit, INS_rbit, INS_rbit, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) HARDWARE_INTRINSIC(ArmBase, Yield, 0, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_SpecialSideEffect_Other) @@ -563,11 +569,9 @@ HARDWARE_INTRINSIC(Fp16, CompareGreaterThanOrEqual, HARDWARE_INTRINSIC(Fp16, CompareLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_fcmp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Fp16, CompareLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_fcmp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Fp16, CompareNotEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_fcmp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Fp16, ConvertToDouble, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_fcvt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Fp16, ConvertToHalf, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_scvtf, INS_ucvtf, INS_scvtf, INS_ucvtf, INS_fcvt, INS_fcvt, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Fp16, ConvertToHalf, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_scvtf, INS_ucvtf, INS_scvtf, INS_ucvtf, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Fp16, ConvertToInt32, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzs, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Fp16, ConvertToInt64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzs, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) -HARDWARE_INTRINSIC(Fp16, ConvertToSingle, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_fcvt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Fp16, ConvertToUInt32, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzu, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Fp16, ConvertToUInt64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzu, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Fp16, Divide, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_fdiv, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 19ad8f33274e91..3fb3146db98493 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -4575,16 +4575,22 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, retNode = impSimdToScalarHalf(retNode, retClsHnd); } #elif defined(TARGET_ARM64) - if (compOpportunisticallyDependsOn(InstructionSet_Fp16)) + // FCVT between half and single/double is part of the Armv8.0 FP baseline and does not + // require FEAT_FP16, so those conversions are always accelerated. Integer -> half + // conversions (SCVTF/UCVTF with a half destination) require FEAT_FP16. + if (varTypeIsFloating(op1Type)) { GenTree* op1 = impPopStack().val; + op1 = gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, op1, op1Type, 16); + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, NI_ArmBase_ConvertToHalf, op1Type, 16); + retNode = impSimdToScalarHalf(retNode, retClsHnd); + } + else if (compOpportunisticallyDependsOn(InstructionSet_Fp16)) + { // The integer scalar convert instructions read the source directly from a general - // purpose register, so only floating-point sources need to be moved into a vector. - if (varTypeIsFloating(op1Type)) - { - op1 = gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, op1, op1Type, 16); - } + // purpose register, so no move into a vector register is required. + GenTree* op1 = impPopStack().val; retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, NI_Fp16_ConvertToHalf, op1Type, 16); retNode = impSimdToScalarHalf(retNode, retClsHnd); @@ -4660,44 +4666,48 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, retNode = gtNewSimdToScalarNode(TYP_FLOAT, retNode, TYP_FLOAT, 16); } #elif defined(TARGET_ARM64) - if (compOpportunisticallyDependsOn(InstructionSet_Fp16)) - { - NamedIntrinsic opId = NI_Illegal; + // Half -> single/double is a baseline FCVT (no FEAT_FP16 needed); Half -> integer + // (FCVTZS/FCVTZU with a half operand) requires FEAT_FP16. + NamedIntrinsic opId = NI_Illegal; + bool isFp16 = false; - switch (retType) - { - case TYP_FLOAT: - opId = NI_Fp16_ConvertToSingle; - break; - case TYP_DOUBLE: - opId = NI_Fp16_ConvertToDouble; - break; - case TYP_INT: - opId = NI_Fp16_ConvertToInt32; - break; - case TYP_UINT: - opId = NI_Fp16_ConvertToUInt32; - break; - case TYP_LONG: - opId = NI_Fp16_ConvertToInt64; - break; - case TYP_ULONG: - opId = NI_Fp16_ConvertToUInt64; - break; - default: - break; - } + switch (retType) + { + case TYP_FLOAT: + opId = NI_ArmBase_ConvertToSingle; + break; + case TYP_DOUBLE: + opId = NI_ArmBase_ConvertToDouble; + break; + case TYP_INT: + opId = NI_Fp16_ConvertToInt32; + isFp16 = true; + break; + case TYP_UINT: + opId = NI_Fp16_ConvertToUInt32; + isFp16 = true; + break; + case TYP_LONG: + opId = NI_Fp16_ConvertToInt64; + isFp16 = true; + break; + case TYP_ULONG: + opId = NI_Fp16_ConvertToUInt64; + isFp16 = true; + break; + default: + break; + } - if (opId != NI_Illegal) - { - GenTree* op1 = impPopStack().val; - op1 = impSimdCreateScalarHalf(op1); + if ((opId != NI_Illegal) && (!isFp16 || compOpportunisticallyDependsOn(InstructionSet_Fp16))) + { + GenTree* op1 = impPopStack().val; + op1 = impSimdCreateScalarHalf(op1); - // The Arm64 scalar convert instructions produce their result directly in the - // target register (an FP register for float/double, a general purpose register - // for integers), so no vector extraction is needed. - retNode = gtNewSimdHWIntrinsicNode(genActualType(retType), op1, opId, TYP_USHORT, 16); - } + // The Arm64 scalar convert instructions produce their result directly in the + // target register (an FP register for float/double, a general purpose register + // for integers), so no vector extraction is needed. + retNode = gtNewSimdHWIntrinsicNode(genActualType(retType), op1, opId, TYP_USHORT, 16); } #endif // TARGET_XARCH } From 9da051c7c59ecc9866536d41cc8e770260e5193a Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Fri, 10 Jul 2026 13:21:29 -0700 Subject: [PATCH 21/22] Wire up Half constant getters and fix the class handle used for generic dispatch The Half constant getters (One, Zero, Epsilon, NaN, MinValue, MaxValue, PositiveInfinity, NegativeInfinity) now intercept as intrinsics and fold to scalar constants, and op_Increment/op_Decrement create the 0x3C00 constant directly rather than converting 1.0f at runtime. More importantly, all Half-returning cases now use sig->retTypeSigClass rather than clsHnd when calling impSimdToScalarHalf. clsHnd is the constrained. token's class handle (the interface/canonical type) under generic constrained dispatch, not System.Half, so it tripped the isSystemHalfClass assert whenever a Half operator or getter was invoked through a generic constraint (e.g. T.PositiveInfinity or x++ on T : INumber). Also drops the stray UTF-8 BOM from hwintrinsiclistarm64.h. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/hwintrinsiclistarm64.h | 2 +- src/coreclr/jit/importercalls.cpp | 127 +++++++++++++++--- .../System.Private.CoreLib/src/System/Half.cs | 16 +-- 3 files changed, 118 insertions(+), 27 deletions(-) diff --git a/src/coreclr/jit/hwintrinsiclistarm64.h b/src/coreclr/jit/hwintrinsiclistarm64.h index 51f5f068379d6e..39ef6158c01b9c 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64.h +++ b/src/coreclr/jit/hwintrinsiclistarm64.h @@ -1,4 +1,4 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. /*****************************************************************************/ diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 3fb3146db98493..7ef6eb00fbfe67 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -4731,7 +4731,7 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, op2 = impSimdCreateScalarHalf(op2); op1 = impSimdCreateScalarHalf(op1); retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, opId, TYP_USHORT, 16); - retNode = impSimdToScalarHalf(retNode, clsHnd); + retNode = impSimdToScalarHalf(retNode, sig->retTypeSigClass); } #elif defined(TARGET_ARM64) if (compOpportunisticallyDependsOn(InstructionSet_Fp16)) @@ -4745,7 +4745,7 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, op2 = impSimdCreateScalarHalf(op2); op1 = impSimdCreateScalarHalf(op1); retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, opId, TYP_USHORT, 16); - retNode = impSimdToScalarHalf(retNode, clsHnd); + retNode = impSimdToScalarHalf(retNode, sig->retTypeSigClass); } #endif // TARGET_XARCH break; @@ -4768,7 +4768,7 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, GenTree* op2 = gtNewZeroConNode(TYP_SIMD16); op1 = impSimdCreateScalarHalf(op1); retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op2, op1, opId, TYP_USHORT, 16); - retNode = impSimdToScalarHalf(retNode, clsHnd); + retNode = impSimdToScalarHalf(retNode, sig->retTypeSigClass); } #elif defined(TARGET_ARM64) if (compOpportunisticallyDependsOn(InstructionSet_Fp16)) @@ -4779,7 +4779,7 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, GenTree* op1 = impPopStack().val; op1 = impSimdCreateScalarHalf(op1); retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, opId, TYP_USHORT, 16); - retNode = impSimdToScalarHalf(retNode, clsHnd); + retNode = impSimdToScalarHalf(retNode, sig->retTypeSigClass); } #endif // TARGET_XARCH break; @@ -4799,7 +4799,7 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, op1 = impSimdCreateScalarHalf(op1); retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, op3, NI_AVX10v1_FusedMultiplyAddScalar, TYP_USHORT, 16); - retNode = impSimdToScalarHalf(retNode, clsHnd); + retNode = impSimdToScalarHalf(retNode, sig->retTypeSigClass); } #elif defined(TARGET_ARM64) if (compOpportunisticallyDependsOn(InstructionSet_Fp16)) @@ -4815,7 +4815,7 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, // fmadd computes Rd = Rn * Rm + Ra, so (op1 * op2) + op3 == x * y + z. retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, op3, NI_Fp16_FusedMultiplyAdd, TYP_USHORT, 16); - retNode = impSimdToScalarHalf(retNode, clsHnd); + retNode = impSimdToScalarHalf(retNode, sig->retTypeSigClass); } #endif // TARGET_XARCH break; @@ -4838,7 +4838,7 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, op1 = impSimdCreateScalarHalf(op1); retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op2, op1, gtNewIconNode(halfRoundingMode, TYP_INT), NI_AVX10v1_RoundScaleScalar, TYP_USHORT, 16); - retNode = impSimdToScalarHalf(retNode, clsHnd); + retNode = impSimdToScalarHalf(retNode, sig->retTypeSigClass); } #elif defined(TARGET_ARM64) // todo-arm64-half: We only optimize the single-argument overloads for now. @@ -4851,7 +4851,7 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, GenTree* op1 = impPopStack().val; op1 = impSimdCreateScalarHalf(op1); retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, opId, TYP_USHORT, 16); - retNode = impSimdToScalarHalf(retNode, clsHnd); + retNode = impSimdToScalarHalf(retNode, sig->retTypeSigClass); } #endif // TARGET_XARCH break; @@ -4905,15 +4905,14 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, GenTree* op1 = impPopStack().val; - // Materialize 1.0 as a Half in lane 0 of a vector by converting 1.0f -> half. - GenTree* oneVec = gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, gtNewDconNodeF(1.0f), TYP_FLOAT, 16); - GenTree* zeroVec = gtNewZeroConNode(TYP_SIMD16); - oneVec = gtNewSimdHWIntrinsicNode(TYP_SIMD16, zeroVec, oneVec, - NI_AVX10v1_ConvertScalarToVector128Half, TYP_FLOAT, 16); + // Increment/decrement by the Half constant 1.0 (0x3C00). Creating the constant + // directly avoids a runtime float -> half conversion. + GenTree* oneVec = + gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, gtNewIconNode(0x3C00, TYP_INT), TYP_USHORT, 16); op1 = impSimdCreateScalarHalf(op1); retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, oneVec, opId, TYP_USHORT, 16); - retNode = impSimdToScalarHalf(retNode, clsHnd); + retNode = impSimdToScalarHalf(retNode, sig->retTypeSigClass); } #elif defined(TARGET_ARM64) if (compOpportunisticallyDependsOn(InstructionSet_Fp16)) @@ -4923,18 +4922,78 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, GenTree* op1 = impPopStack().val; - // Materialize 1.0 as a Half in lane 0 of a vector by converting 1.0f -> half. - GenTree* oneVec = gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, gtNewDconNodeF(1.0f), TYP_FLOAT, 16); - oneVec = gtNewSimdHWIntrinsicNode(TYP_SIMD16, oneVec, NI_Fp16_ConvertToHalf, TYP_FLOAT, 16); + // Increment/decrement by the Half constant 1.0 (0x3C00). Creating the constant + // directly avoids a runtime float -> half conversion. + GenTree* oneVec = + gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, gtNewIconNode(0x3C00, TYP_INT), TYP_USHORT, 16); op1 = impSimdCreateScalarHalf(op1); retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, oneVec, opId, TYP_USHORT, 16); - retNode = impSimdToScalarHalf(retNode, clsHnd); + retNode = impSimdToScalarHalf(retNode, sig->retTypeSigClass); } #endif // TARGET_XARCH break; } + case NI_System_Half_get_MinValue: + case NI_System_Half_get_MaxValue: + case NI_System_Half_get_Epsilon: + case NI_System_Half_get_NaN: + case NI_System_Half_get_PositiveInfinity: + case NI_System_Half_get_NegativeInfinity: + case NI_System_Half_get_One: + case NI_System_Half_get_Zero: + { +#if defined(TARGET_XARCH) || defined(TARGET_ARM64) + uint16_t halfBits = 0; + + switch (ni) + { + case NI_System_Half_get_MinValue: + halfBits = 0xFBFF; // -65504 + break; + case NI_System_Half_get_MaxValue: + halfBits = 0x7BFF; // 65504 + break; + case NI_System_Half_get_Epsilon: + halfBits = 0x0001; // ~5.9604645e-08 (smallest positive subnormal) + break; + case NI_System_Half_get_NaN: + halfBits = 0xFE00; // Negative NaN + break; + case NI_System_Half_get_PositiveInfinity: + halfBits = 0x7C00; // +Infinity + break; + case NI_System_Half_get_NegativeInfinity: + halfBits = 0xFC00; // -Infinity + break; + case NI_System_Half_get_One: + halfBits = 0x3C00; // 1.0 + break; + case NI_System_Half_get_Zero: + halfBits = 0x0000; // 0.0 + break; + default: + unreached(); + } + +#if defined(TARGET_XARCH) + if (compOpportunisticallyDependsOn(InstructionSet_AVX10v1)) +#else + if (compOpportunisticallyDependsOn(InstructionSet_Fp16)) +#endif + { + // Create the Half constant directly from its bit pattern rather than materializing + // it via a runtime conversion. The return type is always System.Half, so its class + // handle can be taken from the signature (which is reliable even when the getter is + // reached via a generic constrained call). + retNode = gtNewSimdCreateScalarNode(TYP_SIMD16, gtNewIconNode(halfBits, TYP_INT), TYP_USHORT, 16); + retNode = impSimdToScalarHalf(retNode, sig->retTypeSigClass); + } +#endif // TARGET_XARCH || TARGET_ARM64 + break; + } + case NI_System_Math_FusedMultiplyAdd: { assert(varTypeIsFloating(callType)); @@ -12693,6 +12752,38 @@ NamedIntrinsic Compiler::lookupHalfNamedIntrinsic(CORINFO_METHOD_HANDLE method, { result = NI_System_Half_op_Decrement; } + else if (strcmp(methodName, "get_MinValue") == 0) + { + result = NI_System_Half_get_MinValue; + } + else if (strcmp(methodName, "get_MaxValue") == 0) + { + result = NI_System_Half_get_MaxValue; + } + else if (strcmp(methodName, "get_Epsilon") == 0) + { + result = NI_System_Half_get_Epsilon; + } + else if (strcmp(methodName, "get_NaN") == 0) + { + result = NI_System_Half_get_NaN; + } + else if (strcmp(methodName, "get_PositiveInfinity") == 0) + { + result = NI_System_Half_get_PositiveInfinity; + } + else if (strcmp(methodName, "get_NegativeInfinity") == 0) + { + result = NI_System_Half_get_NegativeInfinity; + } + else if (strcmp(methodName, "get_One") == 0) + { + result = NI_System_Half_get_One; + } + else if (strcmp(methodName, "get_Zero") == 0) + { + result = NI_System_Half_get_Zero; + } return result; } diff --git a/src/libraries/System.Private.CoreLib/src/System/Half.cs b/src/libraries/System.Private.CoreLib/src/System/Half.cs index 79d6a848283df1..4625db29d88a2b 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Half.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Half.cs @@ -87,19 +87,19 @@ public readonly struct Half // Well-defined and commonly used values - public static Half Epsilon => new Half(EpsilonBits); // 5.9604645E-08 + public static Half Epsilon { [Intrinsic] get => new Half(EpsilonBits); } // 5.9604645E-08 - public static Half PositiveInfinity => new Half(PositiveInfinityBits); // 1.0 / 0.0; + public static Half PositiveInfinity { [Intrinsic] get => new Half(PositiveInfinityBits); } // 1.0 / 0.0; - public static Half NegativeInfinity => new Half(NegativeInfinityBits); // -1.0 / 0.0 + public static Half NegativeInfinity { [Intrinsic] get => new Half(NegativeInfinityBits); } // -1.0 / 0.0 - public static Half NaN => new Half(NegativeQNaNBits); // 0.0 / 0.0 + public static Half NaN { [Intrinsic] get => new Half(NegativeQNaNBits); } // 0.0 / 0.0 /// - public static Half MinValue => new Half(MinValueBits); // -65504 + public static Half MinValue { [Intrinsic] get => new Half(MinValueBits); } // -65504 /// - public static Half MaxValue => new Half(MaxValueBits); // 65504 + public static Half MaxValue { [Intrinsic] get => new Half(MaxValueBits); } // 65504 internal readonly ushort _value; @@ -1766,13 +1766,13 @@ public static int Sign(Half value) // /// - public static Half One => new Half(PositiveOneBits); + public static Half One { [Intrinsic] get => new Half(PositiveOneBits); } /// static int INumberBase.Radix => 2; /// - public static Half Zero => new Half(PositiveZeroBits); + public static Half Zero { [Intrinsic] get => new Half(PositiveZeroBits); } /// public static Half Abs(Half value) => new Half((ushort)(value._value & ~SignMask)); From a4a1fdd09c4eda4a55ea095cf4051a58d3cb9f23 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Fri, 10 Jul 2026 16:23:59 -0700 Subject: [PATCH 22/22] Use the standard TODO- prefix for the Half rounding follow-up comments Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/importercalls.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 7ef6eb00fbfe67..80b7bd5120c57e 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -4827,7 +4827,7 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, case NI_System_Half_Truncate: { #if defined(TARGET_XARCH) - // todo-xarch-half: We only optimize the single-argument overloads for now. + // TODO-CQ-XArch: We only optimize the single-argument overloads for now. if (compOpportunisticallyDependsOn(InstructionSet_AVX10v1) && (sig->numArgs == 1)) { GenTree* op1 = impPopStack().val; @@ -4841,7 +4841,7 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, retNode = impSimdToScalarHalf(retNode, sig->retTypeSigClass); } #elif defined(TARGET_ARM64) - // todo-arm64-half: We only optimize the single-argument overloads for now. + // TODO-ARM64-CQ: We only optimize the single-argument overloads for now. if (compOpportunisticallyDependsOn(InstructionSet_Fp16) && (sig->numArgs == 1)) { // Arm64 has a dedicated rounding instruction per mode, so no rounding immediate is needed.