diff --git a/src/coreclr/inc/clrconfigvalues.h b/src/coreclr/inc/clrconfigvalues.h index e923170681b432..2954839a3ef851 100644 --- a/src/coreclr/inc/clrconfigvalues.h +++ b/src/coreclr/inc/clrconfigvalues.h @@ -700,6 +700,7 @@ RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Atomics, W("EnableArm64At RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Crc32, W("EnableArm64Crc32"), 1, "Allows Arm64 Crc32+ hardware intrinsics to be disabled") RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Dczva, W("EnableArm64Dczva"), 1, "Allows Arm64 Dczva+ hardware intrinsics to be disabled") RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Dp, W("EnableArm64Dp"), 1, "Allows Arm64 Dp+ hardware intrinsics to be disabled") +RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Fp16, W("EnableArm64Fp16"), 1, "Allows Arm64 Fp16+ hardware intrinsics to be disabled") RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Rdm, W("EnableArm64Rdm"), 1, "Allows Arm64 Rdm+ hardware intrinsics to be disabled") RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Sha1, W("EnableArm64Sha1"), 1, "Allows Arm64 Sha1+ hardware intrinsics to be disabled") RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Sha256, W("EnableArm64Sha256"), 1, "Allows Arm64 Sha256+ hardware intrinsics to be disabled") diff --git a/src/coreclr/inc/corinfoinstructionset.h b/src/coreclr/inc/corinfoinstructionset.h index e4910526fdd92d..af0d9c342ec918 100644 --- a/src/coreclr/inc/corinfoinstructionset.h +++ b/src/coreclr/inc/corinfoinstructionset.h @@ -23,38 +23,40 @@ enum CORINFO_InstructionSet InstructionSet_Crc32=4, InstructionSet_Dp=5, InstructionSet_Rdm=6, - InstructionSet_Sha1=7, - InstructionSet_Sha256=8, - InstructionSet_Atomics=9, - InstructionSet_Vector64=10, - InstructionSet_Vector128=11, - InstructionSet_VectorT=12, - InstructionSet_Dczva=13, - InstructionSet_Rcpc=14, - InstructionSet_VectorT128=15, - InstructionSet_Rcpc2=16, - InstructionSet_Sve=17, - InstructionSet_Sve2=18, - InstructionSet_Sha3=19, - InstructionSet_Sm4=20, - InstructionSet_SveAes=21, - InstructionSet_SveSha3=22, - InstructionSet_SveSm4=23, - InstructionSet_ArmBase_Arm64=24, - InstructionSet_AdvSimd_Arm64=25, - InstructionSet_Aes_Arm64=26, - InstructionSet_Crc32_Arm64=27, - InstructionSet_Dp_Arm64=28, - InstructionSet_Rdm_Arm64=29, - InstructionSet_Sha1_Arm64=30, - InstructionSet_Sha256_Arm64=31, - InstructionSet_Sve_Arm64=32, - InstructionSet_Sve2_Arm64=33, - InstructionSet_Sha3_Arm64=34, - InstructionSet_Sm4_Arm64=35, - InstructionSet_SveAes_Arm64=36, - InstructionSet_SveSha3_Arm64=37, - InstructionSet_SveSm4_Arm64=38, + InstructionSet_Fp16=7, + InstructionSet_Sha1=8, + InstructionSet_Sha256=9, + InstructionSet_Atomics=10, + InstructionSet_Vector64=11, + InstructionSet_Vector128=12, + InstructionSet_VectorT=13, + InstructionSet_Dczva=14, + InstructionSet_Rcpc=15, + InstructionSet_VectorT128=16, + InstructionSet_Rcpc2=17, + InstructionSet_Sve=18, + InstructionSet_Sve2=19, + InstructionSet_Sha3=20, + InstructionSet_Sm4=21, + InstructionSet_SveAes=22, + InstructionSet_SveSha3=23, + InstructionSet_SveSm4=24, + InstructionSet_ArmBase_Arm64=25, + InstructionSet_AdvSimd_Arm64=26, + InstructionSet_Aes_Arm64=27, + InstructionSet_Crc32_Arm64=28, + InstructionSet_Dp_Arm64=29, + InstructionSet_Rdm_Arm64=30, + InstructionSet_Fp16_Arm64=31, + InstructionSet_Sha1_Arm64=32, + InstructionSet_Sha256_Arm64=33, + InstructionSet_Sve_Arm64=34, + InstructionSet_Sve2_Arm64=35, + InstructionSet_Sha3_Arm64=36, + InstructionSet_Sm4_Arm64=37, + InstructionSet_SveAes_Arm64=38, + InstructionSet_SveSha3_Arm64=39, + InstructionSet_SveSm4_Arm64=40, #endif // TARGET_ARM64 #ifdef TARGET_RISCV64 InstructionSet_RiscV64Base=1, @@ -269,6 +271,8 @@ struct CORINFO_InstructionSetFlags AddInstructionSet(InstructionSet_Dp_Arm64); if (HasInstructionSet(InstructionSet_Rdm)) AddInstructionSet(InstructionSet_Rdm_Arm64); + if (HasInstructionSet(InstructionSet_Fp16)) + AddInstructionSet(InstructionSet_Fp16_Arm64); if (HasInstructionSet(InstructionSet_Sha1)) AddInstructionSet(InstructionSet_Sha1_Arm64); if (HasInstructionSet(InstructionSet_Sha256)) @@ -369,6 +373,10 @@ inline CORINFO_InstructionSetFlags EnsureInstructionSetFlagsAreValid(CORINFO_Ins resultflags.RemoveInstructionSet(InstructionSet_Rdm); if (resultflags.HasInstructionSet(InstructionSet_Rdm_Arm64) && !resultflags.HasInstructionSet(InstructionSet_Rdm)) resultflags.RemoveInstructionSet(InstructionSet_Rdm_Arm64); + if (resultflags.HasInstructionSet(InstructionSet_Fp16) && !resultflags.HasInstructionSet(InstructionSet_Fp16_Arm64)) + resultflags.RemoveInstructionSet(InstructionSet_Fp16); + if (resultflags.HasInstructionSet(InstructionSet_Fp16_Arm64) && !resultflags.HasInstructionSet(InstructionSet_Fp16)) + resultflags.RemoveInstructionSet(InstructionSet_Fp16_Arm64); if (resultflags.HasInstructionSet(InstructionSet_Sha1) && !resultflags.HasInstructionSet(InstructionSet_Sha1_Arm64)) resultflags.RemoveInstructionSet(InstructionSet_Sha1); if (resultflags.HasInstructionSet(InstructionSet_Sha1_Arm64) && !resultflags.HasInstructionSet(InstructionSet_Sha1)) @@ -415,6 +423,8 @@ inline CORINFO_InstructionSetFlags EnsureInstructionSetFlagsAreValid(CORINFO_Ins resultflags.RemoveInstructionSet(InstructionSet_Dp); if (resultflags.HasInstructionSet(InstructionSet_Rdm) && !resultflags.HasInstructionSet(InstructionSet_AdvSimd)) resultflags.RemoveInstructionSet(InstructionSet_Rdm); + if (resultflags.HasInstructionSet(InstructionSet_Fp16) && !resultflags.HasInstructionSet(InstructionSet_AdvSimd)) + resultflags.RemoveInstructionSet(InstructionSet_Fp16); if (resultflags.HasInstructionSet(InstructionSet_Sha1) && !resultflags.HasInstructionSet(InstructionSet_ArmBase)) resultflags.RemoveInstructionSet(InstructionSet_Sha1); if (resultflags.HasInstructionSet(InstructionSet_Sha256) && !resultflags.HasInstructionSet(InstructionSet_ArmBase)) @@ -693,6 +703,10 @@ inline const char *InstructionSetToString(CORINFO_InstructionSet instructionSet) return "Rdm"; case InstructionSet_Rdm_Arm64 : return "Rdm_Arm64"; + case InstructionSet_Fp16 : + return "Fp16"; + case InstructionSet_Fp16_Arm64 : + return "Fp16_Arm64"; case InstructionSet_Sha1 : return "Sha1"; case InstructionSet_Sha1_Arm64 : @@ -945,6 +959,7 @@ inline CORINFO_InstructionSet InstructionSetFromR2RInstructionSet(ReadyToRunInst case READYTORUN_INSTRUCTION_Crc32: return InstructionSet_Crc32; case READYTORUN_INSTRUCTION_Dp: return InstructionSet_Dp; case READYTORUN_INSTRUCTION_Rdm: return InstructionSet_Rdm; + case READYTORUN_INSTRUCTION_Fp16: return InstructionSet_Fp16; case READYTORUN_INSTRUCTION_Sha1: return InstructionSet_Sha1; case READYTORUN_INSTRUCTION_Sha256: return InstructionSet_Sha256; case READYTORUN_INSTRUCTION_Atomics: return InstructionSet_Atomics; diff --git a/src/coreclr/inc/jiteeversionguid.h b/src/coreclr/inc/jiteeversionguid.h index 2e2b7b91b04503..8094e7840117bb 100644 --- a/src/coreclr/inc/jiteeversionguid.h +++ b/src/coreclr/inc/jiteeversionguid.h @@ -37,11 +37,11 @@ #include -constexpr GUID JITEEVersionIdentifier = { /* a09d20fa-2c93-476b-86aa-7daaa3e52ddf */ - 0xa09d20fa, - 0x2c93, - 0x476b, - {0x86, 0xaa, 0x7d, 0xaa, 0xa3, 0xe5, 0x2d, 0xdf} +constexpr GUID JITEEVersionIdentifier = { /* 5de4a33b-7441-46a7-b3a3-704fa6cd0f31 */ + 0x5de4a33b, + 0x7441, + 0x46a7, + {0xb3, 0xa3, 0x70, 0x4f, 0xa6, 0xcd, 0x0f, 0x31} }; #endif // JIT_EE_VERSIONING_GUID_H diff --git a/src/coreclr/inc/readytoruninstructionset.h b/src/coreclr/inc/readytoruninstructionset.h index 6e491f85381089..99f61a1035b9c1 100644 --- a/src/coreclr/inc/readytoruninstructionset.h +++ b/src/coreclr/inc/readytoruninstructionset.h @@ -100,6 +100,7 @@ enum ReadyToRunInstructionSet READYTORUN_INSTRUCTION_SveSm4=90, READYTORUN_INSTRUCTION_WasmBase=91, READYTORUN_INSTRUCTION_PackedSimd=92, + READYTORUN_INSTRUCTION_Fp16=93, }; diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index 1fc303b4162b3d..07518288ced19e 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -629,6 +629,7 @@ class CodeGen final : public CodeGenInterface #if defined(TARGET_ARM64) void genArm64EmitterUnitTestsGeneral(); void genArm64EmitterUnitTestsAdvSimd(); + void genArm64EmitterUnitTestsFp16(); void genArm64EmitterUnitTestsSve(); void genArm64EmitterUnitTestsPac(); #endif diff --git a/src/coreclr/jit/codegenarm64test.cpp b/src/coreclr/jit/codegenarm64test.cpp index 0bf47c1aa50a0a..c4ff7f7566b716 100644 --- a/src/coreclr/jit/codegenarm64test.cpp +++ b/src/coreclr/jit/codegenarm64test.cpp @@ -9173,4 +9173,66 @@ void CodeGen::genArm64EmitterUnitTestsPac() theEmitter->emitIns_R_R(INS_pacia, EA_8BYTE, REG_R27, REG_SP); // PACIA , theEmitter->emitIns_R_R(INS_pacib, EA_8BYTE, REG_R28, REG_SP); // PACIB , } + +//----------------------------------------------------------------------------- +// genArm64EmitterUnitTestsFp16: Emit the scalar half-precision (FEAT_FP16) +// floating-point instructions used to accelerate `System.Half`, so their +// encodings and disassembly can be verified. +// +void CodeGen::genArm64EmitterUnitTestsFp16() +{ + emitter* theEmitter = GetEmitter(); + + genDefineTempLabel(genCreateTempLabel()); + + // IF_DV_3D: scalar half-precision, two source registers + theEmitter->emitIns_R_R_R(INS_fadd, EA_2BYTE, REG_V0, REG_V1, REG_V2); // FADD Hd, Hn, Hm + theEmitter->emitIns_R_R_R(INS_fsub, EA_2BYTE, REG_V3, REG_V4, REG_V5); // FSUB Hd, Hn, Hm + theEmitter->emitIns_R_R_R(INS_fmul, EA_2BYTE, REG_V6, REG_V7, REG_V8); // FMUL Hd, Hn, Hm + theEmitter->emitIns_R_R_R(INS_fdiv, EA_2BYTE, REG_V9, REG_V10, REG_V11); // FDIV Hd, Hn, Hm + theEmitter->emitIns_R_R_R(INS_fnmul, EA_2BYTE, REG_V12, REG_V13, REG_V14); // FNMUL Hd, Hn, Hm + theEmitter->emitIns_R_R_R(INS_fmax, EA_2BYTE, REG_V15, REG_V16, REG_V17); // FMAX Hd, Hn, Hm + theEmitter->emitIns_R_R_R(INS_fmin, EA_2BYTE, REG_V18, REG_V19, REG_V20); // FMIN Hd, Hn, Hm + theEmitter->emitIns_R_R_R(INS_fmaxnm, EA_2BYTE, REG_V21, REG_V22, REG_V23); // FMAXNM Hd, Hn, Hm + theEmitter->emitIns_R_R_R(INS_fminnm, EA_2BYTE, REG_V24, REG_V25, REG_V26); // FMINNM Hd, Hn, Hm + theEmitter->emitIns_R_R_R(INS_fabd, EA_2BYTE, REG_V27, REG_V28, REG_V29); // FABD Hd, Hn, Hm + + // IF_DV_2G: scalar half-precision, one source register + theEmitter->emitIns_R_R(INS_fsqrt, EA_2BYTE, REG_V0, REG_V1); // FSQRT Hd, Hn + theEmitter->emitIns_R_R(INS_frinta, EA_2BYTE, REG_V2, REG_V3); // FRINTA Hd, Hn + theEmitter->emitIns_R_R(INS_frinti, EA_2BYTE, REG_V4, REG_V5); // FRINTI Hd, Hn + theEmitter->emitIns_R_R(INS_frintm, EA_2BYTE, REG_V6, REG_V7); // FRINTM Hd, Hn + theEmitter->emitIns_R_R(INS_frintn, EA_2BYTE, REG_V8, REG_V9); // FRINTN Hd, Hn + theEmitter->emitIns_R_R(INS_frintp, EA_2BYTE, REG_V10, REG_V11); // FRINTP Hd, Hn + theEmitter->emitIns_R_R(INS_frintx, EA_2BYTE, REG_V12, REG_V13); // FRINTX Hd, Hn + theEmitter->emitIns_R_R(INS_frintz, EA_2BYTE, REG_V14, REG_V15); // FRINTZ Hd, Hn + theEmitter->emitIns_R_R(INS_frecpe, EA_2BYTE, REG_V16, REG_V17); // FRECPE Hd, Hn + theEmitter->emitIns_R_R(INS_frsqrte, EA_2BYTE, REG_V18, REG_V19); // FRSQRTE Hd, Hn + + // IF_DV_2K: scalar half-precision compare + theEmitter->emitIns_R_R(INS_fcmp, EA_2BYTE, REG_V0, REG_V1); // FCMP Hn, Hm + theEmitter->emitIns_R_R(INS_fcmpe, EA_2BYTE, REG_V2, REG_V3); // FCMPE Hn, Hm + + // IF_DV_4A: scalar half-precision fused multiply-add + theEmitter->emitIns_R_R_R_R(INS_fmadd, EA_2BYTE, REG_V0, REG_V1, REG_V2, REG_V3); // FMADD Hd, Hn, Hm, Ha + theEmitter->emitIns_R_R_R_R(INS_fnmadd, EA_2BYTE, REG_V4, REG_V5, REG_V6, REG_V7); // FNMADD Hd, Hn, Hm, Ha + + // IF_DV_2J: convert between half and single/double + theEmitter->emitIns_R_R(INS_fcvt, EA_4BYTE, REG_V0, REG_V1, INS_OPTS_H_TO_S); // FCVT Sd, Hn + theEmitter->emitIns_R_R(INS_fcvt, EA_8BYTE, REG_V2, REG_V3, INS_OPTS_H_TO_D); // FCVT Dd, Hn + theEmitter->emitIns_R_R(INS_fcvt, EA_2BYTE, REG_V4, REG_V5, INS_OPTS_S_TO_H); // FCVT Hd, Sn + theEmitter->emitIns_R_R(INS_fcvt, EA_2BYTE, REG_V6, REG_V7, INS_OPTS_D_TO_H); // FCVT Hd, Dn + + // IF_DV_2H: convert half to integer (truncating toward zero) + theEmitter->emitIns_R_R(INS_fcvtzs, EA_4BYTE, REG_R0, REG_V1, INS_OPTS_H_TO_4BYTE); // FCVTZS Wd, Hn + theEmitter->emitIns_R_R(INS_fcvtzs, EA_8BYTE, REG_R2, REG_V3, INS_OPTS_H_TO_8BYTE); // FCVTZS Xd, Hn + theEmitter->emitIns_R_R(INS_fcvtzu, EA_4BYTE, REG_R4, REG_V5, INS_OPTS_H_TO_4BYTE); // FCVTZU Wd, Hn + theEmitter->emitIns_R_R(INS_fcvtzu, EA_8BYTE, REG_R6, REG_V7, INS_OPTS_H_TO_8BYTE); // FCVTZU Xd, Hn + + // IF_DV_2I: convert integer to half + theEmitter->emitIns_R_R(INS_scvtf, EA_2BYTE, REG_V0, REG_R1, INS_OPTS_4BYTE_TO_H); // SCVTF Hd, Wn + theEmitter->emitIns_R_R(INS_scvtf, EA_2BYTE, REG_V2, REG_R3, INS_OPTS_8BYTE_TO_H); // SCVTF Hd, Xn + theEmitter->emitIns_R_R(INS_ucvtf, EA_2BYTE, REG_V4, REG_R5, INS_OPTS_4BYTE_TO_H); // UCVTF Hd, Wn + theEmitter->emitIns_R_R(INS_ucvtf, EA_2BYTE, REG_V6, REG_R7, INS_OPTS_8BYTE_TO_H); // UCVTF Hd, Xn +} #endif // defined(TARGET_ARM64) && defined(DEBUG) diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index 49f67585225024..0f7d01d9b8d34d 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -2788,6 +2788,10 @@ void CodeGen::genEmitterUnitTests() { genArm64EmitterUnitTestsAdvSimd(); } + if (unitTestSectionAll || (strstr(unitTestSection, "fp16") != nullptr)) + { + genArm64EmitterUnitTestsFp16(); + } if (unitTestSectionAll || (strstr(unitTestSection, "sve") != nullptr)) { genArm64EmitterUnitTestsSve(); diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index 105ed99fe35427..c11a22e6b66422 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -6080,6 +6080,11 @@ int Compiler::compCompileAfterInit(CORINFO_MODULE_HANDLE classPtr, instructionSetFlags.AddInstructionSet(InstructionSet_Rdm); } + if (JitConfig.EnableArm64Fp16() != 0) + { + instructionSetFlags.AddInstructionSet(InstructionSet_Fp16); + } + if (JitConfig.EnableArm64Sha1() != 0) { instructionSetFlags.AddInstructionSet(InstructionSet_Sha1); diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 5e7ef8beb9c7ee..ce8f409d7c9986 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -5322,6 +5322,13 @@ class Compiler NamedIntrinsic lookupPrimitiveFloatNamedIntrinsic(CORINFO_METHOD_HANDLE method, const char* methodName); NamedIntrinsic lookupPrimitiveIntNamedIntrinsic(CORINFO_METHOD_HANDLE method, const char* methodName); + NamedIntrinsic lookupHalfNamedIntrinsic(CORINFO_METHOD_HANDLE method, const char* methodName); +#if defined(FEATURE_HW_INTRINSICS) && (defined(TARGET_XARCH) || defined(TARGET_ARM64)) + NamedIntrinsic lookupHalfIntrinsic(NamedIntrinsic ni); +#endif // FEATURE_HW_INTRINSICS && (TARGET_XARCH || TARGET_ARM64) +#if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_XARCH) + int lookupHalfRoundingMode(NamedIntrinsic ni); +#endif // FEATURE_HW_INTRINSICS && TARGET_XARCH GenTree* impUnsupportedNamedIntrinsic(unsigned helper, CORINFO_METHOD_HANDLE method, CORINFO_SIG_INFO* sig, diff --git a/src/coreclr/jit/emit.h b/src/coreclr/jit/emit.h index 780f06bd1b138c..d7b306be853966 100644 --- a/src/coreclr/jit/emit.h +++ b/src/coreclr/jit/emit.h @@ -2069,18 +2069,20 @@ class emitter #define PERFSCORE_THROUGHPUT_ZERO 0.0f // Only used for pseudo-instructions that don't generate code -#define PERFSCORE_THROUGHPUT_9X (1.0f / 9.0f) -#define PERFSCORE_THROUGHPUT_6X (1.0f / 6.0f) // Hextuple issue -#define PERFSCORE_THROUGHPUT_5X 0.20f // Pentuple issue -#define PERFSCORE_THROUGHPUT_4X 0.25f // Quad issue -#define PERFSCORE_THROUGHPUT_3X (1.0f / 3.0f) // Three issue -#define PERFSCORE_THROUGHPUT_2X 0.5f // Dual issue +#define PERFSCORE_THROUGHPUT_9X (1.0f / 9.0f) +#define PERFSCORE_THROUGHPUT_6X (1.0f / 6.0f) // Hextuple issue +#define PERFSCORE_THROUGHPUT_5X 0.20f // Pentuple issue +#define PERFSCORE_THROUGHPUT_4X 0.25f // Quad issue +#define PERFSCORE_THROUGHPUT_3X (1.0f / 3.0f) // Three issue +#define PERFSCORE_THROUGHPUT_2X 0.5f // Dual issue +#define PERFSCORE_THROUGHPUT_1P5X 0.67f // Dual issue #define PERFSCORE_THROUGHPUT_1C 1.0f // Single Issue #define PERFSCORE_THROUGHPUT_2C 2.0f // slower - 2 cycles #define PERFSCORE_THROUGHPUT_3C 3.0f // slower - 3 cycles #define PERFSCORE_THROUGHPUT_4C 4.0f // slower - 4 cycles +#define PERFSCORE_THROUGHPUT_4P5C 4.5f // slower - 4.5 cycles #define PERFSCORE_THROUGHPUT_5C 5.0f // slower - 5 cycles #define PERFSCORE_THROUGHPUT_6C 6.0f // slower - 6 cycles #define PERFSCORE_THROUGHPUT_7C 7.0f // slower - 7 cycles diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 49a836ed881ae5..eb3730e46cc5c0 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -725,7 +725,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) case IF_DV_2G: // DV_2G .........X...... ......nnnnnddddd Vd Vn (fmov, fcvtXX - register) case IF_DV_2K: // DV_2K .........X.mmmmm ......nnnnn..... Vn Vm (fcmp) assert(insOptsNone(id->idInsOpt())); - assert(isValidVectorElemsizeFloat(id->idOpSize())); + assert(isValidVectorElemsizeFloat(id->idOpSize()) || (id->idOpSize() == EA_2BYTE)); assert(isVectorRegister(id->idReg1())); assert(isVectorRegister(id->idReg2())); break; @@ -735,7 +735,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) dstsize = optGetDstsize(id->idInsOpt()); srcsize = optGetSrcsize(id->idInsOpt()); assert(isValidGeneralDatasize(dstsize)); - assert(isValidVectorElemsizeFloat(srcsize)); + assert(isValidVectorElemsizeFloat(srcsize) || (srcsize == EA_2BYTE)); assert(dstsize == id->idOpSize()); assert(isGeneralRegister(id->idReg1())); assert(isVectorRegister(id->idReg2())); @@ -746,7 +746,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) dstsize = optGetDstsize(id->idInsOpt()); srcsize = optGetSrcsize(id->idInsOpt()); assert(isValidGeneralDatasize(srcsize)); - assert(isValidVectorElemsizeFloat(dstsize)); + assert(isValidVectorElemsizeFloat(dstsize) || (dstsize == EA_2BYTE)); assert(dstsize == id->idOpSize()); assert(isVectorRegister(id->idReg1())); assert(isGeneralRegister(id->idReg2())); @@ -958,7 +958,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) break; case IF_DV_4A: // DV_4A .........X.mmmmm .aaaaannnnnddddd Rd Rn Rm Ra (scalar) - assert(isValidGeneralDatasize(id->idOpSize())); + assert(isValidScalarDatasize(id->idOpSize())); assert(isVectorRegister(id->idReg1())); assert(isVectorRegister(id->idReg2())); assert(isVectorRegister(id->idReg3())); @@ -3613,6 +3613,7 @@ emitter::code_t emitter::emitInsCode(instruction ins, insFormat fmt) case INS_OPTS_8BYTE_TO_D: case INS_OPTS_S_TO_D: case INS_OPTS_H_TO_D: + case INS_OPTS_H_TO_8BYTE: return EA_8BYTE; @@ -3622,11 +3623,14 @@ emitter::code_t emitter::emitInsCode(instruction ins, insFormat fmt) case INS_OPTS_8BYTE_TO_S: case INS_OPTS_D_TO_S: case INS_OPTS_H_TO_S: + case INS_OPTS_H_TO_4BYTE: return EA_4BYTE; case INS_OPTS_S_TO_H: case INS_OPTS_D_TO_H: + case INS_OPTS_4BYTE_TO_H: + case INS_OPTS_8BYTE_TO_H: return EA_2BYTE; @@ -3647,6 +3651,7 @@ emitter::code_t emitter::emitInsCode(instruction ins, insFormat fmt) case INS_OPTS_8BYTE_TO_S: case INS_OPTS_D_TO_S: case INS_OPTS_D_TO_H: + case INS_OPTS_8BYTE_TO_H: return EA_8BYTE; @@ -3656,11 +3661,14 @@ emitter::code_t emitter::emitInsCode(instruction ins, insFormat fmt) case INS_OPTS_4BYTE_TO_D: case INS_OPTS_S_TO_D: case INS_OPTS_S_TO_H: + case INS_OPTS_4BYTE_TO_H: return EA_4BYTE; case INS_OPTS_H_TO_S: case INS_OPTS_H_TO_D: + case INS_OPTS_H_TO_4BYTE: + case INS_OPTS_H_TO_8BYTE: return EA_2BYTE; @@ -4751,7 +4759,7 @@ void emitter::emitIns_R_R(instruction ins, case INS_fcmp: case INS_fcmpe: assert(insOptsNone(opt)); - assert(isValidVectorElemsizeFloat(size)); + assert(isValidVectorElemsizeFloat(size) || (size == EA_2BYTE)); assert(isVectorRegister(reg1)); assert(isVectorRegister(reg2)); fmt = IF_DV_2K; @@ -4873,7 +4881,7 @@ void emitter::emitIns_R_R(instruction ins, { assert(isGeneralRegister(reg2)); assert(insOptsConvertIntToFloat(opt)); - assert(isValidVectorElemsizeFloat(size)); + assert(isValidVectorElemsizeFloat(size) || (size == EA_2BYTE)); fmt = IF_DV_2I; } } @@ -4905,7 +4913,7 @@ void emitter::emitIns_R_R(instruction ins, { // Scalar operation assert(insOptsNone(opt)); - assert(isValidVectorElemsizeFloat(size)); + assert(isValidVectorElemsizeFloat(size) || (size == EA_2BYTE)); assert(isVectorRegister(reg1)); assert(isVectorRegister(reg2)); fmt = IF_DV_2G; @@ -10118,6 +10126,10 @@ void emitter::emitIns_Call(const EmitCallParams& params) { return 0x00400000; // set the bit at location 22 } + else if (size == EA_2BYTE) + { + return 0x00C00000; // set the bits at location 23 and 22 (ftype=11, half-precision) + } assert(size == EA_4BYTE); return 0x00000000; } @@ -10327,6 +10339,16 @@ void emitter::emitIns_Call(const EmitCallParams& params) result = 0x80400000; // sf=1, type=01 break; + case INS_OPTS_H_TO_4BYTE: // Half to INT32 + assert(fmt == IF_DV_2H); + result = 0x00C00000; // sf=0, type=11 + break; + + case INS_OPTS_H_TO_8BYTE: // Half to INT64 + assert(fmt == IF_DV_2H); + result = 0x80C00000; // sf=1, type=11 + break; + case INS_OPTS_4BYTE_TO_S: // INT32 to Single assert(fmt == IF_DV_2I); result = 0x00000000; // sf=0, type=00 @@ -10347,6 +10369,16 @@ void emitter::emitIns_Call(const EmitCallParams& params) result = 0x80400000; // sf=1, type=01 break; + case INS_OPTS_4BYTE_TO_H: // INT32 to Half + assert(fmt == IF_DV_2I); + result = 0x00C00000; // sf=0, type=11 + break; + + case INS_OPTS_8BYTE_TO_H: // INT64 to Half + assert(fmt == IF_DV_2I); + result = 0x80C00000; // sf=1, type=11 + break; + default: assert(!"Invalid 'conversion' value"); break; @@ -16472,8 +16504,8 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins } else { - // S-form - assert(id->idOpSize() == EA_4BYTE); + // S-form or H-form + assert((id->idOpSize() == EA_4BYTE) || (id->idOpSize() == EA_2BYTE)); result.insThroughput = PERFSCORE_THROUGHPUT_9C; result.insLatency = PERFSCORE_LATENCY_12C; } @@ -16633,8 +16665,8 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins } else { - // S-form - assert(id->idOpSize() == EA_4BYTE); + // S-form or H-form + assert((id->idOpSize() == EA_4BYTE) || (id->idOpSize() == EA_2BYTE)); result.insThroughput = PERFSCORE_THROUGHPUT_3C; result.insLatency = PERFSCORE_LATENCY_10C; } diff --git a/src/coreclr/jit/emitarm64.h b/src/coreclr/jit/emitarm64.h index f2a4c9e7f0b652..38b4505d32c118 100644 --- a/src/coreclr/jit/emitarm64.h +++ b/src/coreclr/jit/emitarm64.h @@ -1113,7 +1113,7 @@ inline static bool isValidGeneralDatasize(emitAttr size) inline static bool isValidScalarDatasize(emitAttr size) { - return (size == EA_8BYTE) || (size == EA_4BYTE); + return (size == EA_8BYTE) || (size == EA_4BYTE) || (size == EA_2BYTE); } inline static bool isValidScalableDatasize(emitAttr size) @@ -1331,12 +1331,12 @@ inline static bool insOptsConvertFloatToFloat(insOpts opt) inline static bool insOptsConvertFloatToInt(insOpts opt) { - return ((opt >= INS_OPTS_S_TO_4BYTE) && (opt <= INS_OPTS_D_TO_8BYTE)); + return ((opt >= INS_OPTS_S_TO_4BYTE) && (opt <= INS_OPTS_H_TO_8BYTE)); } inline static bool insOptsConvertIntToFloat(insOpts opt) { - return ((opt >= INS_OPTS_4BYTE_TO_S) && (opt <= INS_OPTS_8BYTE_TO_D)); + return ((opt >= INS_OPTS_4BYTE_TO_S) && (opt <= INS_OPTS_8BYTE_TO_H)); } inline static bool insOptsScalable(insOpts opt) diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index 2d8b9328edcbb8..7bf1d19bcd56b3 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -129,7 +129,8 @@ bool emitter::Is3OpRmwInstruction(instruction ins) return ((ins >= FIRST_FMA_INSTRUCTION) && (ins <= LAST_FMA_INSTRUCTION)) || (IsAVXVNNIFamilyInstruction(ins)) || ((ins >= FIRST_AVX512BMM_INSTRUCTION) && (ins <= LAST_AVX512BMM_INSTRUCTION)) || - ((ins >= FIRST_AVXIFMA_INSTRUCTION) && (ins <= LAST_AVXIFMA_INSTRUCTION)); + ((ins >= FIRST_AVXIFMA_INSTRUCTION) && (ins <= LAST_AVXIFMA_INSTRUCTION)) || + ((ins >= FIRST_AVX10V1_FMA_INSTR) && (ins <= LAST_AVX10V1_FMA_INSTR)); } } } @@ -3185,7 +3186,7 @@ emitter::code_t emitter::emitExtractEvexPrefix(instruction ins, code_t& code) co // 1. An escape byte 0F (For isa before AVX10.2) // 2. A map number from 0 to 7 (For AVX10.2 and above) leadingBytes = check; - assert((leadingBytes == 0x0F) || ((m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX10v2) || + assert((leadingBytes == 0x0F) || ((m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX10v1) || (m_compiler->compIsaSupportedDebugOnly(InstructionSet_APX))) && (leadingBytes >= 0x00) && (leadingBytes <= 0x07))); @@ -3212,7 +3213,7 @@ emitter::code_t emitter::emitExtractEvexPrefix(instruction ins, code_t& code) co // 0x0000RM11. leadingBytes = (code >> 16) & 0xFF; assert(leadingBytes == 0x0F || - ((m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX10v2) || + ((m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX10v1) || m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX512BMM)) && leadingBytes >= 0x00 && leadingBytes <= 0x07) || (IsApxExtendedEvexInstruction(ins) && leadingBytes == 0)); @@ -3268,14 +3269,15 @@ emitter::code_t emitter::emitExtractEvexPrefix(instruction ins, code_t& code) co case 0x05: { - assert(m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX10v2)); + assert(m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX10v1)); evexPrefix |= (0x05 << 16); break; } case 0x06: { - assert(m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX512BMM)); + assert(m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX10v1) || + m_compiler->compIsaSupportedDebugOnly(InstructionSet_AVX512BMM)); evexPrefix |= (0x6 << 16); break; } @@ -3872,7 +3874,8 @@ unsigned emitter::emitGetAdjustedSize(instrDesc* id, code_t code) const } emitAttr attr = id->idOpSize(); - if ((attr == EA_2BYTE) && (ins != INS_movzx) && (ins != INS_movsx) && !TakesApxExtendedEvexPrefix(id)) + if ((attr == EA_2BYTE) && (ins != INS_movzx) && (ins != INS_movsx) && !IsSimdInstruction(ins) && + !TakesApxExtendedEvexPrefix(id)) { // Most 16-bit operand instructions will need a 0x66 prefix. prefixAdjustedSize++; @@ -3891,7 +3894,7 @@ unsigned emitter::emitGetAdjustedSize(instrDesc* id, code_t code) const emitAttr attr = id->idOpSize(); - if ((attr == EA_2BYTE) && (ins != INS_movzx) && (ins != INS_movsx)) + if ((attr == EA_2BYTE) && (ins != INS_movzx) && (ins != INS_movsx) && !IsSimdInstruction(ins)) { // Most 16-bit operand instructions will need a 0x66 prefix. adjustedSize++; @@ -5503,7 +5506,7 @@ UNATIVE_OFFSET emitter::emitInsSizeAM(instrDesc* id, code_t code) assert((attrSize == EA_4BYTE) || (attrSize == EA_PTRSIZE) // Only for x64 || (attrSize == EA_16BYTE) || (attrSize == EA_32BYTE) || (attrSize == EA_64BYTE) // only for x64 - || (ins == INS_movzx) || (ins == INS_movsx) || + || (ins == INS_movzx) || (ins == INS_movsx) || (ins == INS_vmovsh) || (ins == INS_cmpxchg) // kmov instructions reach this path with EA_8BYTE size, even on x86 || IsKMOVInstruction(ins) @@ -7542,6 +7545,7 @@ bool emitter::IsMovInstruction(instruction ins) case INS_movq: case INS_movsd_simd: case INS_movss: + case INS_vmovsh: case INS_movsx: case INS_movupd: case INS_movups: @@ -7702,6 +7706,13 @@ bool emitter::HasSideEffect(instruction ins, emitAttr size) break; } + case INS_vmovsh: + { + // Clears the upper bits + hasSideEffect = true; + break; + } + case INS_movsx: case INS_movzx: { @@ -7978,6 +7989,7 @@ bool emitter::emitIns_Mov( case INS_vmovdqu64: case INS_movsd_simd: case INS_movss: + case INS_vmovsh: case INS_movupd: case INS_movups: { @@ -11974,6 +11986,11 @@ const char* emitter::emitRegName(regNumber reg, emitAttr attr, bool varName) con case EA_2BYTE: { + if (IsXMMReg(reg)) + { + return emitXMMregName(reg); + } + #if defined(TARGET_AMD64) if (reg > REG_RDI) { @@ -20753,6 +20770,89 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins break; } + case INS_vaddsh: + case INS_vsubsh: + case INS_vmulsh: + case INS_vfmadd213sh: + case INS_vmaxsh: + case INS_vminsh: + case INS_vcvtsh2ss: + { + insLatency = PERFSCORE_LATENCY_4C; + insThroughput = PERFSCORE_THROUGHPUT_2X; + break; + } + + case INS_vdivsh: + { + insLatency = PERFSCORE_LATENCY_14C; + insThroughput = PERFSCORE_THROUGHPUT_4C; + break; + } + + case INS_vsqrtsh: + { + insLatency = PERFSCORE_LATENCY_14C; + insThroughput = PERFSCORE_THROUGHPUT_4P5C; + break; + } + + case INS_vrsqrtsh: + case INS_vcomish: + case INS_vucomish: + case INS_vrcpsh: + { + insLatency = PERFSCORE_LATENCY_4C; + insThroughput = PERFSCORE_THROUGHPUT_1C; + break; + } + + case INS_vrndscalesh: + { + insLatency = PERFSCORE_LATENCY_8C; + insThroughput = PERFSCORE_THROUGHPUT_1C; + break; + } + + case INS_vcvtss2sh: + { + insLatency = PERFSCORE_LATENCY_6C; + insThroughput = PERFSCORE_THROUGHPUT_1P5X; + break; + } + + case INS_vcvtsd2sh: + { + insLatency = PERFSCORE_LATENCY_7C; + insThroughput = PERFSCORE_THROUGHPUT_1C; + break; + } + + case INS_vcvtsh2sd: + { + insLatency = PERFSCORE_LATENCY_10C; + insThroughput = PERFSCORE_THROUGHPUT_1C; + break; + } + + case INS_vcvtsi2sh32: + case INS_vcvtsi2sh64: + case INS_vcvtsh2si32: + case INS_vcvtsh2si64: + case INS_vcvtusi2sh32: + case INS_vcvtusi2sh64: + case INS_vcvtsh2usi32: + case INS_vcvtsh2usi64: + case INS_vcvttsh2si32: + case INS_vcvttsh2si64: + case INS_vcvttsh2usi32: + case INS_vcvttsh2usi64: + { + insLatency = PERFSCORE_LATENCY_7C; + insThroughput = PERFSCORE_THROUGHPUT_1C; + break; + } + case INS_vpmovdb: case INS_vpmovdw: case INS_vpmovqb: diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 9ad468aa31c7b2..b0c934b3d833cd 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -4463,6 +4463,7 @@ unsigned Compiler::gtSetMultiOpOrder(GenTreeMultiOp* multiOp) case NI_AVX_Divide: case NI_AVX512_Divide: case NI_AVX512_DivideScalar: + case NI_AVX10v1_DivideScalar: { costEx = (simdBaseType == TYP_DOUBLE) ? 14 : 11; break; @@ -4513,6 +4514,7 @@ unsigned Compiler::gtSetMultiOpOrder(GenTreeMultiOp* multiOp) case NI_AVX_Sqrt: case NI_AVX512_Sqrt: case NI_AVX512_SqrtScalar: + case NI_AVX10v1_SqrtScalar: { costEx = (simdBaseType == TYP_DOUBLE) ? 16 : 12; break; @@ -30705,6 +30707,7 @@ bool GenTreeHWIntrinsic::OperIsEmbRoundingEnabled() const case NI_AVX512_FusedMultiplySubtractNegated: case NI_AVX512_FusedMultiplySubtractNegatedScalar: case NI_AVX512_FusedMultiplySubtractScalar: + case NI_AVX10v1_FusedMultiplyAddScalar: { return numArgs == 4; } @@ -30722,6 +30725,13 @@ bool GenTreeHWIntrinsic::OperIsEmbRoundingEnabled() const case NI_AVX512_X64_ConvertScalarToVector128Double: case NI_AVX512_X64_ConvertScalarToVector128Single: #endif // TARGET_AMD64 + case NI_AVX10v1_AddScalar: + case NI_AVX10v1_DivideScalar: + case NI_AVX10v1_MultiplyScalar: + case NI_AVX10v1_SubtractScalar: + case NI_AVX10v1_ConvertScalarToVector128Half: + case NI_AVX10v1_ConvertScalarToVector128Single: + case NI_AVX10v1_ConvertScalarToVector128Double: { return numArgs == 3; } diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index 421b6adffd2d27..74dd91842ea60e 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -932,7 +932,7 @@ static const HWIntrinsicIsaRange hwintrinsicIsaRangeArray[] = { { FIRST_NI_AVX512, LAST_NI_AVX512 }, // AVX512 { FIRST_NI_AVX512v2, LAST_NI_AVX512v2 }, // AVX512v2 { FIRST_NI_AVX512v3, LAST_NI_AVX512v3 }, // AVX512v3 - { NI_Illegal, NI_Illegal }, // AVX10v1 + { FIRST_NI_AVX10v1, LAST_NI_AVX10v1 }, // AVX10v1 { FIRST_NI_AVX10v2, LAST_NI_AVX10v2 }, // AVX10v2 { NI_Illegal, NI_Illegal }, // APX { FIRST_NI_AES, LAST_NI_AES }, // AES @@ -980,6 +980,7 @@ static const HWIntrinsicIsaRange hwintrinsicIsaRangeArray[] = { { FIRST_NI_Crc32, LAST_NI_Crc32 }, // Crc32 { FIRST_NI_Dp, LAST_NI_Dp }, // Dp { FIRST_NI_Rdm, LAST_NI_Rdm }, // Rdm + { FIRST_NI_Fp16, LAST_NI_Fp16 }, // Fp16 { FIRST_NI_Sha1, LAST_NI_Sha1 }, // Sha1 { FIRST_NI_Sha256, LAST_NI_Sha256 }, // Sha256 { NI_Illegal, NI_Illegal }, // Atomics @@ -1003,6 +1004,7 @@ static const HWIntrinsicIsaRange hwintrinsicIsaRangeArray[] = { { FIRST_NI_Crc32_Arm64, LAST_NI_Crc32_Arm64 }, // Crc32_Arm64 { NI_Illegal, NI_Illegal }, // Dp_Arm64 { FIRST_NI_Rdm_Arm64, LAST_NI_Rdm_Arm64 }, // Rdm_Arm64 + { NI_Illegal, NI_Illegal }, // Fp16_Arm64 { NI_Illegal, NI_Illegal }, // Sha1_Arm64 { NI_Illegal, NI_Illegal }, // Sha256_Arm64 { NI_Illegal, NI_Illegal }, // Sve_Arm64 diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index 85d24554e9336c..fe533351076eb4 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -1293,6 +1293,97 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) } break; + case NI_ArmBase_ConvertToSingle: + GetEmitter()->emitIns_R_R(ins, EA_4BYTE, targetReg, op1Reg, INS_OPTS_H_TO_S); + break; + + case NI_ArmBase_ConvertToDouble: + GetEmitter()->emitIns_R_R(ins, EA_8BYTE, targetReg, op1Reg, INS_OPTS_H_TO_D); + break; + + case NI_ArmBase_ConvertToHalf: + { + // Baseline FCVT precision conversion; the source (base) type selects the width. + assert((intrin.baseType == TYP_FLOAT) || (intrin.baseType == TYP_DOUBLE)); + insOpts cvtOption = (intrin.baseType == TYP_FLOAT) ? INS_OPTS_S_TO_H : INS_OPTS_D_TO_H; + GetEmitter()->emitIns_R_R(ins, EA_2BYTE, targetReg, op1Reg, cvtOption); + break; + } + + case NI_Fp16_ConvertToInt32: + case NI_Fp16_ConvertToUInt32: + GetEmitter()->emitIns_R_R(ins, EA_4BYTE, targetReg, op1Reg, INS_OPTS_H_TO_4BYTE); + break; + + case NI_Fp16_ConvertToInt64: + case NI_Fp16_ConvertToUInt64: + GetEmitter()->emitIns_R_R(ins, EA_8BYTE, targetReg, op1Reg, INS_OPTS_H_TO_8BYTE); + break; + + case NI_Fp16_ConvertToHalf: + { + // The instruction is already selected by the source (base) type; only the + // conversion option needs to distinguish the source width and signedness. + insOpts cvtOption = INS_OPTS_NONE; + + switch (intrin.baseType) + { + case TYP_INT: + case TYP_UINT: + cvtOption = INS_OPTS_4BYTE_TO_H; + break; + case TYP_LONG: + case TYP_ULONG: + cvtOption = INS_OPTS_8BYTE_TO_H; + break; + default: + unreached(); + } + + GetEmitter()->emitIns_R_R(ins, EA_2BYTE, targetReg, op1Reg, cvtOption); + break; + } + + case NI_Fp16_CompareEqual: + case NI_Fp16_CompareGreaterThan: + case NI_Fp16_CompareGreaterThanOrEqual: + case NI_Fp16_CompareLessThan: + case NI_Fp16_CompareLessThanOrEqual: + case NI_Fp16_CompareNotEqual: + { + insCond cond = INS_COND_EQ; + + switch (intrin.id) + { + case NI_Fp16_CompareEqual: + cond = INS_COND_EQ; + break; + case NI_Fp16_CompareGreaterThan: + cond = INS_COND_GT; + break; + case NI_Fp16_CompareGreaterThanOrEqual: + cond = INS_COND_GE; + break; + case NI_Fp16_CompareLessThan: + // 'mi' rather than 'lt' so an unordered (NaN) comparison yields false. + cond = INS_COND_MI; + break; + case NI_Fp16_CompareLessThanOrEqual: + // 'ls' rather than 'le' so an unordered (NaN) comparison yields false. + cond = INS_COND_LS; + break; + case NI_Fp16_CompareNotEqual: + cond = INS_COND_NE; + break; + default: + unreached(); + } + + GetEmitter()->emitIns_R_R(INS_fcmp, EA_2BYTE, op1Reg, op2Reg); + GetEmitter()->emitIns_R_COND(INS_cset, EA_4BYTE, targetReg, cond); + break; + } + case NI_Crc32_ComputeCrc32: case NI_Crc32_ComputeCrc32C: case NI_Crc32_Arm64_ComputeCrc32: diff --git a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp index 3e8c533ff6c825..45ffb0fa06eb4d 100644 --- a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp @@ -1036,6 +1036,7 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) case InstructionSet_AVX512: case InstructionSet_AVX512_X64: case InstructionSet_AVX512v2: + case InstructionSet_AVX10v1: case InstructionSet_AVX10v2: case InstructionSet_AVX10v2_X64: case InstructionSet_AVXVNNIINT: @@ -1890,6 +1891,7 @@ void CodeGen::genNonTableDrivenHWIntrinsicsJumpTableFallback(GenTreeHWIntrinsic* case NI_AVX512_FusedMultiplyAdd: case NI_AVX512_FusedMultiplyAddScalar: + case NI_AVX10v1_FusedMultiplyAddScalar: case NI_AVX512_FusedMultiplyAddNegated: case NI_AVX512_FusedMultiplyAddNegatedScalar: case NI_AVX512_FusedMultiplyAddSubtract: @@ -3701,6 +3703,35 @@ void CodeGen::genAvxFamilyIntrinsic(GenTreeHWIntrinsic* node, insOpts instOption break; } + case NI_AVX10v1_ConvertToInt32WithTruncation: + case NI_AVX10v1_ConvertToUInt32WithTruncation: +#if defined(TARGET_AMD64) + case NI_AVX10v1_ConvertToInt64WithTruncation: + case NI_AVX10v1_ConvertToUInt64WithTruncation: +#endif // TARGET_AMD64 + { + // The source is a scalar `Half` (bound as `TYP_USHORT`) held in a vector register while the + // result is produced directly into a general purpose register. Use the size of the target + // integer type so the correct register width is displayed (e.g. `eax` rather than `rax`). + assert(baseType == TYP_USHORT); + attr = emitTypeSize(targetType); + genHWIntrinsic_R_RM(node, ins, attr, targetReg, op1, instOptions); + break; + } + + case NI_AVX10v1_ConvertScalarToVector128Half: + { + // For integer sources the value is read directly from a general purpose register, so the + // operand size must reflect the source type (e.g. `ecx` rather than `rcx`). Floating-point + // sources come from a vector register and use the full 128-bit size. + if (varTypeIsIntegral(baseType)) + { + attr = emitActualTypeSize(baseType); + } + genHWIntrinsic_R_R_RM(node, ins, attr, instOptions); + break; + } + case NI_AVXVNNIINT_MultiplyWideningAndAddSaturate: case NI_AVXVNNIINT_V512_MultiplyWideningAndAddSaturate: { diff --git a/src/coreclr/jit/hwintrinsiclistarm64.h b/src/coreclr/jit/hwintrinsiclistarm64.h index c04f9bf9a8be30..39ef6158c01b9c 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64.h +++ b/src/coreclr/jit/hwintrinsiclistarm64.h @@ -498,7 +498,13 @@ HARDWARE_INTRINSIC(Aes, PolynomialMultiplyWideningUpper, // TYP_BYTE, TYP_UBYTE, TYP_SHORT, TYP_USHORT, TYP_INT, TYP_UINT, TYP_LONG, TYP_ULONG, TYP_FLOAT, TYP_DOUBLE // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Base Intrinsics -#define FIRST_NI_ArmBase NI_ArmBase_LeadingZeroCount +// NOTE: ConvertToDouble/ConvertToHalf/ConvertToSingle below are JIT-internal only (no managed API). They accelerate +// the baseline half<->single/double FCVT conversions used by System.Half. FCVT between half and single/double is part +// of the base FP/AdvSimd baseline (Armv8.0) and does NOT require FEAT_FP16, so they live in ArmBase rather than Fp16. +#define FIRST_NI_ArmBase NI_ArmBase_ConvertToDouble +HARDWARE_INTRINSIC(ArmBase, ConvertToDouble, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_fcvt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(ArmBase, ConvertToHalf, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fcvt, INS_fcvt, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(ArmBase, ConvertToSingle, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_fcvt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(ArmBase, LeadingZeroCount, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_clz, INS_clz, INS_clz, INS_clz, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoFloatingPointUsed) HARDWARE_INTRINSIC(ArmBase, ReverseElementBits, 0, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rbit, INS_rbit, INS_rbit, INS_rbit, INS_invalid, INS_invalid, -1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed) HARDWARE_INTRINSIC(ArmBase, Yield, 0, 0, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_SpecialSideEffect_Other) @@ -549,6 +555,37 @@ HARDWARE_INTRINSIC(Dp, DotProduct, HARDWARE_INTRINSIC(Dp, DotProductBySelectedQuadruplet, -1, 4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sdot, INS_udot, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDByIndexedElement, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics) #define LAST_NI_Dp NI_Dp_DotProductBySelectedQuadruplet +// ************************************************************************************************************************************************************************************************************************************************************************************************************ +// ISA Function name SIMD size NumArg Instructions IntCost FltCost Category Flags +// TYP_BYTE, TYP_UBYTE, TYP_SHORT, TYP_USHORT, TYP_INT, TYP_UINT, TYP_LONG, TYP_ULONG, TYP_FLOAT, TYP_DOUBLE +// ************************************************************************************************************************************************************************************************************************************************************************************************************ +// FP16 Intrinsics +#define FIRST_NI_Fp16 NI_Fp16_Add +HARDWARE_INTRINSIC(Fp16, Add, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_fadd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Fp16, Ceiling, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_frintp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Fp16, CompareEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_fcmp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Fp16, CompareGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_fcmp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Fp16, CompareGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_fcmp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Fp16, CompareLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_fcmp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Fp16, CompareLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_fcmp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Fp16, CompareNotEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_fcmp, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Fp16, ConvertToHalf, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_scvtf, INS_ucvtf, INS_scvtf, INS_ucvtf, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Fp16, ConvertToInt32, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzs, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Fp16, ConvertToInt64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzs, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Fp16, ConvertToUInt32, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzu, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Fp16, ConvertToUInt64, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_fcvtzu, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Fp16, Divide, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_fdiv, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Fp16, Floor, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_frintm, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Fp16, FusedMultiplyAdd, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_fmadd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Fp16, Multiply, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_fmul, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_Commutative|HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Fp16, ReciprocalEstimate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_frecpe, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Fp16, ReciprocalSqrtEstimate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_frsqrte, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Fp16, RoundToNearest, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_frintn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Fp16, Sqrt, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_fsqrt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Fp16, Subtract, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_fsub, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +HARDWARE_INTRINSIC(Fp16, Truncate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_frintz, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_SIMDScalar) +#define LAST_NI_Fp16 NI_Fp16_Truncate + // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // ISA Function name SIMD size NumArg Instructions IntCost FltCost Category Flags // TYP_BYTE, TYP_UBYTE, TYP_SHORT, TYP_USHORT, TYP_INT, TYP_UINT, TYP_LONG, TYP_ULONG, TYP_FLOAT, TYP_DOUBLE diff --git a/src/coreclr/jit/hwintrinsiclistxarch.h b/src/coreclr/jit/hwintrinsiclistxarch.h index 12bea05dbae65f..6de21e5056368c 100644 --- a/src/coreclr/jit/hwintrinsiclistxarch.h +++ b/src/coreclr/jit/hwintrinsiclistxarch.h @@ -656,6 +656,42 @@ HARDWARE_INTRINSIC(AVX512v3, MultiplyWideningAndAdd, HARDWARE_INTRINSIC(AVX512v3, MultiplyWideningAndAddSaturate, 64, 3, INS_invalid, INS_vpdpbusds, INS_vpdpwssds, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg) #define LAST_NI_AVX512v3 NI_AVX512v3_MultiplyWideningAndAddSaturate +// *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** +// ISA Function name SIMD size NumArg Instructions IntCost FltCost Category Flags +// TYP_BYTE, TYP_UBYTE, TYP_SHORT, TYP_USHORT, TYP_INT, TYP_UINT, TYP_LONG, TYP_ULONG, TYP_FLOAT, TYP_DOUBLE +// *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** +// Intrinsics for AVX10v1 +#define FIRST_NI_AVX10v1 NI_AVX10v1_AddScalar +HARDWARE_INTRINSIC(AVX10v1, AddScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vaddsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_SIMDScalar, HW_Flag_EmbRoundingCompatible|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarOrderedEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vcomish, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarOrderedGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vcomish, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarOrderedGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vcomish, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarOrderedLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vcomish, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarOrderedLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vcomish, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarOrderedNotEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vcomish, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarUnorderedEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vucomish, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarUnorderedGreaterThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vucomish, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarUnorderedGreaterThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vucomish, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarUnorderedLessThan, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vucomish, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarUnorderedLessThanOrEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vucomish, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, CompareScalarUnorderedNotEqual, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vucomish, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, 3, HW_Category_SIMDScalar, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX10v1, ConvertScalarToVector128Double, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vcvtsh2sd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX10v1, ConvertScalarToVector128Half, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtsi2sh32, INS_vcvtusi2sh32, INS_vcvtsi2sh64, INS_vcvtusi2sh64, INS_vcvtss2sh, INS_vcvtsd2sh, 5, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX10v1, ConvertScalarToVector128Single, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vcvtsh2ss, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 5, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits|HW_Flag_EmbRoundingCompatible) +HARDWARE_INTRINSIC(AVX10v1, ConvertToInt32WithTruncation, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vcvttsh2si32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 7, 7, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX10v1, ConvertToInt64WithTruncation, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vcvttsh2si64, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 7, 7, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX10v1, ConvertToUInt32WithTruncation, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vcvttsh2usi32, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 7, 7, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX10v1, ConvertToUInt64WithTruncation, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vcvttsh2usi64, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 7, 7, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(AVX10v1, DivideScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vdivsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX10v1, FusedMultiplyAddScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_vfmadd213sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_SIMDScalar, HW_Flag_SpecialCodeGen|HW_Flag_FmaIntrinsic|HW_Flag_RmwIntrinsic|HW_Flag_EmbRoundingCompatible|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX10v1, MultiplyScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vmulsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX10v1, ReciprocalScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vrcpsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX10v1, ReciprocalSqrtScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vrsqrtsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX10v1, RoundScaleScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_vrndscalesh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 8, 8, HW_Category_IMM, HW_Flag_FullRangeIMM|HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX10v1, SqrtScalar, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vsqrtsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +HARDWARE_INTRINSIC(AVX10v1, SubtractScalar, 16, -1, INS_invalid, INS_invalid, INS_invalid, INS_vsubsh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, 4, HW_Category_SIMDScalar, HW_Flag_CopyUpperBits) +#define LAST_NI_AVX10v1 NI_AVX10v1_SubtractScalar + // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // ISA Function name SIMD size NumArg Instructions IntCost FltCost Category Flags // TYP_BYTE, TYP_UBYTE, TYP_SHORT, TYP_USHORT, TYP_INT, TYP_UINT, TYP_LONG, TYP_ULONG, TYP_FLOAT, TYP_DOUBLE @@ -809,6 +845,8 @@ HARDWARE_INTRINSIC(GFNI_V512, GaloisFieldMultiply, HARDWARE_INTRINSIC(X86Base, COMIS, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_comiss, INS_comisd, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics) HARDWARE_INTRINSIC(X86Base, PTEST, 16, 2, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_NoRMWSemantics|HW_Flag_NoEvexSemantics) HARDWARE_INTRINSIC(X86Base, UCOMIS, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ucomiss, INS_ucomisd, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(AVX10v1, VCOMISH, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vcomish, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics) +HARDWARE_INTRINSIC(AVX10v1, VUCOMISH, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_vucomish, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, 3, HW_Category_SIMDScalar, HW_Flag_NoRMWSemantics) HARDWARE_INTRINSIC(AVX, PTEST, -1, 2, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_ptest, INS_vtestps, INS_vtestpd, -1, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) HARDWARE_INTRINSIC(AVX2, AndNotVector, 32, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pandnd, INS_pandnd, INS_pandnd, INS_pandnd, INS_invalid, INS_invalid, 1, -1, HW_Category_SimpleSIMD, HW_Flag_NormalizeSmallTypeToInt) HARDWARE_INTRINSIC(AVX2, AndNotScalar, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_andn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 1, -1, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed|HW_Flag_NoEvexSemantics) diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 61db69f06ddb25..80b7bd5120c57e 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -4519,62 +4519,478 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, if (retType == TYP_STRUCT) { + // Converting some arithmetic type -> Half assert(isSystemHalfClass(retClsHnd)); assert(varTypeIsArithmetic(op1Type)); - switch (op1Type) +#if defined(TARGET_XARCH) + if (compOpportunisticallyDependsOn(InstructionSet_AVX10v1)) { - case TYP_FLOAT: + bool supported = false; + + switch (op1Type) { -#if defined(TARGET_XARCH) - if (compOpportunisticallyDependsOn(InstructionSet_AVX2)) - { - GenTree* op1 = impPopStack().val; - op1 = gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, op1, TYP_FLOAT, 16); + case TYP_FLOAT: + case TYP_DOUBLE: + case TYP_INT: + case TYP_UINT: + supported = true; + break; +#ifdef TARGET_AMD64 + case TYP_LONG: + case TYP_ULONG: + supported = true; + break; +#endif // TARGET_AMD64 + default: + break; + } - retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, gtNewIconNode(0), - NI_AVX2_ConvertToVector128Half, TYP_FLOAT, 16); - retNode = impSimdToScalarHalf(retNode, retClsHnd); + if (supported) + { + GenTree* op1 = impPopStack().val; + GenTree* zeroVec = gtNewZeroConNode(TYP_SIMD16); + + // The integer scalar convert instructions read the source directly from a general + // purpose register, so only floating-point sources need to be moved into a vector. + if (varTypeIsFloating(op1Type)) + { + op1 = gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, op1, op1Type, 16); } -#endif + + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, zeroVec, op1, + NI_AVX10v1_ConvertScalarToVector128Half, op1Type, 16); + retNode = impSimdToScalarHalf(retNode, retClsHnd); break; } + } - default: - { - unreached(); - } + if ((op1Type == TYP_FLOAT) && compOpportunisticallyDependsOn(InstructionSet_AVX2)) + { + GenTree* op1 = impPopStack().val; + op1 = gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, op1, TYP_FLOAT, 16); + + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, gtNewIconNode(0), + NI_AVX2_ConvertToVector128Half, TYP_FLOAT, 16); + retNode = impSimdToScalarHalf(retNode, retClsHnd); + } +#elif defined(TARGET_ARM64) + // FCVT between half and single/double is part of the Armv8.0 FP baseline and does not + // require FEAT_FP16, so those conversions are always accelerated. Integer -> half + // conversions (SCVTF/UCVTF with a half destination) require FEAT_FP16. + if (varTypeIsFloating(op1Type)) + { + GenTree* op1 = impPopStack().val; + op1 = gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, op1, op1Type, 16); + + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, NI_ArmBase_ConvertToHalf, op1Type, 16); + retNode = impSimdToScalarHalf(retNode, retClsHnd); } + else if (compOpportunisticallyDependsOn(InstructionSet_Fp16)) + { + // The integer scalar convert instructions read the source directly from a general + // purpose register, so no move into a vector register is required. + GenTree* op1 = impPopStack().val; + + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, NI_Fp16_ConvertToHalf, op1Type, 16); + retNode = impSimdToScalarHalf(retNode, retClsHnd); + } +#endif // TARGET_XARCH } else { + // Converting Half -> some arithmetic type assert(varTypeIsArithmetic(retType)); assert((op1Type == TYP_STRUCT) && isSystemHalfClass(op1ClsHnd)); - switch (retType) +#if defined(TARGET_XARCH) + if (compOpportunisticallyDependsOn(InstructionSet_AVX10v1)) { - case TYP_FLOAT: + NamedIntrinsic opId = NI_Illegal; + + switch (retType) { -#if defined(TARGET_XARCH) - if (compOpportunisticallyDependsOn(InstructionSet_AVX2)) - { - GenTree* op1 = impPopStack().val; - op1 = impSimdCreateScalarHalf(op1); + case TYP_FLOAT: + opId = NI_AVX10v1_ConvertScalarToVector128Single; + break; + case TYP_DOUBLE: + opId = NI_AVX10v1_ConvertScalarToVector128Double; + break; + case TYP_INT: + opId = NI_AVX10v1_ConvertToInt32WithTruncation; + break; + case TYP_UINT: + opId = NI_AVX10v1_ConvertToUInt32WithTruncation; + break; +#ifdef TARGET_AMD64 + case TYP_LONG: + opId = NI_AVX10v1_ConvertToInt64WithTruncation; + break; + case TYP_ULONG: + opId = NI_AVX10v1_ConvertToUInt64WithTruncation; + break; +#endif // TARGET_AMD64 + default: + break; + } + + if (opId != NI_Illegal) + { + GenTree* op1 = impPopStack().val; + op1 = impSimdCreateScalarHalf(op1); - retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, NI_AVX2_ConvertToVector128Single, - TYP_USHORT, 16); - retNode = gtNewSimdToScalarNode(TYP_FLOAT, retNode, TYP_FLOAT, 16); + if (varTypeIsFloating(retType)) + { + GenTree* zeroVec = gtNewZeroConNode(TYP_SIMD16); + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, zeroVec, op1, opId, TYP_USHORT, 16); + retNode = gtNewSimdToScalarNode(retType, retNode, retType, 16); + } + else + { + // The result is produced directly in a general purpose register. The node + // type must be the actual machine type (TYP_INT/TYP_LONG); signedness is + // encoded by the selected instruction (vcvttsh2si vs vcvttsh2usi). + retNode = gtNewSimdHWIntrinsicNode(genActualType(retType), op1, opId, TYP_USHORT, 16); } -#endif break; } + } + + if ((retType == TYP_FLOAT) && compOpportunisticallyDependsOn(InstructionSet_AVX2)) + { + GenTree* op1 = impPopStack().val; + op1 = impSimdCreateScalarHalf(op1); + + retNode = + gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, NI_AVX2_ConvertToVector128Single, TYP_USHORT, 16); + retNode = gtNewSimdToScalarNode(TYP_FLOAT, retNode, TYP_FLOAT, 16); + } +#elif defined(TARGET_ARM64) + // Half -> single/double is a baseline FCVT (no FEAT_FP16 needed); Half -> integer + // (FCVTZS/FCVTZU with a half operand) requires FEAT_FP16. + NamedIntrinsic opId = NI_Illegal; + bool isFp16 = false; + switch (retType) + { + case TYP_FLOAT: + opId = NI_ArmBase_ConvertToSingle; + break; + case TYP_DOUBLE: + opId = NI_ArmBase_ConvertToDouble; + break; + case TYP_INT: + opId = NI_Fp16_ConvertToInt32; + isFp16 = true; + break; + case TYP_UINT: + opId = NI_Fp16_ConvertToUInt32; + isFp16 = true; + break; + case TYP_LONG: + opId = NI_Fp16_ConvertToInt64; + isFp16 = true; + break; + case TYP_ULONG: + opId = NI_Fp16_ConvertToUInt64; + isFp16 = true; + break; default: - { - unreached(); - } + break; } + + if ((opId != NI_Illegal) && (!isFp16 || compOpportunisticallyDependsOn(InstructionSet_Fp16))) + { + GenTree* op1 = impPopStack().val; + op1 = impSimdCreateScalarHalf(op1); + + // The Arm64 scalar convert instructions produce their result directly in the + // target register (an FP register for float/double, a general purpose register + // for integers), so no vector extraction is needed. + retNode = gtNewSimdHWIntrinsicNode(genActualType(retType), op1, opId, TYP_USHORT, 16); + } +#endif // TARGET_XARCH + } + break; + } + + case NI_System_Half_op_Addition: + case NI_System_Half_op_Subtraction: + case NI_System_Half_op_Multiply: + case NI_System_Half_op_Division: + { +#if defined(TARGET_XARCH) + if (compOpportunisticallyDependsOn(InstructionSet_AVX10v1)) + { + NamedIntrinsic opId = lookupHalfIntrinsic(ni); + assert(opId != NI_Illegal); + + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + + op2 = impSimdCreateScalarHalf(op2); + op1 = impSimdCreateScalarHalf(op1); + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, opId, TYP_USHORT, 16); + retNode = impSimdToScalarHalf(retNode, sig->retTypeSigClass); + } +#elif defined(TARGET_ARM64) + if (compOpportunisticallyDependsOn(InstructionSet_Fp16)) + { + NamedIntrinsic opId = lookupHalfIntrinsic(ni); + assert(opId != NI_Illegal); + + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + + op2 = impSimdCreateScalarHalf(op2); + op1 = impSimdCreateScalarHalf(op1); + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, opId, TYP_USHORT, 16); + retNode = impSimdToScalarHalf(retNode, sig->retTypeSigClass); + } +#endif // TARGET_XARCH + break; + } + + case NI_System_Half_Sqrt: + case NI_System_Half_ReciprocalEstimate: + case NI_System_Half_ReciprocalSqrtEstimate: + { +#if defined(TARGET_XARCH) + if (compOpportunisticallyDependsOn(InstructionSet_AVX10v1)) + { + NamedIntrinsic opId = lookupHalfIntrinsic(ni); + assert(opId != NI_Illegal); + + GenTree* op1 = impPopStack().val; + + // These scalar ops compute their result from lane 0 of the second operand and take the + // upper bits from the first. We only consume lane 0, so a zeroed upper-bits source is fine. + GenTree* op2 = gtNewZeroConNode(TYP_SIMD16); + op1 = impSimdCreateScalarHalf(op1); + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op2, op1, opId, TYP_USHORT, 16); + retNode = impSimdToScalarHalf(retNode, sig->retTypeSigClass); + } +#elif defined(TARGET_ARM64) + if (compOpportunisticallyDependsOn(InstructionSet_Fp16)) + { + NamedIntrinsic opId = lookupHalfIntrinsic(ni); + assert(opId != NI_Illegal); + + GenTree* op1 = impPopStack().val; + op1 = impSimdCreateScalarHalf(op1); + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, opId, TYP_USHORT, 16); + retNode = impSimdToScalarHalf(retNode, sig->retTypeSigClass); + } +#endif // TARGET_XARCH + break; + } + + case NI_System_Half_FusedMultiplyAdd: + { +#if defined(TARGET_XARCH) + if (compOpportunisticallyDependsOn(InstructionSet_AVX10v1)) + { + GenTree* op3 = impPopStack().val; + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + + op3 = impSimdCreateScalarHalf(op3); + op2 = impSimdCreateScalarHalf(op2); + op1 = impSimdCreateScalarHalf(op1); + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, op3, NI_AVX10v1_FusedMultiplyAddScalar, + TYP_USHORT, 16); + retNode = impSimdToScalarHalf(retNode, sig->retTypeSigClass); + } +#elif defined(TARGET_ARM64) + if (compOpportunisticallyDependsOn(InstructionSet_Fp16)) + { + GenTree* op3 = impPopStack().val; + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + + op3 = impSimdCreateScalarHalf(op3); + op2 = impSimdCreateScalarHalf(op2); + op1 = impSimdCreateScalarHalf(op1); + + // fmadd computes Rd = Rn * Rm + Ra, so (op1 * op2) + op3 == x * y + z. + retNode = + gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, op3, NI_Fp16_FusedMultiplyAdd, TYP_USHORT, 16); + retNode = impSimdToScalarHalf(retNode, sig->retTypeSigClass); + } +#endif // TARGET_XARCH + break; + } + + case NI_System_Half_Round: + case NI_System_Half_Ceiling: + case NI_System_Half_Floor: + case NI_System_Half_Truncate: + { +#if defined(TARGET_XARCH) + // TODO-CQ-XArch: We only optimize the single-argument overloads for now. + if (compOpportunisticallyDependsOn(InstructionSet_AVX10v1) && (sig->numArgs == 1)) + { + GenTree* op1 = impPopStack().val; + + int halfRoundingMode = lookupHalfRoundingMode(ni); + + GenTree* op2 = gtNewZeroConNode(TYP_SIMD16); + op1 = impSimdCreateScalarHalf(op1); + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op2, op1, gtNewIconNode(halfRoundingMode, TYP_INT), + NI_AVX10v1_RoundScaleScalar, TYP_USHORT, 16); + retNode = impSimdToScalarHalf(retNode, sig->retTypeSigClass); + } +#elif defined(TARGET_ARM64) + // TODO-ARM64-CQ: We only optimize the single-argument overloads for now. + if (compOpportunisticallyDependsOn(InstructionSet_Fp16) && (sig->numArgs == 1)) + { + // Arm64 has a dedicated rounding instruction per mode, so no rounding immediate is needed. + NamedIntrinsic opId = lookupHalfIntrinsic(ni); + assert(opId != NI_Illegal); + + GenTree* op1 = impPopStack().val; + op1 = impSimdCreateScalarHalf(op1); + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, opId, TYP_USHORT, 16); + retNode = impSimdToScalarHalf(retNode, sig->retTypeSigClass); + } +#endif // TARGET_XARCH + break; + } + + case NI_System_Half_op_GreaterThan: + case NI_System_Half_op_GreaterThanOrEqual: + case NI_System_Half_op_LessThan: + case NI_System_Half_op_LessThanOrEqual: + case NI_System_Half_op_Equality: + case NI_System_Half_op_Inequality: + { +#if defined(TARGET_XARCH) + if (compOpportunisticallyDependsOn(InstructionSet_AVX10v1)) + { + NamedIntrinsic opId = lookupHalfIntrinsic(ni); + assert(opId != NI_Illegal); + + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + + op2 = impSimdCreateScalarHalf(op2); + op1 = impSimdCreateScalarHalf(op1); + retNode = gtNewSimdHWIntrinsicNode(TYP_INT, op1, op2, opId, TYP_USHORT, 16); + } +#elif defined(TARGET_ARM64) + if (compOpportunisticallyDependsOn(InstructionSet_Fp16)) + { + NamedIntrinsic opId = lookupHalfIntrinsic(ni); + assert(opId != NI_Illegal); + + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + + op2 = impSimdCreateScalarHalf(op2); + op1 = impSimdCreateScalarHalf(op1); + retNode = gtNewSimdHWIntrinsicNode(TYP_INT, op1, op2, opId, TYP_USHORT, 16); + } +#endif // TARGET_XARCH + break; + } + + case NI_System_Half_op_Increment: + case NI_System_Half_op_Decrement: + { +#if defined(TARGET_XARCH) + if (compOpportunisticallyDependsOn(InstructionSet_AVX10v1)) + { + NamedIntrinsic opId = lookupHalfIntrinsic(ni); + assert(opId != NI_Illegal); + + GenTree* op1 = impPopStack().val; + + // Increment/decrement by the Half constant 1.0 (0x3C00). Creating the constant + // directly avoids a runtime float -> half conversion. + GenTree* oneVec = + gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, gtNewIconNode(0x3C00, TYP_INT), TYP_USHORT, 16); + + op1 = impSimdCreateScalarHalf(op1); + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, oneVec, opId, TYP_USHORT, 16); + retNode = impSimdToScalarHalf(retNode, sig->retTypeSigClass); + } +#elif defined(TARGET_ARM64) + if (compOpportunisticallyDependsOn(InstructionSet_Fp16)) + { + NamedIntrinsic opId = lookupHalfIntrinsic(ni); + assert(opId != NI_Illegal); + + GenTree* op1 = impPopStack().val; + + // Increment/decrement by the Half constant 1.0 (0x3C00). Creating the constant + // directly avoids a runtime float -> half conversion. + GenTree* oneVec = + gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, gtNewIconNode(0x3C00, TYP_INT), TYP_USHORT, 16); + + op1 = impSimdCreateScalarHalf(op1); + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, oneVec, opId, TYP_USHORT, 16); + retNode = impSimdToScalarHalf(retNode, sig->retTypeSigClass); } +#endif // TARGET_XARCH + break; + } + + case NI_System_Half_get_MinValue: + case NI_System_Half_get_MaxValue: + case NI_System_Half_get_Epsilon: + case NI_System_Half_get_NaN: + case NI_System_Half_get_PositiveInfinity: + case NI_System_Half_get_NegativeInfinity: + case NI_System_Half_get_One: + case NI_System_Half_get_Zero: + { +#if defined(TARGET_XARCH) || defined(TARGET_ARM64) + uint16_t halfBits = 0; + + switch (ni) + { + case NI_System_Half_get_MinValue: + halfBits = 0xFBFF; // -65504 + break; + case NI_System_Half_get_MaxValue: + halfBits = 0x7BFF; // 65504 + break; + case NI_System_Half_get_Epsilon: + halfBits = 0x0001; // ~5.9604645e-08 (smallest positive subnormal) + break; + case NI_System_Half_get_NaN: + halfBits = 0xFE00; // Negative NaN + break; + case NI_System_Half_get_PositiveInfinity: + halfBits = 0x7C00; // +Infinity + break; + case NI_System_Half_get_NegativeInfinity: + halfBits = 0xFC00; // -Infinity + break; + case NI_System_Half_get_One: + halfBits = 0x3C00; // 1.0 + break; + case NI_System_Half_get_Zero: + halfBits = 0x0000; // 0.0 + break; + default: + unreached(); + } + +#if defined(TARGET_XARCH) + if (compOpportunisticallyDependsOn(InstructionSet_AVX10v1)) +#else + if (compOpportunisticallyDependsOn(InstructionSet_Fp16)) +#endif + { + // Create the Half constant directly from its bit pattern rather than materializing + // it via a runtime conversion. The return type is always System.Half, so its class + // handle can be taken from the signature (which is reliable even when the getter is + // reached via a generic constrained call). + retNode = gtNewSimdCreateScalarNode(TYP_SIMD16, gtNewIconNode(halfBits, TYP_INT), TYP_USHORT, 16); + retNode = impSimdToScalarHalf(retNode, sig->retTypeSigClass); + } +#endif // TARGET_XARCH || TARGET_ARM64 break; } @@ -10901,7 +11317,7 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method) { if (strcmp(className, "Half") == 0) { - result = lookupPrimitiveFloatNamedIntrinsic(method, methodName); + result = lookupHalfNamedIntrinsic(method, methodName); } break; } @@ -12234,6 +12650,284 @@ NamedIntrinsic Compiler::lookupPrimitiveFloatNamedIntrinsic(CORINFO_METHOD_HANDL return result; } +//------------------------------------------------------------------------ +// lookupHalfNamedIntrinsic: map a System.Half method to its jit named intrinsic value +// +// Arguments: +// method -- method handle for method +// methodName -- name of the method +// +// Return Value: +// Id for the named intrinsic, or Illegal if none. +// +// Notes: +// method should have CORINFO_FLG_INTRINSIC set in its attributes, +// otherwise it is not a named jit intrinsic. +// +NamedIntrinsic Compiler::lookupHalfNamedIntrinsic(CORINFO_METHOD_HANDLE method, const char* methodName) +{ + NamedIntrinsic result = NI_Illegal; + + if (strcmp(methodName, "op_Addition") == 0) + { + result = NI_System_Half_op_Addition; + } + else if (strcmp(methodName, "op_Subtraction") == 0) + { + result = NI_System_Half_op_Subtraction; + } + else if (strcmp(methodName, "op_Multiply") == 0) + { + result = NI_System_Half_op_Multiply; + } + else if (strcmp(methodName, "op_Division") == 0) + { + result = NI_System_Half_op_Division; + } + else if (strcmp(methodName, "op_Equality") == 0) + { + result = NI_System_Half_op_Equality; + } + else if (strcmp(methodName, "op_Inequality") == 0) + { + result = NI_System_Half_op_Inequality; + } + else if (strcmp(methodName, "op_GreaterThan") == 0) + { + result = NI_System_Half_op_GreaterThan; + } + else if (strcmp(methodName, "op_GreaterThanOrEqual") == 0) + { + result = NI_System_Half_op_GreaterThanOrEqual; + } + else if (strcmp(methodName, "op_LessThan") == 0) + { + result = NI_System_Half_op_LessThan; + } + else if (strcmp(methodName, "op_LessThanOrEqual") == 0) + { + result = NI_System_Half_op_LessThanOrEqual; + } + else if (strcmp(methodName, "op_Explicit") == 0) + { + result = NI_System_Half_op_Explicit; + } + else if (strcmp(methodName, "Sqrt") == 0) + { + result = NI_System_Half_Sqrt; + } + else if (strcmp(methodName, "ReciprocalEstimate") == 0) + { + result = NI_System_Half_ReciprocalEstimate; + } + else if (strcmp(methodName, "ReciprocalSqrtEstimate") == 0) + { + result = NI_System_Half_ReciprocalSqrtEstimate; + } + else if (strcmp(methodName, "FusedMultiplyAdd") == 0) + { + result = NI_System_Half_FusedMultiplyAdd; + } + else if (strcmp(methodName, "Round") == 0) + { + result = NI_System_Half_Round; + } + else if (strcmp(methodName, "Ceiling") == 0) + { + result = NI_System_Half_Ceiling; + } + else if (strcmp(methodName, "Floor") == 0) + { + result = NI_System_Half_Floor; + } + else if (strcmp(methodName, "Truncate") == 0) + { + result = NI_System_Half_Truncate; + } + else if (strcmp(methodName, "op_Increment") == 0) + { + result = NI_System_Half_op_Increment; + } + else if (strcmp(methodName, "op_Decrement") == 0) + { + result = NI_System_Half_op_Decrement; + } + else if (strcmp(methodName, "get_MinValue") == 0) + { + result = NI_System_Half_get_MinValue; + } + else if (strcmp(methodName, "get_MaxValue") == 0) + { + result = NI_System_Half_get_MaxValue; + } + else if (strcmp(methodName, "get_Epsilon") == 0) + { + result = NI_System_Half_get_Epsilon; + } + else if (strcmp(methodName, "get_NaN") == 0) + { + result = NI_System_Half_get_NaN; + } + else if (strcmp(methodName, "get_PositiveInfinity") == 0) + { + result = NI_System_Half_get_PositiveInfinity; + } + else if (strcmp(methodName, "get_NegativeInfinity") == 0) + { + result = NI_System_Half_get_NegativeInfinity; + } + else if (strcmp(methodName, "get_One") == 0) + { + result = NI_System_Half_get_One; + } + else if (strcmp(methodName, "get_Zero") == 0) + { + result = NI_System_Half_get_Zero; + } + + return result; +} + +#if defined(FEATURE_HW_INTRINSICS) && (defined(TARGET_XARCH) || defined(TARGET_ARM64)) +//------------------------------------------------------------------------ +// lookupHalfIntrinsic: map a System.Half named intrinsic to the internal scalar +// hardware intrinsic that implements it +// +// Arguments: +// ni -- the System.Half named intrinsic +// +// Return Value: +// The corresponding scalar hardware intrinsic, or NI_Illegal if none. +// +NamedIntrinsic Compiler::lookupHalfIntrinsic(NamedIntrinsic ni) +{ +#if defined(TARGET_XARCH) + assert(compOpportunisticallyDependsOn(InstructionSet_AVX10v1)); + + switch (ni) + { + case NI_System_Half_op_Addition: + return NI_AVX10v1_AddScalar; + case NI_System_Half_op_Increment: + return NI_AVX10v1_AddScalar; + case NI_System_Half_op_Subtraction: + return NI_AVX10v1_SubtractScalar; + case NI_System_Half_op_Decrement: + return NI_AVX10v1_SubtractScalar; + case NI_System_Half_op_Multiply: + return NI_AVX10v1_MultiplyScalar; + case NI_System_Half_op_Division: + return NI_AVX10v1_DivideScalar; + case NI_System_Half_Sqrt: + return NI_AVX10v1_SqrtScalar; + case NI_System_Half_ReciprocalEstimate: + return NI_AVX10v1_ReciprocalScalar; + case NI_System_Half_ReciprocalSqrtEstimate: + return NI_AVX10v1_ReciprocalSqrtScalar; + case NI_System_Half_FusedMultiplyAdd: + return NI_AVX10v1_FusedMultiplyAddScalar; + case NI_System_Half_op_GreaterThan: + return NI_AVX10v1_CompareScalarOrderedGreaterThan; + case NI_System_Half_op_GreaterThanOrEqual: + return NI_AVX10v1_CompareScalarOrderedGreaterThanOrEqual; + case NI_System_Half_op_LessThan: + return NI_AVX10v1_CompareScalarOrderedLessThan; + case NI_System_Half_op_LessThanOrEqual: + return NI_AVX10v1_CompareScalarOrderedLessThanOrEqual; + case NI_System_Half_op_Equality: + return NI_AVX10v1_CompareScalarOrderedEqual; + case NI_System_Half_op_Inequality: + return NI_AVX10v1_CompareScalarOrderedNotEqual; + case NI_System_Half_Round: + case NI_System_Half_Ceiling: + case NI_System_Half_Floor: + case NI_System_Half_Truncate: + return NI_AVX10v1_RoundScaleScalar; + default: + return NI_Illegal; + } +#elif defined(TARGET_ARM64) + assert(compOpportunisticallyDependsOn(InstructionSet_Fp16)); + + switch (ni) + { + case NI_System_Half_op_Addition: + return NI_Fp16_Add; + case NI_System_Half_op_Increment: + return NI_Fp16_Add; + case NI_System_Half_op_Subtraction: + return NI_Fp16_Subtract; + case NI_System_Half_op_Decrement: + return NI_Fp16_Subtract; + case NI_System_Half_op_Multiply: + return NI_Fp16_Multiply; + case NI_System_Half_op_Division: + return NI_Fp16_Divide; + case NI_System_Half_Sqrt: + return NI_Fp16_Sqrt; + case NI_System_Half_ReciprocalEstimate: + return NI_Fp16_ReciprocalEstimate; + case NI_System_Half_ReciprocalSqrtEstimate: + return NI_Fp16_ReciprocalSqrtEstimate; + case NI_System_Half_FusedMultiplyAdd: + return NI_Fp16_FusedMultiplyAdd; + case NI_System_Half_op_GreaterThan: + return NI_Fp16_CompareGreaterThan; + case NI_System_Half_op_GreaterThanOrEqual: + return NI_Fp16_CompareGreaterThanOrEqual; + case NI_System_Half_op_LessThan: + return NI_Fp16_CompareLessThan; + case NI_System_Half_op_LessThanOrEqual: + return NI_Fp16_CompareLessThanOrEqual; + case NI_System_Half_op_Equality: + return NI_Fp16_CompareEqual; + case NI_System_Half_op_Inequality: + return NI_Fp16_CompareNotEqual; + case NI_System_Half_Round: + return NI_Fp16_RoundToNearest; + case NI_System_Half_Ceiling: + return NI_Fp16_Ceiling; + case NI_System_Half_Floor: + return NI_Fp16_Floor; + case NI_System_Half_Truncate: + return NI_Fp16_Truncate; + default: + return NI_Illegal; + } +#endif // TARGET_ARM64 +} +#endif // FEATURE_HW_INTRINSICS && (TARGET_XARCH || TARGET_ARM64) + +#if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_XARCH) +//------------------------------------------------------------------------ +// lookupHalfRoundingMode: map a System.Half rounding named intrinsic to the +// immediate rounding mode used by RoundScaleScalar +// +// Arguments: +// ni -- the System.Half named intrinsic +// +// Return Value: +// The rounding mode immediate (0=nearest, 1=-inf, 2=+inf, 3=zero). +// +int Compiler::lookupHalfRoundingMode(NamedIntrinsic ni) +{ + switch (ni) + { + case NI_System_Half_Round: + return static_cast(FloatRoundingMode::ToNearestInteger); + case NI_System_Half_Ceiling: + return static_cast(FloatRoundingMode::ToPositiveInfinity); + case NI_System_Half_Floor: + return static_cast(FloatRoundingMode::ToNegativeInfinity); + case NI_System_Half_Truncate: + return static_cast(FloatRoundingMode::ToZero); + default: + noway_assert(!"Should have one of the above Half intrinsics"); + return -1; + } +} +#endif // FEATURE_HW_INTRINSICS && TARGET_XARCH + //------------------------------------------------------------------------ // lookupPrimitiveIntNamedIntrinsic: map method to jit named intrinsic value // diff --git a/src/coreclr/jit/instr.h b/src/coreclr/jit/instr.h index bb75395bfccdc6..f2b7310d5c641d 100644 --- a/src/coreclr/jit/instr.h +++ b/src/coreclr/jit/instr.h @@ -402,12 +402,18 @@ enum insOpts : unsigned INS_OPTS_S_TO_8BYTE, // Single to INT64 INS_OPTS_D_TO_8BYTE, // Double to INT64 + INS_OPTS_H_TO_4BYTE, // Half to INT32 + INS_OPTS_H_TO_8BYTE, // Half to INT64 + INS_OPTS_4BYTE_TO_S, // INT32 to Single INS_OPTS_4BYTE_TO_D, // INT32 to Double INS_OPTS_8BYTE_TO_S, // INT64 to Single INS_OPTS_8BYTE_TO_D, // INT64 to Double + INS_OPTS_4BYTE_TO_H, // INT32 to Half + INS_OPTS_8BYTE_TO_H, // INT64 to Half + INS_OPTS_S_TO_D, // Single to Double INS_OPTS_D_TO_S, // Double to Single diff --git a/src/coreclr/jit/instrsxarch.h b/src/coreclr/jit/instrsxarch.h index 959aab27c0a5f9..2c4375f6d6a0c8 100644 --- a/src/coreclr/jit/instrsxarch.h +++ b/src/coreclr/jit/instrsxarch.h @@ -986,7 +986,7 @@ INST3(vpshufbitqmb, "vpshufbitqmb", IUM_WR, BAD_CODE, BAD_ // Instructions for AVX512-BF16, AVX512-FP16 INST3(vaddph, "vaddph", IUM_WR, BAD_CODE, BAD_CODE, PCKFLTMAP(0x05, 0x58), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Add Packed FP16 Values -INST3(vaddsh, "vaddsh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x58), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Add Scalar FP16 Values +INST3(vaddsh, "vaddsh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x58), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Add Scalar FP16 Values INST3(vcmpph, "vcmpph", IUM_WR, BAD_CODE, BAD_CODE, PSSE3A(0x00, 0xC2), 3C, 1C, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Compare Packed FP16 Values INST3(vcmpsh, "vcmpsh", IUM_WR, BAD_CODE, BAD_CODE, PSSE3A(0xF3, 0xC2), 3C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Compare Scalar FP16 Values INST3(vcomish, "vcomish", IUM_RD, BAD_CODE, BAD_CODE, PCKFLTMAP(0x05, 0x2F), 3C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | REX_W0 | Encoding_EVEX) // Compare Scalar Ordered FP16 Values and Set EFLAGS @@ -1004,16 +1004,16 @@ INST3(vcvtph2uw, "vcvtph2uw", IUM_WR, BAD_CODE, BAD_ INST3(vcvtph2w, "vcvtph2w", IUM_WR, BAD_CODE, BAD_CODE, PCKDBLMAP(0x05, 0x7D), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Convert Packed FP16 Values to Packed Signed WORD Integers INST3(vcvtps2phx, "vcvtps2phx", IUM_WR, BAD_CODE, BAD_CODE, PCKDBLMAP(0x05, 0x1D), ILLEGAL, ILLEGAL, INS_TT_FULL, Input_32Bit | KMask_Base4 | REX_W0 | Encoding_EVEX) // Convert Packed Single Precision FP Values to Packed FP16 Values INST3(vcvtqq2ph, "vcvtqq2ph", IUM_WR, BAD_CODE, BAD_CODE, PCKFLTMAP(0x05, 0x5B), ILLEGAL, ILLEGAL, INS_TT_FULL, Input_64Bit | KMask_Base2 | REX_W1 | Encoding_EVEX) // Convert Packed Signed QWORD Integers to Packed FP16 Values -INST3(vcvtsd2sh, "vcvtsd2sh", IUM_WR, BAD_CODE, BAD_CODE, SSEDBLMAP(0x05, 0x5A), 7C, 1C, INS_TT_TUPLE1_SCALAR, Input_64Bit | KMask_Base1 | REX_W1 | Encoding_EVEX) // Convert Scalar Double Precision FP Value to Scalar FP16 Value -INST3(vcvtsh2sd, "vcvtsh2sd", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x5A), 7C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Convert Scalar FP16 Value to Scalar Double Precision FP Value +INST3(vcvtsd2sh, "vcvtsd2sh", IUM_WR, BAD_CODE, BAD_CODE, SSEDBLMAP(0x05, 0x5A), 7C, 1C, INS_TT_TUPLE1_SCALAR, Input_64Bit | KMask_Base1 | REX_W1 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Convert Scalar Double Precision FP Value to Scalar FP16 Value +INST3(vcvtsh2sd, "vcvtsh2sd", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x5A), 7C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Convert Scalar FP16 Value to Scalar Double Precision FP Value INST3(vcvtsh2si32, "vcvtsh2si", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x2D), 8C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Convert Scalar FP16 Value to Scalar Signed DWORD Integer INST3(vcvtsh2si64, "vcvtsh2si", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x2D), 8C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W1 | Encoding_EVEX) // Convert Scalar FP16 Value to Scalar Signed QWORD Integer -INST3(vcvtsh2ss, "vcvtsh2ss", IUM_WR, BAD_CODE, BAD_CODE, PCKFLTMAP(0x06, 0x13), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Convert Scalar FP16 Value to Scalar Single Precision FP Value +INST3(vcvtsh2ss, "vcvtsh2ss", IUM_WR, BAD_CODE, BAD_CODE, PCKFLTMAP(0x06, 0x13), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Convert Scalar FP16 Value to Scalar Single Precision FP Value INST3(vcvtsh2usi32, "vcvtsh2usi", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x79), 7C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Convert Scalar FP16 Value to Scalar Unsigned DWORD Integer INST3(vcvtsh2usi64, "vcvtsh2usi", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x79), 7C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W1 | Encoding_EVEX) // Convert Scalar FP16 Value to Scalar Unsigned QWORD Integer -INST3(vcvtsi2sh32, "vcvtsi2sh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x2A), 7C, 1C, INS_TT_TUPLE1_SCALAR, Input_32Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Convert Scalar Signed DWORD Integer to Scalar FP16 Value -INST3(vcvtsi2sh64, "vcvtsi2sh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x2A), 7C, 1C, INS_TT_TUPLE1_SCALAR, Input_64Bit | KMask_Base1 | REX_W1 | Encoding_EVEX) // Convert Scalar Signed QWORD Integer to Scalar FP16 Value -INST3(vcvtss2sh, "vcvtss2sh", IUM_WR, BAD_CODE, BAD_CODE, PCKFLTMAP(0x05, 0x1D), 4C, 1C, INS_TT_TUPLE1_SCALAR, Input_32Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Convert Scalar Single Precision FP Value to Scalar FP16 Value +INST3(vcvtsi2sh32, "vcvtsi2sh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x2A), 7C, 1C, INS_TT_TUPLE1_SCALAR, Input_32Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Convert Scalar Signed DWORD Integer to Scalar FP16 Value +INST3(vcvtsi2sh64, "vcvtsi2sh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x2A), 7C, 1C, INS_TT_TUPLE1_SCALAR, Input_64Bit | KMask_Base1 | REX_W1 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Convert Scalar Signed QWORD Integer to Scalar FP16 Value +INST3(vcvtss2sh, "vcvtss2sh", IUM_WR, BAD_CODE, BAD_CODE, PCKFLTMAP(0x05, 0x1D), 4C, 1C, INS_TT_TUPLE1_SCALAR, Input_32Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Convert Scalar Single Precision FP Value to Scalar FP16 Value INST3(vcvttph2dq, "vcvttph2dq", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x5B), 8C, 1C, INS_TT_HALF_MEM, Input_16Bit | KMask_Base4 | REX_W0 | Encoding_EVEX) // Convert with Truncation Packed FP16 Values to Packed Signed DWORD Integers INST3(vcvttph2qq, "vcvttph2qq", IUM_WR, BAD_CODE, BAD_CODE, PCKDBLMAP(0x05, 0x7A), ILLEGAL, ILLEGAL, INS_TT_QUARTER_MEM, Input_16Bit | KMask_Base2 | REX_W0 | Encoding_EVEX) // Convert with Truncation Packed FP16 Values to Packed Signed QWORD Integers INST3(vcvttph2udq, "vcvttph2udq", IUM_WR, BAD_CODE, BAD_CODE, PCKFLTMAP(0x05, 0x78), 8C, 1C, INS_TT_HALF_MEM, Input_16Bit | KMask_Base4 | REX_W0 | Encoding_EVEX) // Convert with Truncation Packed FP16 Values to Packed Unsigned DWORD Integers @@ -1026,33 +1026,34 @@ INST3(vcvttsh2usi32, "vcvttsh2usi", IUM_WR, BAD_CODE, BAD_ INST3(vcvttsh2usi64, "vcvttsh2usi", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x78), 8C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W1 | Encoding_EVEX) // Convert with Truncation Scalar FP16 Value to Scalar Unsigned QWORD Integer INST3(vcvtudq2ph, "vcvtudq2ph", IUM_WR, BAD_CODE, BAD_CODE, SSEDBLMAP(0x05, 0x7A), ILLEGAL, ILLEGAL, INS_TT_FULL, Input_32Bit | KMask_Base4 | REX_W0 | Encoding_EVEX) // Convert Packed Single Precision FP Values to Packed FP16 Values INST3(vcvtuqq2ph, "vcvtuqq2ph", IUM_WR, BAD_CODE, BAD_CODE, SSEDBLMAP(0x05, 0x7A), ILLEGAL, ILLEGAL, INS_TT_FULL, Input_64Bit | KMask_Base2 | REX_W1 | Encoding_EVEX) // Convert Packed Single Precision FP Values to Packed FP16 Values -INST3(vcvtusi2sh32, "vcvtusi2sh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x7B), 7C, 1C, INS_TT_TUPLE1_SCALAR, Input_32Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Convert Scalar Unsigned DWORD Integer to Scalar FP16 Value -INST3(vcvtusi2sh64, "vcvtusi2sh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x7B), 7C, 1C, INS_TT_TUPLE1_SCALAR, Input_64Bit | KMask_Base1 | REX_W1 | Encoding_EVEX) // Convert Scalar Unsigned QWORD Integer to Scalar FP16 Value +INST3(vcvtusi2sh32, "vcvtusi2sh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x7B), 7C, 1C, INS_TT_TUPLE1_SCALAR, Input_32Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Convert Scalar Unsigned DWORD Integer to Scalar FP16 Value +INST3(vcvtusi2sh64, "vcvtusi2sh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x7B), 7C, 1C, INS_TT_TUPLE1_SCALAR, Input_64Bit | KMask_Base1 | REX_W1 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Convert Scalar Unsigned QWORD Integer to Scalar FP16 Value INST3(vcvtuw2ph, "vcvtuw2ph", IUM_WR, BAD_CODE, BAD_CODE, SSEDBLMAP(0x05, 0x7D), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Convert Packed Single Precision FP Values to Packed FP16 Values INST3(vcvtw2ph, "vcvtw2ph", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x7D), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Convert Packed Single Precision FP Values to Packed FP16 Values INST3(vdivph, "vdivph", IUM_WR, BAD_CODE, BAD_CODE, PCKFLTMAP(0x05, 0x5E), ILLEGAL, ILLEGAL, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Divide Packed FP16 Values -INST3(vdivsh, "vdivsh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x5E), ILLEGAL, ILLEGAL, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Divide Scalar FP16 Values +INST3(vdivsh, "vdivsh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x5E), ILLEGAL, ILLEGAL, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Divide Scalar FP16 Values INST3(vdpbf16ps, "vdpbf16ps", IUM_WR, BAD_CODE, BAD_CODE, PSSE38(0xF3, 0x52), 8C, 2C, INS_TT_FULL, Input_16Bit | KMask_Base4 | REX_W0 | Encoding_EVEX) // Dot Product of BF16 Pairs Accumulated Into Packed Single Precision INST3(vfcmaddcph, "vfcmaddcph", IUM_RW, BAD_CODE, BAD_CODE, SSEDBLMAP(0x06, 0x56), 8C, 1C, INS_TT_FULL, Input_32Bit | KMask_Base4 | REX_W0 | Encoding_EVEX) // Complex Multiply and Accumulate Packed FP16 Values INST3(vfcmaddcsh, "vfcmaddcsh", IUM_RW, BAD_CODE, BAD_CODE, SSEDBLMAP(0x06, 0x57), 8C, 1C, INS_TT_TUPLE1_SCALAR, Input_32Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Complex Multiply and Accumulate Scalar FP16 Values INST3(vfcmulcph, "vfcmulcph", IUM_WR, BAD_CODE, BAD_CODE, SSEDBLMAP(0x06, 0xD6), 8C, 1C, INS_TT_FULL, Input_32Bit | KMask_Base4 | REX_W0 | Encoding_EVEX) // Complex Multiply Packed FP16 Values INST3(vfcmulcsh, "vfcmulcsh", IUM_WR, BAD_CODE, BAD_CODE, SSEDBLMAP(0x06, 0xD7), 8C, 1C, INS_TT_TUPLE1_SCALAR, Input_32Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Complex Multiply Scalar FP16 Values +#define FIRST_AVX10V1_FMA_INSTR INS_vfmadd132ph INST3(vfmadd132ph, "vvfmadd132ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x98), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Multiply-Add of Packed FP16 Values -INST3(vfmadd132sh, "vvfmadd132sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x99), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Multiply-Add of Scalar FP16 Values INST3(vfmadd213ph, "vvfmadd213ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xA8), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Multiply-Add of Packed FP16 Values -INST3(vfmadd213sh, "vvfmadd213sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xA9), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Multiply-Add of Scalar FP16 Values INST3(vfmadd231ph, "vvfmadd231ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xB8), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Multiply-Add of Packed FP16 Values -INST3(vfmadd231sh, "vvfmadd231sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xB9), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Multiply-Add of Scalar FP16 Values +INST3(vfmadd132sh, "vfmadd132sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x99), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Multiply-Add of Scalar FP16 Values +INST3(vfmadd213sh, "vfmadd213sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xA9), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstDstSrcAVXInstruction) // Fused Multiply-Add of Scalar FP16 Values +INST3(vfmadd231sh, "vfmadd231sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xB9), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Multiply-Add of Scalar FP16 Values INST3(vfmaddcph, "vfmaddcph", IUM_RW, BAD_CODE, BAD_CODE, SSEFLTMAP(0x06, 0x56), 8C, 1C, INS_TT_FULL, Input_32Bit | KMask_Base4 | REX_W0 | Encoding_EVEX) // Complex Multiply and Accumulate Packed FP16 Values INST3(vfmaddcsh, "vfmaddcsh", IUM_RW, BAD_CODE, BAD_CODE, SSEFLTMAP(0x06, 0x57), 8C, 1C, INS_TT_TUPLE1_SCALAR, Input_32Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Complex Multiply and Accumulate Scalar FP16 Values INST3(vfmaddsub132ph, "vvfmaddsub132ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x96), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Multiply-Alternating Add/Subtract of Packed FP16 Values INST3(vfmaddsub213ph, "vvfmaddsub213ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xA6), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Multiply-Alternating Add/Subtract of Packed FP16 Values INST3(vfmaddsub231ph, "vvfmaddsub231ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xB6), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Multiply-Alternating Add/Subtract of Packed FP16 Values INST3(vfmsub132ph, "vvfmsub132ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x9A), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Multiply-Subtract of Packed FP16 Values -INST3(vfmsub132sh, "vvfmsub132sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x9B), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Multiply-Subtract of Scalar FP16 Values INST3(vfmsub213ph, "vvfmsub213ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xAA), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Multiply-Subtract of Packed FP16 Values -INST3(vfmsub213sh, "vvfmsub213sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xAB), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Multiply-Subtract of Scalar FP16 Values INST3(vfmsub231ph, "vvfmsub231ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xBA), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Multiply-Subtract of Packed FP16 Values +INST3(vfmsub132sh, "vvfmsub132sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x9B), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Multiply-Subtract of Scalar FP16 Values +INST3(vfmsub213sh, "vvfmsub213sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xAB), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Multiply-Subtract of Scalar FP16 Values INST3(vfmsub231sh, "vvfmsub231sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xBB), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Multiply-Subtract of Scalar FP16 Values INST3(vfmsubadd132ph, "vvfmsubadd132ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x97), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Multiply-Alternating Subtract/Add of Packed FP16 Values INST3(vfmsubadd213ph, "vvfmsubadd213ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xA7), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Multiply-Alternating Subtract/Add of Packed FP16 Values @@ -1060,17 +1061,18 @@ INST3(vfmsubadd231ph, "vvfmsubadd231ph", IUM_RW, BAD_CODE, BAD_ INST3(vfmulcph, "vfmulcph", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x06, 0xD6), 8C, 1C, INS_TT_FULL, Input_32Bit | KMask_Base4 | REX_W0 | Encoding_EVEX) // Complex Multiply Packed FP16 Values INST3(vfmulcsh, "vfmulcsh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x06, 0xD7), 8C, 1C, INS_TT_TUPLE1_SCALAR, Input_32Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Complex Multiply Scalar FP16 Values INST3(vfnmadd132ph, "vvfnmadd132ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x9C), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Add of Packed FP16 Values -INST3(vfnmadd132sh, "vvfnmadd132sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x9D), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Add of Scalar FP16 Values INST3(vfnmadd213ph, "vvfnmadd213ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xAC), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Add of Packed FP16 Values -INST3(vfnmadd213sh, "vvfnmadd213sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xAD), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Add of Scalar FP16 Values INST3(vfnmadd231ph, "vvfnmadd231ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xBC), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Add of Packed FP16 Values +INST3(vfnmadd132sh, "vvfnmadd132sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x9D), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Add of Scalar FP16 Values +INST3(vfnmadd213sh, "vvfnmadd213sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xAD), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Add of Scalar FP16 Values INST3(vfnmadd231sh, "vvfnmadd231sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xBD), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Add of Scalar FP16 Values INST3(vfnmsub132ph, "vvfnmsub132ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x9E), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Subtract of Packed FP16 Values -INST3(vfnmsub132sh, "vvfnmsub132sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x9F), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Subtract of Scalar FP16 Values INST3(vfnmsub213ph, "vvfnmsub213ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xAE), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Subtract of Packed FP16 Values -INST3(vfnmsub213sh, "vvfnmsub213sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xAF), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Subtract of Scalar FP16 Values INST3(vfnmsub231ph, "vvfnmsub231ph", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xBE), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Subtract of Packed FP16 Values +INST3(vfnmsub132sh, "vvfnmsub132sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x9F), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Subtract of Scalar FP16 Values +INST3(vfnmsub213sh, "vvfnmsub213sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xAF), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Subtract of Scalar FP16 Values INST3(vfnmsub231sh, "vvfnmsub231sh", IUM_RW, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0xBF), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Fused Negative Multiply-Subtract of Scalar FP16 Values +#define LAST_AVX10V1_FMA_INSTR INS_vfnmsub231sh INST3(vfpclassph, "vfpclassph", IUM_WR, BAD_CODE, BAD_CODE, PSSE3A(0x00, 0x66), 3C, 1C, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Test Types of Packed FP16 Values INST3(vfpclasssh, "vfpclasssh", IUM_WR, BAD_CODE, BAD_CODE, PSSE3A(0x00, 0x67), 3C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Test Types of Scalar FP16 Values INST3(vgetexpph, "vgetexpph", IUM_WR, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x42), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Convert Exponents of Packed FP16 Values to FP16 Values @@ -1078,27 +1080,27 @@ INST3(vgetexpsh, "vgetexpsh", IUM_WR, BAD_CODE, BAD_ INST3(vgetmantph, "vgetmantph", IUM_WR, BAD_CODE, BAD_CODE, PSSE3A(0x00, 0x26), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Extract Normalized Mantissas from Packed FP16 Values INST3(vgetmantsh, "vgetmantsh", IUM_WR, BAD_CODE, BAD_CODE, PSSE3A(0x00, 0x27), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Extract Normalized Mantissas from Scalar FP16 Values INST3(vmaxph, "vmaxph", IUM_WR, BAD_CODE, BAD_CODE, PCKFLTMAP(0x05, 0x5F), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Return Maximum of Packed FP16 Values -INST3(vmaxsh, "vmaxsh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x5F), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Return Maximum of Scalar FP16 Values -INST3(vminsh, "vminsh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x00, 0x5D), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Return Minimum of Scalar FP16 Values -INST3(vminph, "vminph", IUM_WR, BAD_CODE, BAD_CODE, PCKFLTMAP(0x00, 0x5D), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Return Minimum of Packed FP16 Values -INST3(vmovsh, "vmovsh", IUM_WR, SSEFLTMAP(0x00, 0x11), BAD_CODE, SSEFLTMAP(0x00, 0x10), ILLEGAL, ILLEGAL, INS_TT_TUPLE1_SCALAR, Input_16Bit | REX_W0 | Encoding_EVEX) // Move Scalar FP16 Value +INST3(vmaxsh, "vmaxsh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x5F), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Return Maximum of Scalar FP16 Values +INST3(vminsh, "vminsh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x5D), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Return Minimum of Scalar FP16 Values +INST3(vminph, "vminph", IUM_WR, BAD_CODE, BAD_CODE, PCKFLTMAP(0x05, 0x5D), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Return Minimum of Packed FP16 Values +INST3(vmovsh, "vmovsh", IUM_WR, SSEFLTMAP(0x05, 0x11), BAD_CODE, SSEFLTMAP(0x05, 0x10), ILLEGAL, ILLEGAL, INS_TT_TUPLE1_SCALAR, Input_16Bit | REX_W0 | Encoding_EVEX) // Move Scalar FP16 Value INST3(vmovw, "vmovw", IUM_WR, PCKDBLMAP(0x06, 0x7E), BAD_CODE, PCKDBLMAP(0x00, 0x6E), ILLEGAL, ILLEGAL, INS_TT_TUPLE1_SCALAR, Input_16Bit | REX_WIG | Encoding_EVEX) // Move Word INST3(vmulph, "vmulph", IUM_WR, BAD_CODE, BAD_CODE, PCKFLTMAP(0x05, 0x59), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Multiply Packed FP16 Values -INST3(vmulsh, "vmulsh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x59), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Multiply Scalar FP16 Values +INST3(vmulsh, "vmulsh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x59), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Multiply Scalar FP16 Values INST3(vrcpph, "vrcpph", IUM_WR, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x4C), ILLEGAL, ILLEGAL, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Compute REciprocals of Packed FP16 Values -INST3(vrcpsh, "vrcpsh", IUM_WR, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x4D), 4C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Compute REciprocals of Scalar FP16 Values +INST3(vrcpsh, "vrcpsh", IUM_WR, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x4D), 4C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Compute REciprocals of Scalar FP16 Values INST3(vreduceph, "vreduceph", IUM_WR, BAD_CODE, BAD_CODE, PSSE3A(0x00, 0x56), 12C, 2C, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Perform Reduction Transformation on Packed FP16 Values INST3(vreducesh, "vreducesh", IUM_WR, BAD_CODE, BAD_CODE, PSSE3A(0x00, 0x57), 12C, 2C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Perform Reduction Transformation on Scalar FP16 Values INST3(vrndscaleph, "vrndscaleph", IUM_WR, BAD_CODE, BAD_CODE, PSSE3A(0x00, 0x08), 8C, 1C, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Round Packed FP16 Values to Include a Given Number of Fraction Bits -INST3(vrndscalesh, "vrndscalesh", IUM_WR, BAD_CODE, BAD_CODE, PSSE3A(0x00, 0x0A), 8C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Round Scalar FP16 Values to Include a Given Number of Fraction Bits +INST3(vrndscalesh, "vrndscalesh", IUM_WR, BAD_CODE, BAD_CODE, PSSE3A(0x00, 0x0A), 8C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Round Scalar FP16 Values to Include a Given Number of Fraction Bits INST3(vrsqrtph, "vrsqrtph", IUM_WR, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x4E), ILLEGAL, ILLEGAL, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Compute REciprocals of Square Roots of Packed FP16 Values -INST3(vrsqrtsh, "vrsqrtsh", IUM_WR, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x4F), 5C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Compute REciprocals of Square Roots of Scalar FP16 Values +INST3(vrsqrtsh, "vrsqrtsh", IUM_WR, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x4F), 5C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Compute REciprocals of Square Roots of Scalar FP16 Values INST3(vscalefph, "vscalefph", IUM_WR, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x2C), 8C, 1C, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Scale Packed FP16 Values with FP16 Values INST3(vscalefsh, "vscalefsh", IUM_WR, BAD_CODE, BAD_CODE, PCKDBLMAP(0x06, 0x2D), 8C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Scale Scalar FP16 Values with FP16 Values INST3(vsqrtph, "vsqrtph", IUM_WR, BAD_CODE, BAD_CODE, PCKFLTMAP(0x05, 0x51), ILLEGAL, ILLEGAL, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Compute Square Root of Packed FP16 Values -INST3(vsqrtsh, "vsqrtsh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x51), 15C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Compute Square Root of Scalar FP16 Values +INST3(vsqrtsh, "vsqrtsh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x51), 15C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Compute Square Root of Scalar FP16 Values INST3(vsubph, "vsubph", IUM_WR, BAD_CODE, BAD_CODE, PCKFLTMAP(0x05, 0x5C), 4C, 2X, INS_TT_FULL_MEM, Input_16Bit | KMask_Base8 | REX_W0 | Encoding_EVEX) // Subtract Packed FP16 Values -INST3(vsubsh, "vsubsh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x5C), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX) // Subtract Scalar FP16 Values +INST3(vsubsh, "vsubsh", IUM_WR, BAD_CODE, BAD_CODE, SSEFLTMAP(0x05, 0x5C), 4C, 2X, INS_TT_TUPLE1_SCALAR, Input_16Bit | KMask_Base1 | REX_W0 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Subtract Scalar FP16 Values INST3(vucomish, "vucomish", IUM_RD, BAD_CODE, BAD_CODE, PCKFLTMAP(0x05, 0x2E), 3C, 1C, INS_TT_TUPLE1_SCALAR, Input_16Bit | REX_W0 | Encoding_EVEX) // Compare Scalar Unordered FP16 Values and Set EFLAGS // AVX512-VP2INTERSECT diff --git a/src/coreclr/jit/jitconfigvalues.h b/src/coreclr/jit/jitconfigvalues.h index bedf9e217c71c5..80e9c0647c8273 100644 --- a/src/coreclr/jit/jitconfigvalues.h +++ b/src/coreclr/jit/jitconfigvalues.h @@ -441,6 +441,7 @@ RELEASE_CONFIG_INTEGER(EnableArm64Atomics, "EnableArm64Atomics", RELEASE_CONFIG_INTEGER(EnableArm64Crc32, "EnableArm64Crc32", 1) // Allows Arm64 Crc32+ hardware intrinsics to be disabled RELEASE_CONFIG_INTEGER(EnableArm64Dczva, "EnableArm64Dczva", 1) // Allows Arm64 Dczva+ hardware intrinsics to be disabled RELEASE_CONFIG_INTEGER(EnableArm64Dp, "EnableArm64Dp", 1) // Allows Arm64 Dp+ hardware intrinsics to be disabled +RELEASE_CONFIG_INTEGER(EnableArm64Fp16, "EnableArm64Fp16", 1) // Allows Arm64 Fp16+ hardware intrinsics to be disabled RELEASE_CONFIG_INTEGER(EnableArm64Rdm, "EnableArm64Rdm", 1) // Allows Arm64 Rdm+ hardware intrinsics to be disabled RELEASE_CONFIG_INTEGER(EnableArm64Sha1, "EnableArm64Sha1", 1) // Allows Arm64 Sha1+ hardware intrinsics to be disabled RELEASE_CONFIG_INTEGER(EnableArm64Sha256, "EnableArm64Sha256", 1) // Allows Arm64 Sha256+ hardware intrinsics to be disabled diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index 0359b806c6e1bc..e4b41d92fa57b7 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -1113,6 +1113,8 @@ void Lowering::LowerHWIntrinsicCC(GenTreeHWIntrinsic* node, NamedIntrinsic newIn { case NI_X86Base_COMIS: case NI_X86Base_UCOMIS: + case NI_AVX10v1_VCOMISH: + case NI_AVX10v1_VUCOMISH: // In some cases we can generate better code if we swap the operands: // - If the condition is not one of the "preferred" floating point conditions we can swap // the operands and change the condition to avoid generating an extra JP/JNP branch. @@ -2380,6 +2382,44 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) LowerHWIntrinsicCC(node, NI_X86Base_UCOMIS, GenCondition::FGT); break; + case NI_AVX10v1_CompareScalarOrderedEqual: + LowerHWIntrinsicCC(node, NI_AVX10v1_VCOMISH, GenCondition::FEQ); + break; + case NI_AVX10v1_CompareScalarOrderedNotEqual: + LowerHWIntrinsicCC(node, NI_AVX10v1_VCOMISH, GenCondition::FNEU); + break; + case NI_AVX10v1_CompareScalarOrderedLessThan: + LowerHWIntrinsicCC(node, NI_AVX10v1_VCOMISH, GenCondition::FLT); + break; + case NI_AVX10v1_CompareScalarOrderedLessThanOrEqual: + LowerHWIntrinsicCC(node, NI_AVX10v1_VCOMISH, GenCondition::FLE); + break; + case NI_AVX10v1_CompareScalarOrderedGreaterThan: + LowerHWIntrinsicCC(node, NI_AVX10v1_VCOMISH, GenCondition::FGT); + break; + case NI_AVX10v1_CompareScalarOrderedGreaterThanOrEqual: + LowerHWIntrinsicCC(node, NI_AVX10v1_VCOMISH, GenCondition::FGE); + break; + + case NI_AVX10v1_CompareScalarUnorderedEqual: + LowerHWIntrinsicCC(node, NI_AVX10v1_VUCOMISH, GenCondition::FEQ); + break; + case NI_AVX10v1_CompareScalarUnorderedNotEqual: + LowerHWIntrinsicCC(node, NI_AVX10v1_VUCOMISH, GenCondition::FNEU); + break; + case NI_AVX10v1_CompareScalarUnorderedLessThan: + LowerHWIntrinsicCC(node, NI_AVX10v1_VUCOMISH, GenCondition::FLT); + break; + case NI_AVX10v1_CompareScalarUnorderedLessThanOrEqual: + LowerHWIntrinsicCC(node, NI_AVX10v1_VUCOMISH, GenCondition::FLE); + break; + case NI_AVX10v1_CompareScalarUnorderedGreaterThan: + LowerHWIntrinsicCC(node, NI_AVX10v1_VUCOMISH, GenCondition::FGT); + break; + case NI_AVX10v1_CompareScalarUnorderedGreaterThanOrEqual: + LowerHWIntrinsicCC(node, NI_AVX10v1_VUCOMISH, GenCondition::FGE); + break; + case NI_X86Base_TestC: LowerHWIntrinsicCC(node, NI_X86Base_PTEST, GenCondition::C); break; @@ -9535,6 +9575,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) case NI_AVX512_GetMantissaScalar: case NI_AVX512_RoundScaleScalar: case NI_AVX512_ReduceScalar: + case NI_AVX10v1_RoundScaleScalar: { // These intrinsics have both 2 and 3-operand overloads. // @@ -10224,6 +10265,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) case NI_AES_CarrylessMultiply: case NI_AES_V256_CarrylessMultiply: case NI_AES_V512_CarrylessMultiply: + case NI_AVX10v1_RoundScaleScalar: case NI_AVX10v2_MinMax: case NI_AVX10v2_MinMaxScalar: case NI_AVX10v2_MultipleSumAbsoluteDifferences: diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index 58a9e2cbef9cc6..6cfd53c2e4206e 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -2489,6 +2489,7 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou case NI_AVX512_FusedMultiplySubtractAdd: case NI_AVX512_FusedMultiplySubtractNegated: case NI_AVX512_FusedMultiplySubtractNegatedScalar: + case NI_AVX10v1_FusedMultiplyAddScalar: { // While this operation is RMW, it is also almost freely reorderable // and so we do not need to set the operands as delay free unless diff --git a/src/coreclr/jit/namedintrinsiclist.h b/src/coreclr/jit/namedintrinsiclist.h index 0560d07c142b0b..1ab2b2e14ab05f 100644 --- a/src/coreclr/jit/namedintrinsiclist.h +++ b/src/coreclr/jit/namedintrinsiclist.h @@ -25,7 +25,38 @@ enum NamedIntrinsic : unsigned short NI_System_SpanHelpers_Memmove, + NI_System_Half_FusedMultiplyAdd, + NI_System_Half_ReciprocalEstimate, + NI_System_Half_ReciprocalSqrtEstimate, + NI_System_Half_Round, + NI_System_Half_Sqrt, + + NI_System_Half_op_Addition, + NI_System_Half_op_Decrement, + NI_System_Half_op_Division, + NI_System_Half_op_Equality, NI_System_Half_op_Explicit, + NI_System_Half_op_GreaterThan, + NI_System_Half_op_GreaterThanOrEqual, + NI_System_Half_op_Increment, + NI_System_Half_op_Inequality, + NI_System_Half_op_LessThan, + NI_System_Half_op_LessThanOrEqual, + NI_System_Half_op_Multiply, + NI_System_Half_op_Subtraction, + + NI_System_Half_get_MinValue, + NI_System_Half_get_MaxValue, + NI_System_Half_get_Epsilon, + NI_System_Half_get_NaN, + NI_System_Half_get_PositiveInfinity, + NI_System_Half_get_NegativeInfinity, + NI_System_Half_get_One, + NI_System_Half_get_Zero, + + NI_System_Half_Ceiling, + NI_System_Half_Floor, + NI_System_Half_Truncate, NI_SYSTEM_MATH_START, NI_System_Math_Abs, diff --git a/src/coreclr/jit/valuenumfuncs.h b/src/coreclr/jit/valuenumfuncs.h index 7ff916a2def4a7..956c460fc13282 100644 --- a/src/coreclr/jit/valuenumfuncs.h +++ b/src/coreclr/jit/valuenumfuncs.h @@ -193,7 +193,7 @@ ValueNumFuncDef(SimdType, 2, false, false) // A value number function to compos ValueNumFuncDef(HWI_INTRINSIC_START, -1, false, false) #define HARDWARE_INTRINSIC(isa, name, simdSize, numArgs, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, intCost, fltCost, category, flag) \ -ValueNumFuncDef(HWI_##isa##_##name, ((numArgs == -1) ? -1 : (numArgs + 1)), ((flag) & HW_Flag_Commutative) >> 0, false) // All of the HARDWARE_INTRINSICS for x86/x64 +ValueNumFuncDef(HWI_##isa##_##name, ((numArgs == -1) ? -1 : (numArgs + 1)), ((flag) & HW_Flag_Commutative) >> 0, false) // All of the HARDWARE_INTRINSICS #include "hwintrinsiclist.h" ValueNumFuncDef(HWI_INTRINSIC_END, -1, false, false) diff --git a/src/coreclr/tools/Common/Compiler/HardwareIntrinsicHelpers.cs b/src/coreclr/tools/Common/Compiler/HardwareIntrinsicHelpers.cs index e40bc64c73a289..2044f35875349f 100644 --- a/src/coreclr/tools/Common/Compiler/HardwareIntrinsicHelpers.cs +++ b/src/coreclr/tools/Common/Compiler/HardwareIntrinsicHelpers.cs @@ -232,6 +232,7 @@ private static class Arm64IntrinsicConstants public const int SveAes = (1 << 13); public const int SveSha3 = (1 << 14); public const int SveSm4 = (1 << 15); + public const int Fp16 = (1 << 16); public static void AddToBuilder(InstructionSetSupportBuilder builder, int flags) { @@ -267,6 +268,8 @@ public static void AddToBuilder(InstructionSetSupportBuilder builder, int flags) builder.AddSupportedInstructionSet("sve_sha3"); if ((flags & SveSm4) != 0) builder.AddSupportedInstructionSet("sve_sm4"); + if ((flags & Fp16) != 0) + builder.AddSupportedInstructionSet("fp16"); } public static int FromInstructionSet(InstructionSet instructionSet) @@ -289,6 +292,8 @@ public static int FromInstructionSet(InstructionSet instructionSet) InstructionSet.ARM64_Dp_Arm64 => Dp, InstructionSet.ARM64_Rdm => Rdm, InstructionSet.ARM64_Rdm_Arm64 => Rdm, + InstructionSet.ARM64_Fp16 => Fp16, + InstructionSet.ARM64_Fp16_Arm64 => Fp16, InstructionSet.ARM64_Sha1 => Sha1, InstructionSet.ARM64_Sha1_Arm64 => Sha1, InstructionSet.ARM64_Sha256 => Sha256, diff --git a/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSet.cs b/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSet.cs index 14e8c9585a0495..c0c0b53008603d 100644 --- a/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSet.cs +++ b/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSet.cs @@ -103,5 +103,6 @@ public enum ReadyToRunInstructionSet SveSm4 = 90, WasmBase = 91, PackedSimd = 92, + Fp16 = 93, } } diff --git a/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSetHelper.cs b/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSetHelper.cs index 196200fc55434f..c3a74601719dac 100644 --- a/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSetHelper.cs +++ b/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSetHelper.cs @@ -34,6 +34,8 @@ public static class ReadyToRunInstructionSetHelper case InstructionSet.ARM64_Dp_Arm64: return ReadyToRunInstructionSet.Dp; case InstructionSet.ARM64_Rdm: return ReadyToRunInstructionSet.Rdm; case InstructionSet.ARM64_Rdm_Arm64: return ReadyToRunInstructionSet.Rdm; + case InstructionSet.ARM64_Fp16: return ReadyToRunInstructionSet.Fp16; + case InstructionSet.ARM64_Fp16_Arm64: return ReadyToRunInstructionSet.Fp16; case InstructionSet.ARM64_Sha1: return ReadyToRunInstructionSet.Sha1; case InstructionSet.ARM64_Sha1_Arm64: return ReadyToRunInstructionSet.Sha1; case InstructionSet.ARM64_Sha256: return ReadyToRunInstructionSet.Sha256; diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs b/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs index 264d2cd0bdbcd6..c83e4ae21ea413 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs @@ -23,6 +23,7 @@ public enum InstructionSet ARM64_Crc32 = InstructionSet_ARM64.Crc32, ARM64_Dp = InstructionSet_ARM64.Dp, ARM64_Rdm = InstructionSet_ARM64.Rdm, + ARM64_Fp16 = InstructionSet_ARM64.Fp16, ARM64_Sha1 = InstructionSet_ARM64.Sha1, ARM64_Sha256 = InstructionSet_ARM64.Sha256, ARM64_Atomics = InstructionSet_ARM64.Atomics, @@ -46,6 +47,7 @@ public enum InstructionSet ARM64_Crc32_Arm64 = InstructionSet_ARM64.Crc32_Arm64, ARM64_Dp_Arm64 = InstructionSet_ARM64.Dp_Arm64, ARM64_Rdm_Arm64 = InstructionSet_ARM64.Rdm_Arm64, + ARM64_Fp16_Arm64 = InstructionSet_ARM64.Fp16_Arm64, ARM64_Sha1_Arm64 = InstructionSet_ARM64.Sha1_Arm64, ARM64_Sha256_Arm64 = InstructionSet_ARM64.Sha256_Arm64, ARM64_Sve_Arm64 = InstructionSet_ARM64.Sve_Arm64, @@ -165,38 +167,40 @@ public enum InstructionSet_ARM64 Crc32 = 4, Dp = 5, Rdm = 6, - Sha1 = 7, - Sha256 = 8, - Atomics = 9, - Vector64 = 10, - Vector128 = 11, - VectorT = 12, - Dczva = 13, - Rcpc = 14, - VectorT128 = 15, - Rcpc2 = 16, - Sve = 17, - Sve2 = 18, - Sha3 = 19, - Sm4 = 20, - SveAes = 21, - SveSha3 = 22, - SveSm4 = 23, - ArmBase_Arm64 = 24, - AdvSimd_Arm64 = 25, - Aes_Arm64 = 26, - Crc32_Arm64 = 27, - Dp_Arm64 = 28, - Rdm_Arm64 = 29, - Sha1_Arm64 = 30, - Sha256_Arm64 = 31, - Sve_Arm64 = 32, - Sve2_Arm64 = 33, - Sha3_Arm64 = 34, - Sm4_Arm64 = 35, - SveAes_Arm64 = 36, - SveSha3_Arm64 = 37, - SveSm4_Arm64 = 38, + Fp16 = 7, + Sha1 = 8, + Sha256 = 9, + Atomics = 10, + Vector64 = 11, + Vector128 = 12, + VectorT = 13, + Dczva = 14, + Rcpc = 15, + VectorT128 = 16, + Rcpc2 = 17, + Sve = 18, + Sve2 = 19, + Sha3 = 20, + Sm4 = 21, + SveAes = 22, + SveSha3 = 23, + SveSm4 = 24, + ArmBase_Arm64 = 25, + AdvSimd_Arm64 = 26, + Aes_Arm64 = 27, + Crc32_Arm64 = 28, + Dp_Arm64 = 29, + Rdm_Arm64 = 30, + Fp16_Arm64 = 31, + Sha1_Arm64 = 32, + Sha256_Arm64 = 33, + Sve_Arm64 = 34, + Sve2_Arm64 = 35, + Sha3_Arm64 = 36, + Sm4_Arm64 = 37, + SveAes_Arm64 = 38, + SveSha3_Arm64 = 39, + SveSm4_Arm64 = 40, } public enum InstructionSet_RiscV64 @@ -512,6 +516,10 @@ public static InstructionSetFlags ExpandInstructionSetByImplicationHelper(Target resultflags.AddInstructionSet(InstructionSet.ARM64_Rdm_Arm64); if (resultflags.HasInstructionSet(InstructionSet.ARM64_Rdm_Arm64)) resultflags.AddInstructionSet(InstructionSet.ARM64_Rdm); + if (resultflags.HasInstructionSet(InstructionSet.ARM64_Fp16)) + resultflags.AddInstructionSet(InstructionSet.ARM64_Fp16_Arm64); + if (resultflags.HasInstructionSet(InstructionSet.ARM64_Fp16_Arm64)) + resultflags.AddInstructionSet(InstructionSet.ARM64_Fp16); if (resultflags.HasInstructionSet(InstructionSet.ARM64_Sha1)) resultflags.AddInstructionSet(InstructionSet.ARM64_Sha1_Arm64); if (resultflags.HasInstructionSet(InstructionSet.ARM64_Sha1_Arm64)) @@ -558,6 +566,8 @@ public static InstructionSetFlags ExpandInstructionSetByImplicationHelper(Target resultflags.AddInstructionSet(InstructionSet.ARM64_AdvSimd); if (resultflags.HasInstructionSet(InstructionSet.ARM64_Rdm)) resultflags.AddInstructionSet(InstructionSet.ARM64_AdvSimd); + if (resultflags.HasInstructionSet(InstructionSet.ARM64_Fp16)) + resultflags.AddInstructionSet(InstructionSet.ARM64_AdvSimd); if (resultflags.HasInstructionSet(InstructionSet.ARM64_Sha1)) resultflags.AddInstructionSet(InstructionSet.ARM64_ArmBase); if (resultflags.HasInstructionSet(InstructionSet.ARM64_Sha256)) @@ -835,6 +845,8 @@ private static InstructionSetFlags ExpandInstructionSetByReverseImplicationHelpe resultflags.AddInstructionSet(InstructionSet.ARM64_Dp); if (resultflags.HasInstructionSet(InstructionSet.ARM64_Rdm_Arm64)) resultflags.AddInstructionSet(InstructionSet.ARM64_Rdm); + if (resultflags.HasInstructionSet(InstructionSet.ARM64_Fp16_Arm64)) + resultflags.AddInstructionSet(InstructionSet.ARM64_Fp16); if (resultflags.HasInstructionSet(InstructionSet.ARM64_Sha1_Arm64)) resultflags.AddInstructionSet(InstructionSet.ARM64_Sha1); if (resultflags.HasInstructionSet(InstructionSet.ARM64_Sha256_Arm64)) @@ -863,6 +875,8 @@ private static InstructionSetFlags ExpandInstructionSetByReverseImplicationHelpe resultflags.AddInstructionSet(InstructionSet.ARM64_Dp); if (resultflags.HasInstructionSet(InstructionSet.ARM64_AdvSimd)) resultflags.AddInstructionSet(InstructionSet.ARM64_Rdm); + if (resultflags.HasInstructionSet(InstructionSet.ARM64_AdvSimd)) + resultflags.AddInstructionSet(InstructionSet.ARM64_Fp16); if (resultflags.HasInstructionSet(InstructionSet.ARM64_ArmBase)) resultflags.AddInstructionSet(InstructionSet.ARM64_Sha1); if (resultflags.HasInstructionSet(InstructionSet.ARM64_ArmBase)) @@ -1132,6 +1146,7 @@ public static IEnumerable ArchitectureToValidInstructionSets yield return new InstructionSetInfo("crc", "Crc32", InstructionSet.ARM64_Crc32, true); yield return new InstructionSetInfo("dotprod", "Dp", InstructionSet.ARM64_Dp, true); yield return new InstructionSetInfo("rdma", "Rdm", InstructionSet.ARM64_Rdm, true); + yield return new InstructionSetInfo("fp16", "", InstructionSet.ARM64_Fp16, true); yield return new InstructionSetInfo("sha1", "Sha1", InstructionSet.ARM64_Sha1, true); yield return new InstructionSetInfo("sha2", "Sha256", InstructionSet.ARM64_Sha256, true); yield return new InstructionSetInfo("lse", "", InstructionSet.ARM64_Atomics, true); @@ -1332,6 +1347,8 @@ public void Set64BitInstructionSetVariants(TargetArchitecture architecture) AddInstructionSet(InstructionSet.ARM64_Dp_Arm64); if (HasInstructionSet(InstructionSet.ARM64_Rdm)) AddInstructionSet(InstructionSet.ARM64_Rdm_Arm64); + if (HasInstructionSet(InstructionSet.ARM64_Fp16)) + AddInstructionSet(InstructionSet.ARM64_Fp16_Arm64); if (HasInstructionSet(InstructionSet.ARM64_Sha1)) AddInstructionSet(InstructionSet.ARM64_Sha1_Arm64); if (HasInstructionSet(InstructionSet.ARM64_Sha256)) @@ -1410,6 +1427,7 @@ public void Set64BitInstructionSetVariantsUnconditionally(TargetArchitecture arc AddInstructionSet(InstructionSet.ARM64_Crc32_Arm64); AddInstructionSet(InstructionSet.ARM64_Dp_Arm64); AddInstructionSet(InstructionSet.ARM64_Rdm_Arm64); + AddInstructionSet(InstructionSet.ARM64_Fp16_Arm64); AddInstructionSet(InstructionSet.ARM64_Sha1_Arm64); AddInstructionSet(InstructionSet.ARM64_Sha256_Arm64); AddInstructionSet(InstructionSet.ARM64_Sve_Arm64); diff --git a/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt b/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt index b961d2accdb9e3..411786b5186ac2 100644 --- a/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt +++ b/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt @@ -26,7 +26,7 @@ ; DO NOT CHANGE R2R NUMERIC VALUES OF THE EXISTING SETS. Changing R2R numeric values definitions would be R2R format breaking change. ; The ISA definitions should also be mapped to `hwintrinsicIsaRangeArray` in hwintrinsic.cpp. -; NEXT_AVAILABLE_R2R_BIT = 93 +; NEXT_AVAILABLE_R2R_BIT = 94 ; Definition of X86 instruction sets definearch ,X86 ,32Bit ,X64, X64, X86 @@ -215,6 +215,7 @@ instructionset ,ARM64 ,Aes , ,9 ,Aes instructionset ,ARM64 ,Crc32 , ,18 ,Crc32 ,crc instructionset ,ARM64 ,Dp , ,23 ,Dp ,dotprod instructionset ,ARM64 ,Rdm , ,24 ,Rdm ,rdma +instructionset ,ARM64 , ,Fp16 ,93 ,Fp16 ,fp16 instructionset ,ARM64 ,Sha1 , ,19 ,Sha1 ,sha1 instructionset ,ARM64 ,Sha256 , ,20 ,Sha256 ,sha2 instructionset ,ARM64 , ,Atomics ,21 ,Atomics ,lse @@ -239,6 +240,7 @@ instructionset64bit,ARM64 ,Aes instructionset64bit,ARM64 ,Crc32 instructionset64bit,ARM64 ,Dp instructionset64bit,ARM64 ,Rdm +instructionset64bit,ARM64 ,Fp16 instructionset64bit,ARM64 ,Sha1 instructionset64bit,ARM64 ,Sha256 instructionset64bit,ARM64 ,Sve @@ -258,6 +260,7 @@ implication ,ARM64 ,Aes ,ArmBase implication ,ARM64 ,Crc32 ,ArmBase implication ,ARM64 ,Dp ,AdvSimd implication ,ARM64 ,Rdm ,AdvSimd +implication ,ARM64 ,Fp16 ,AdvSimd implication ,ARM64 ,Sha1 ,ArmBase implication ,ARM64 ,Sha256 ,ArmBase implication ,ARM64 ,Vector64 ,AdvSimd diff --git a/src/coreclr/vm/codeman.cpp b/src/coreclr/vm/codeman.cpp index 896e29d5dba09a..59a82886dc87f4 100644 --- a/src/coreclr/vm/codeman.cpp +++ b/src/coreclr/vm/codeman.cpp @@ -1702,6 +1702,11 @@ void EEJitManager::SetCpuInfo() CPUCompileFlags.Set(InstructionSet_Rdm); } + if (((cpuFeatures & ARM64IntrinsicConstants_Fp16) != 0) && CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_EnableArm64Fp16)) + { + CPUCompileFlags.Set(InstructionSet_Fp16); + } + if (((cpuFeatures & ARM64IntrinsicConstants_Sha1) != 0) && CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_EnableArm64Sha1)) { CPUCompileFlags.Set(InstructionSet_Sha1); diff --git a/src/libraries/System.Private.CoreLib/src/System/Half.cs b/src/libraries/System.Private.CoreLib/src/System/Half.cs index 1b1a8557c8c104..4625db29d88a2b 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Half.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Half.cs @@ -87,19 +87,19 @@ public readonly struct Half // Well-defined and commonly used values - public static Half Epsilon => new Half(EpsilonBits); // 5.9604645E-08 + public static Half Epsilon { [Intrinsic] get => new Half(EpsilonBits); } // 5.9604645E-08 - public static Half PositiveInfinity => new Half(PositiveInfinityBits); // 1.0 / 0.0; + public static Half PositiveInfinity { [Intrinsic] get => new Half(PositiveInfinityBits); } // 1.0 / 0.0; - public static Half NegativeInfinity => new Half(NegativeInfinityBits); // -1.0 / 0.0 + public static Half NegativeInfinity { [Intrinsic] get => new Half(NegativeInfinityBits); } // -1.0 / 0.0 - public static Half NaN => new Half(NegativeQNaNBits); // 0.0 / 0.0 + public static Half NaN { [Intrinsic] get => new Half(NegativeQNaNBits); } // 0.0 / 0.0 /// - public static Half MinValue => new Half(MinValueBits); // -65504 + public static Half MinValue { [Intrinsic] get => new Half(MinValueBits); } // -65504 /// - public static Half MaxValue => new Half(MaxValueBits); // 65504 + public static Half MaxValue { [Intrinsic] get => new Half(MaxValueBits); } // 65504 internal readonly ushort _value; @@ -155,6 +155,7 @@ internal static ushort ExtractTrailingSignificandFromBits(ushort bits) } /// + [Intrinsic] public static bool operator <(Half left, Half right) { if (IsNaN(left) || IsNaN(right)) @@ -177,12 +178,14 @@ internal static ushort ExtractTrailingSignificandFromBits(ushort bits) } /// + [Intrinsic] public static bool operator >(Half left, Half right) { return right < left; } /// + [Intrinsic] public static bool operator <=(Half left, Half right) { if (IsNaN(left) || IsNaN(right)) @@ -205,12 +208,14 @@ internal static ushort ExtractTrailingSignificandFromBits(ushort bits) } /// + [Intrinsic] public static bool operator >=(Half left, Half right) { return right <= left; } /// + [Intrinsic] public static bool operator ==(Half left, Half right) { if (IsNaN(left) || IsNaN(right)) @@ -224,6 +229,7 @@ internal static ushort ExtractTrailingSignificandFromBits(ushort bits) } /// + [Intrinsic] public static bool operator !=(Half left, Half right) { return !(left == right); @@ -563,6 +569,7 @@ public bool TryFormat(Span utf8Destination, out int bytesWritten, [StringS /// Explicitly converts a value to its nearest representable half-precision floating-point value. /// The value to convert. /// converted to its nearest representable half-precision floating-point value. + [Intrinsic] public static explicit operator Half(double value) { const int DoubleMaxExponent = 0x7FF; @@ -597,11 +604,13 @@ public static explicit operator Half(double value) /// Explicitly converts a value to its nearest representable half-precision floating-point value. /// The value to convert. /// converted to its nearest representable half-precision floating-point value. + [Intrinsic] public static explicit operator Half(int value) => (Half)(float)value; /// Explicitly converts a value to its nearest representable half-precision floating-point value. /// The value to convert. /// converted to its nearest representable half-precision floating-point value. + [Intrinsic] public static explicit operator Half(long value) => (Half)(float)value; /// Explicitly converts a value to its nearest representable half-precision floating-point value. @@ -786,12 +795,14 @@ public static explicit operator Half(float value) /// The value to convert. /// converted to its nearest representable half-precision floating-point value. [CLSCompliant(false)] + [Intrinsic] public static explicit operator Half(uint value) => (Half)(float)value; /// Explicitly converts a value to its nearest representable half-precision floating-point value. /// The value to convert. /// converted to its nearest representable half-precision floating-point value. [CLSCompliant(false)] + [Intrinsic] public static explicit operator Half(ulong value) => (Half)(float)value; /// Explicitly converts a value to its nearest representable half-precision floating-point value. @@ -845,6 +856,7 @@ public static explicit operator Half(float value) /// Explicitly converts a half-precision floating-point value to its nearest representable value. /// The value to convert. /// converted to its nearest representable value. + [Intrinsic] public static explicit operator int(Half value) => (int)(float)value; /// Explicitly converts a half-precision floating-point value to its nearest representable value, throwing an overflow exception for any values that fall outside the representable range. @@ -856,6 +868,7 @@ public static explicit operator Half(float value) /// Explicitly converts a half-precision floating-point value to its nearest representable value. /// The value to convert. /// converted to its nearest representable value. + [Intrinsic] public static explicit operator long(Half value) => (long)(float)value; /// Explicitly converts a half-precision floating-point value to its nearest representable value, throwing an overflow exception for any values that fall outside the representable range. @@ -916,6 +929,7 @@ public static explicit operator Half(float value) /// The value to convert. /// converted to its nearest representable value. [CLSCompliant(false)] + [Intrinsic] public static explicit operator uint(Half value) => (uint)(float)value; /// Explicitly converts a half-precision floating-point value to its nearest representable value, throwing an overflow exception for any values that fall outside the representable range. @@ -929,6 +943,7 @@ public static explicit operator Half(float value) /// The value to convert. /// converted to its nearest representable value. [CLSCompliant(false)] + [Intrinsic] public static explicit operator ulong(Half value) => (ulong)(float)value; /// Explicitly converts a half-precision floating-point value to its nearest representable value, throwing an overflow exception for any values that fall outside the representable range. @@ -986,6 +1001,7 @@ public static explicit operator Half(float value) /// Explicitly converts a half-precision floating-point value to its nearest representable value. /// The value to convert. /// converted to its nearest representable value. + [Intrinsic] public static explicit operator double(Half value) { bool sign = IsNegative(value); @@ -1181,6 +1197,7 @@ private static double CreateDoubleNaN(bool sign, ulong significand) // /// + [Intrinsic] public static Half operator +(Half left, Half right) => (Half)((float)left + (float)right); // @@ -1262,6 +1279,7 @@ public static bool IsPow2(Half value) // /// + [Intrinsic] public static Half operator --(Half value) { var tmp = (float)value; @@ -1274,6 +1292,7 @@ public static bool IsPow2(Half value) // /// + [Intrinsic] public static Half operator /(Half left, Half right) => (Half)((float)left / (float)right); // @@ -1303,6 +1322,7 @@ public static bool IsPow2(Half value) // /// + [Intrinsic] public static Half Ceiling(Half x) => (Half)MathF.Ceiling((float)x); /// @@ -1314,9 +1334,11 @@ public static TInteger ConvertToIntegerNative(Half value) where TInteger : IBinaryInteger => TInteger.CreateSaturating(value); /// + [Intrinsic] public static Half Floor(Half x) => (Half)MathF.Floor((float)x); /// + [Intrinsic] public static Half Round(Half x) => (Half)MathF.Round((float)x); /// @@ -1329,6 +1351,7 @@ public static TInteger ConvertToIntegerNative(Half value) public static Half Round(Half x, int digits, MidpointRounding mode) => (Half)MathF.Round((float)x, digits, mode); /// + [Intrinsic] public static Half Truncate(Half x) => (Half)MathF.Truncate((float)x); /// @@ -1502,6 +1525,7 @@ public static Half BitIncrement(Half x) } /// + [Intrinsic] public static Half FusedMultiplyAdd(Half left, Half right, Half addend) => (Half)MathF.FusedMultiplyAdd((float)left, (float)right, (float)addend); /// @@ -1539,9 +1563,11 @@ public static int ILogB(Half x) public static Half Lerp(Half value1, Half value2, Half amount) => (Half)float.Lerp((float)value1, (float)value2, (float)amount); /// + [Intrinsic] public static Half ReciprocalEstimate(Half x) => (Half)MathF.ReciprocalEstimate((float)x); /// + [Intrinsic] public static Half ReciprocalSqrtEstimate(Half x) => (Half)MathF.ReciprocalSqrtEstimate((float)x); /// @@ -1577,6 +1603,7 @@ public static int ILogB(Half x) // /// + [Intrinsic] public static Half operator ++(Half value) { var tmp = (float)value; @@ -1625,6 +1652,7 @@ public static int ILogB(Half x) // /// + [Intrinsic] public static Half operator *(Half left, Half right) => (Half)((float)left * (float)right); // @@ -1738,13 +1766,13 @@ public static int Sign(Half value) // /// - public static Half One => new Half(PositiveOneBits); + public static Half One { [Intrinsic] get => new Half(PositiveOneBits); } /// static int INumberBase.Radix => 2; /// - public static Half Zero => new Half(PositiveZeroBits); + public static Half Zero { [Intrinsic] get => new Half(PositiveZeroBits); } /// public static Half Abs(Half value) => new Half((ushort)(value._value & ~SignMask)); @@ -2210,6 +2238,7 @@ public static bool TryParse(ReadOnlySpan utf8Text, NumberStyles style, IFo public static Half RootN(Half x, int n) => (Half)float.RootN((float)x, n); /// + [Intrinsic] public static Half Sqrt(Half x) => (Half)MathF.Sqrt((float)x); // @@ -2234,6 +2263,7 @@ public static bool TryParse(ReadOnlySpan utf8Text, NumberStyles style, IFo // /// + [Intrinsic] public static Half operator -(Half left, Half right) => (Half)((float)left - (float)right); // diff --git a/src/native/minipal/cpufeatures.c b/src/native/minipal/cpufeatures.c index 9456932b62d2b2..b7e9dcc8125ecc 100644 --- a/src/native/minipal/cpufeatures.c +++ b/src/native/minipal/cpufeatures.c @@ -52,6 +52,14 @@ #ifndef HWCAP_ASIMDRDM #define HWCAP_ASIMDRDM (1 << 12) #endif + +#ifndef HWCAP_FPHP +#define HWCAP_FPHP (1 << 9) +#endif + +#ifndef HWCAP_ASIMDHP +#define HWCAP_ASIMDHP (1 << 10) +#endif #ifndef HWCAP_LRCPC #define HWCAP_LRCPC (1 << 15) #endif @@ -549,6 +557,10 @@ int minipal_getcpufeatures(void) if (hwCap & HWCAP_ASIMDRDM) result |= ARM64IntrinsicConstants_Rdm; + // FEAT_FP16 provides both scalar (FPHP) and Advanced SIMD (ASIMDHP) half-precision arithmetic. + if ((hwCap & HWCAP_FPHP) && (hwCap & HWCAP_ASIMDHP)) + result |= ARM64IntrinsicConstants_Fp16; + if (hwCap & HWCAP_SVE) result |= ARM64IntrinsicConstants_Sve; @@ -604,6 +616,9 @@ int minipal_getcpufeatures(void) if ((sysctlbyname("hw.optional.arm.FEAT_RDM", &valueFromSysctl, &sz, NULL, 0) == 0) && (valueFromSysctl != 0)) result |= ARM64IntrinsicConstants_Rdm; + if ((sysctlbyname("hw.optional.arm.FEAT_FP16", &valueFromSysctl, &sz, NULL, 0) == 0) && (valueFromSysctl != 0)) + result |= ARM64IntrinsicConstants_Fp16; + if ((sysctlbyname("hw.optional.arm.FEAT_SHA1", &valueFromSysctl, &sz, NULL, 0) == 0) && (valueFromSysctl != 0)) result |= ARM64IntrinsicConstants_Sha1; @@ -687,6 +702,8 @@ int minipal_getcpufeatures(void) // TODO: IsProcessorFeaturePresent doesn't support LRCPC2 yet. + // TODO: IsProcessorFeaturePresent doesn't support FEAT_FP16 (half-precision) yet. + if (IsProcessorFeaturePresent(PF_ARM_SVE_INSTRUCTIONS_AVAILABLE)) { result |= ARM64IntrinsicConstants_Sve; diff --git a/src/native/minipal/cpufeatures.h b/src/native/minipal/cpufeatures.h index 53d9308f921247..77b9aa17fed992 100644 --- a/src/native/minipal/cpufeatures.h +++ b/src/native/minipal/cpufeatures.h @@ -50,6 +50,7 @@ #define ARM64IntrinsicConstants_SveAes (1 << 13) #define ARM64IntrinsicConstants_SveSha3 (1 << 14) #define ARM64IntrinsicConstants_SveSm4 (1 << 15) +#define ARM64IntrinsicConstants_Fp16 (1 << 16) #include