diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index c9f34aba223553..ea793e4998bbd0 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -1631,6 +1631,13 @@ class CodeGen final : public CodeGenInterface void instGen_Set_Reg_To_Zero(emitAttr size, regNumber reg, insFlags flags = INS_FLAGS_DONT_CARE); + void instGen_Set_Reg_To_Base_Plus_Imm(emitAttr size, + regNumber dstReg, + regNumber baseReg, + ssize_t imm, + insFlags flags = INS_FLAGS_DONT_CARE DEBUGARG(size_t targetHandle = 0) + DEBUGARG(GenTreeFlags gtFlags = GTF_EMPTY)); + void instGen_Set_Reg_To_Imm(emitAttr size, regNumber reg, ssize_t imm, diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index da22a6d3dac14b..2d8808ffb16aad 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -2215,8 +2215,19 @@ void CodeGen::genEHCatchRet(BasicBlock* block) GetEmitter()->emitIns_R_L(INS_adr, EA_PTRSIZE, block->GetTarget(), REG_INTRET); } -// move an immediate value into an integer register +// move an immediate value + base address into an integer register +void CodeGen::instGen_Set_Reg_To_Base_Plus_Imm(emitAttr size, + regNumber dstReg, + regNumber baseReg, + ssize_t imm, + insFlags flags DEBUGARG(size_t targetHandle) + DEBUGARG(GenTreeFlags gtFlags)) +{ + instGen_Set_Reg_To_Imm(size, dstReg, imm); + GetEmitter()->emitIns_R_R_R(INS_add, size, dstReg, dstReg, baseReg); +} +// move an immediate value into an integer register void CodeGen::instGen_Set_Reg_To_Imm(emitAttr size, regNumber reg, ssize_t imm, diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 2cefdf1c92bcc0..b37b819d9203f8 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -4679,6 +4679,7 @@ class Compiler unsigned immNumber, var_types simdBaseType, CorInfoType simdBaseJitType, + CORINFO_CLASS_HANDLE op1ClsHnd, CORINFO_CLASS_HANDLE op2ClsHnd, CORINFO_CLASS_HANDLE op3ClsHnd, unsigned* immSimdSize, diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 964a386241f95c..7bc995a0890621 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -7899,10 +7899,13 @@ void emitter::emitIns_R_S(instruction ins, emitAttr attr, regNumber reg1, int va { useRegForImm = true; regNumber rsvdReg = codeGen->rsGetRsvdReg(); - codeGen->instGen_Set_Reg_To_Imm(EA_PTRSIZE, rsvdReg, imm); + // For larger imm values (> 9 bits), calculate base + imm in a reserved register first. + codeGen->instGen_Set_Reg_To_Base_Plus_Imm(EA_PTRSIZE, rsvdReg, reg2, imm); + reg2 = rsvdReg; + imm = 0; } - break; } + break; default: NYI("emitIns_R_S"); // FP locals? @@ -8150,9 +8153,11 @@ void emitter::emitIns_S_R(instruction ins, emitAttr attr, regNumber reg1, int va { useRegForImm = true; regNumber rsvdReg = codeGen->rsGetRsvdReg(); - codeGen->instGen_Set_Reg_To_Imm(EA_PTRSIZE, rsvdReg, imm); + // For larger imm values (> 9 bits), calculate base + imm in a reserved register first. + codeGen->instGen_Set_Reg_To_Base_Plus_Imm(EA_PTRSIZE, rsvdReg, reg2, imm); + reg2 = rsvdReg; + imm = 0; } - break; } break; diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index 0b963d23e39a57..6f2fe7b68adedb 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -1526,8 +1526,8 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, { unsigned immSimdSize = simdSize; var_types immSimdBaseType = simdBaseType; - getHWIntrinsicImmTypes(intrinsic, sig, 2, simdBaseType, simdBaseJitType, sigReader.op2ClsHnd, - sigReader.op3ClsHnd, &immSimdSize, &immSimdBaseType); + getHWIntrinsicImmTypes(intrinsic, sig, 2, simdBaseType, simdBaseJitType, sigReader.op1ClsHnd, + sigReader.op2ClsHnd, sigReader.op3ClsHnd, &immSimdSize, &immSimdBaseType); HWIntrinsicInfo::lookupImmBounds(intrinsic, immSimdSize, immSimdBaseType, 2, &immLowerBound, &immUpperBound); if (!CheckHWIntrinsicImmRange(intrinsic, simdBaseJitType, immOp2, mustExpand, immLowerBound, immUpperBound, @@ -1559,8 +1559,8 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, #ifdef TARGET_ARM64 unsigned immSimdSize = simdSize; var_types immSimdBaseType = simdBaseType; - getHWIntrinsicImmTypes(intrinsic, sig, 1, simdBaseType, simdBaseJitType, sigReader.op2ClsHnd, - sigReader.op3ClsHnd, &immSimdSize, &immSimdBaseType); + getHWIntrinsicImmTypes(intrinsic, sig, 1, simdBaseType, simdBaseJitType, sigReader.op1ClsHnd, + sigReader.op2ClsHnd, sigReader.op3ClsHnd, &immSimdSize, &immSimdBaseType); HWIntrinsicInfo::lookupImmBounds(intrinsic, immSimdSize, immSimdBaseType, 1, &immLowerBound, &immUpperBound); #else immUpperBound = HWIntrinsicInfo::lookupImmUpperBound(intrinsic); diff --git a/src/coreclr/jit/hwintrinsicarm64.cpp b/src/coreclr/jit/hwintrinsicarm64.cpp index 765f433e614eee..9a5c27dcc7d2d0 100644 --- a/src/coreclr/jit/hwintrinsicarm64.cpp +++ b/src/coreclr/jit/hwintrinsicarm64.cpp @@ -267,6 +267,7 @@ void Compiler::getHWIntrinsicImmOps(NamedIntrinsic intrinsic, // immNumber -- Which immediate to use (1 for most intrinsics) // simdBaseType -- base type of the intrinsic // simdType -- vector size of the intrinsic +// op1ClsHnd -- cls handler for op1 // op2ClsHnd -- cls handler for op2 // op2ClsHnd -- cls handler for op3 // immSimdSize [IN/OUT] -- Size of the immediate to override @@ -277,6 +278,7 @@ void Compiler::getHWIntrinsicImmTypes(NamedIntrinsic intrinsic, unsigned immNumber, var_types simdBaseType, CorInfoType simdBaseJitType, + CORINFO_CLASS_HANDLE op1ClsHnd, CORINFO_CLASS_HANDLE op2ClsHnd, CORINFO_CLASS_HANDLE op3ClsHnd, unsigned* immSimdSize, @@ -292,7 +294,12 @@ void Compiler::getHWIntrinsicImmTypes(NamedIntrinsic intrinsic, var_types indexedElementBaseType; *immSimdSize = 0; - if (sig->numArgs == 3) + if (sig->numArgs == 2) + { + indexedElementBaseJitType = getBaseJitTypeAndSizeOfSIMDType(op1ClsHnd, immSimdSize); + indexedElementBaseType = JitType2PreciseVarType(indexedElementBaseJitType); + } + else if (sig->numArgs == 3) { indexedElementBaseJitType = getBaseJitTypeAndSizeOfSIMDType(op2ClsHnd, immSimdSize); indexedElementBaseType = JitType2PreciseVarType(indexedElementBaseJitType); @@ -377,7 +384,15 @@ void HWIntrinsicInfo::lookupImmBounds( } else if (category == HW_Category_SIMDByIndexedElement) { - immUpperBound = Compiler::getSIMDVectorLength(simdSize, baseType) - 1; + if (intrinsic == NI_Sve_DuplicateSelectedScalarToVector) + { + // For SVE_DUP, the upper bound on index does not depend on the vector length. + immUpperBound = (512 / (BITS_PER_BYTE * genTypeSize(baseType))) - 1; + } + else + { + immUpperBound = Compiler::getSIMDVectorLength(simdSize, baseType) - 1; + } } else { diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index ed59e82e5823f2..e9684a3e2c0711 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -61,7 +61,11 @@ CodeGen::HWIntrinsicImmOpHelper::HWIntrinsicImmOpHelper(CodeGen* codeGen, GenTre const HWIntrinsic intrinInfo(intrin); var_types indexedElementOpType; - if (intrinInfo.numOperands == 3) + if (intrinInfo.numOperands == 2) + { + indexedElementOpType = intrinInfo.op1->TypeGet(); + } + else if (intrinInfo.numOperands == 3) { indexedElementOpType = intrinInfo.op2->TypeGet(); } @@ -357,13 +361,28 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) } else { - HWIntrinsicImmOpHelper helper(this, intrin.op3, node); + if (intrin.numOperands == 2) + { + HWIntrinsicImmOpHelper helper(this, intrin.op2, node); - for (helper.EmitBegin(); !helper.Done(); helper.EmitCaseEnd()) + for (helper.EmitBegin(); !helper.Done(); helper.EmitCaseEnd()) + { + const int elementIndex = helper.ImmValue(); + + GetEmitter()->emitIns_R_R_I(ins, emitSize, targetReg, op1Reg, elementIndex, opt); + } + } + else { - const int elementIndex = helper.ImmValue(); + assert(intrin.numOperands == 3); + HWIntrinsicImmOpHelper helper(this, intrin.op3, node); - GetEmitter()->emitIns_R_R_R_I(ins, emitSize, targetReg, op1Reg, op2Reg, elementIndex, opt); + for (helper.EmitBegin(); !helper.Done(); helper.EmitCaseEnd()) + { + const int elementIndex = helper.ImmValue(); + + GetEmitter()->emitIns_R_R_R_I(ins, emitSize, targetReg, op1Reg, op2Reg, elementIndex, opt); + } } } } diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index 14eaa7b82ae89f..80f4ea5866a0b4 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -68,6 +68,7 @@ HARDWARE_INTRINSIC(Sve, CreateWhileLessThanOrEqualMask8Bit, HARDWARE_INTRINSIC(Sve, Divide, -1, 2, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sdiv, INS_sve_udiv, INS_sve_sdiv, INS_sve_udiv, INS_sve_fdiv, INS_sve_fdiv}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, DotProduct, -1, 3, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sdot, INS_sve_udot, INS_sve_sdot, INS_sve_udot, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve, DotProductBySelectedScalar, -1, 4, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sdot, INS_sve_udot, INS_sve_sdot, INS_sve_udot, INS_invalid, INS_invalid}, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_LowVectorOperation) +HARDWARE_INTRINSIC(Sve, DuplicateSelectedScalarToVector, -1, 2, true, {INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup}, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand) HARDWARE_INTRINSIC(Sve, ExtractVector, -1, 3, true, {INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Sve, FusedMultiplyAdd, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmla, INS_sve_fmla}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Sve, FusedMultiplyAddBySelectedScalar, -1, 4, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmla, INS_sve_fmla}, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_FmaIntrinsic|HW_Flag_LowVectorOperation) diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index f958b392495e17..49f47338276fe0 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -3197,6 +3197,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x3: case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x4: case NI_AdvSimd_Arm64_DuplicateSelectedScalarToVector128: + case NI_Sve_DuplicateSelectedScalarToVector: assert(hasImmediateOperand); assert(varTypeIsIntegral(intrin.op2)); if (intrin.op2->IsCnsIntOrI()) diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 8547fe2e39bae0..4c4c58fd00bf97 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -1371,7 +1371,11 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou { var_types indexedElementOpType; - if (intrin.numOperands == 3) + if (intrin.numOperands == 2) + { + indexedElementOpType = intrin.op1->TypeGet(); + } + else if (intrin.numOperands == 3) { indexedElementOpType = intrin.op2->TypeGet(); } @@ -1678,7 +1682,14 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou { assert(!isRMW); - srcCount += BuildOperandUses(intrin.op2, RBM_ASIMD_INDEXED_H_ELEMENT_ALLOWED_REGS.GetFloatRegSet()); + if (intrin.id == NI_Sve_DuplicateSelectedScalarToVector) + { + srcCount += BuildOperandUses(intrin.op2); + } + else + { + srcCount += BuildOperandUses(intrin.op2, RBM_ASIMD_INDEXED_H_ELEMENT_ALLOWED_REGS.GetFloatRegSet()); + } if (intrin.op3 != nullptr) { diff --git a/src/coreclr/jit/rationalize.cpp b/src/coreclr/jit/rationalize.cpp index a0f08f6dcdef07..0d2e03d4248790 100644 --- a/src/coreclr/jit/rationalize.cpp +++ b/src/coreclr/jit/rationalize.cpp @@ -457,8 +457,8 @@ void Rationalizer::RewriteHWIntrinsicAsUserCall(GenTree** use, ArrayStackgetHWIntrinsicImmTypes(intrinsicId, &sigInfo, 2, simdBaseType, simdBaseJitType, op2ClsHnd, - op3ClsHnd, &immSimdSize, &immSimdBaseType); + comp->getHWIntrinsicImmTypes(intrinsicId, &sigInfo, 2, simdBaseType, simdBaseJitType, op1ClsHnd, + op2ClsHnd, op3ClsHnd, &immSimdSize, &immSimdBaseType); HWIntrinsicInfo::lookupImmBounds(intrinsicId, immSimdSize, immSimdBaseType, 2, &immLowerBound, &immUpperBound); @@ -473,8 +473,8 @@ void Rationalizer::RewriteHWIntrinsicAsUserCall(GenTree** use, ArrayStackgetHWIntrinsicImmTypes(intrinsicId, &sigInfo, 1, simdBaseType, simdBaseJitType, op2ClsHnd, op3ClsHnd, - &immSimdSize, &immSimdBaseType); + comp->getHWIntrinsicImmTypes(intrinsicId, &sigInfo, 1, simdBaseType, simdBaseJitType, op1ClsHnd, op2ClsHnd, + op3ClsHnd, &immSimdSize, &immSimdBaseType); HWIntrinsicInfo::lookupImmBounds(intrinsicId, immSimdSize, immSimdBaseType, 1, &immLowerBound, &immUpperBound); #endif diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs index ee7848a4a5eb1a..42a1d2466910a3 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs @@ -1403,6 +1403,69 @@ internal Arm64() { } public static unsafe Vector DotProductBySelectedScalar(Vector addend, Vector left, Vector right, [ConstantExpected] byte rightIndex) { throw new PlatformNotSupportedException(); } + /// Broadcast a scalar value + + /// + /// svuint8_t svdup_lane[_u8](svuint8_t data, uint8_t index) + /// DUP Zresult.B, Zdata.B[index] + /// + public static unsafe Vector DuplicateSelectedScalarToVector(Vector data, [ConstantExpected(Min = 0, Max = (byte)(63))] byte index) { throw new PlatformNotSupportedException(); } + + /// + /// svfloat64_t svdup_lane[_f64](svfloat64_t data, uint64_t index) + /// DUP Zresult.D, Zdata.D[index] + /// + public static unsafe Vector DuplicateSelectedScalarToVector(Vector data, [ConstantExpected(Min = 0, Max = (byte)(7))] byte index) { throw new PlatformNotSupportedException(); } + + /// + /// svint16_t svdup_lane[_s16](svint16_t data, uint16_t index) + /// DUP Zresult.H, Zdata.H[index] + /// + public static unsafe Vector DuplicateSelectedScalarToVector(Vector data, [ConstantExpected(Min = 0, Max = (byte)(31))] byte index) { throw new PlatformNotSupportedException(); } + + /// + /// svint32_t svdup_lane[_s32](svint32_t data, uint32_t index) + /// DUP Zresult.S, Zdata.S[index] + /// + public static unsafe Vector DuplicateSelectedScalarToVector(Vector data, [ConstantExpected(Min = 0, Max = (byte)(15))] byte index) { throw new PlatformNotSupportedException(); } + + /// + /// svint64_t svdup_lane[_s64](svint64_t data, uint64_t index) + /// DUP Zresult.D, Zdata.D[index] + /// + public static unsafe Vector DuplicateSelectedScalarToVector(Vector data, [ConstantExpected(Min = 0, Max = (byte)(7))] byte index) { throw new PlatformNotSupportedException(); } + + /// + /// svint8_t svdup_lane[_s8](svint8_t data, uint8_t index) + /// DUP Zresult.B, Zdata.B[index] + /// + public static unsafe Vector DuplicateSelectedScalarToVector(Vector data, [ConstantExpected(Min = 0, Max = (byte)(63))] byte index) { throw new PlatformNotSupportedException(); } + + /// + /// svfloat32_t svdup_lane[_f32](svfloat32_t data, uint32_t index) + /// DUP Zresult.S, Zdata.S[index] + /// + public static unsafe Vector DuplicateSelectedScalarToVector(Vector data, [ConstantExpected(Min = 0, Max = (byte)(15))] byte index) { throw new PlatformNotSupportedException(); } + + /// + /// svuint16_t svdup_lane[_u16](svuint16_t data, uint16_t index) + /// DUP Zresult.H, Zdata.H[index] + /// + public static unsafe Vector DuplicateSelectedScalarToVector(Vector data, [ConstantExpected(Min = 0, Max = (byte)(31))] byte index) { throw new PlatformNotSupportedException(); } + + /// + /// svuint32_t svdup_lane[_u32](svuint32_t data, uint32_t index) + /// DUP Zresult.S, Zdata.S[index] + /// + public static unsafe Vector DuplicateSelectedScalarToVector(Vector data, [ConstantExpected(Min = 0, Max = (byte)(15))] byte index) { throw new PlatformNotSupportedException(); } + + /// + /// svuint64_t svdup_lane[_u64](svuint64_t data, uint64_t index) + /// DUP Zresult.D, Zdata.D[index] + /// + public static unsafe Vector DuplicateSelectedScalarToVector(Vector data, [ConstantExpected(Min = 0, Max = (byte)(7))] byte index) { throw new PlatformNotSupportedException(); } + + /// /// svuint8_t svext[_u8](svuint8_t op1, svuint8_t op2, uint64_t imm3) /// EXT Ztied1.B, Ztied1.B, Zop2.B, #imm3 diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs index 7bf2ff6b84e9b4..3014753fc17792 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs @@ -1459,6 +1459,69 @@ internal Arm64() { } public static unsafe Vector DotProductBySelectedScalar(Vector addend, Vector left, Vector right, [ConstantExpected] byte rightIndex) => DotProductBySelectedScalar(addend, left, right, rightIndex); + /// Broadcast a scalar value + + /// + /// svuint8_t svdup_lane[_u8](svuint8_t data, uint8_t index) + /// DUP Zresult.B, Zdata.B[index] + /// + public static unsafe Vector DuplicateSelectedScalarToVector(Vector data, [ConstantExpected(Min = 0, Max = (byte)(63))] byte index) => DuplicateSelectedScalarToVector(data, index); + + /// + /// svfloat64_t svdup_lane[_f64](svfloat64_t data, uint64_t index) + /// DUP Zresult.D, Zdata.D[index] + /// + public static unsafe Vector DuplicateSelectedScalarToVector(Vector data, [ConstantExpected(Min = 0, Max = (byte)(7))] byte index) => DuplicateSelectedScalarToVector(data, index); + + /// + /// svint16_t svdup_lane[_s16](svint16_t data, uint16_t index) + /// DUP Zresult.H, Zdata.H[index] + /// + public static unsafe Vector DuplicateSelectedScalarToVector(Vector data, [ConstantExpected(Min = 0, Max = (byte)(31))] byte index) => DuplicateSelectedScalarToVector(data, index); + + /// + /// svint32_t svdup_lane[_s32](svint32_t data, uint32_t index) + /// DUP Zresult.S, Zdata.S[index] + /// + public static unsafe Vector DuplicateSelectedScalarToVector(Vector data, [ConstantExpected(Min = 0, Max = (byte)(15))] byte index) => DuplicateSelectedScalarToVector(data, index); + + /// + /// svint64_t svdup_lane[_s64](svint64_t data, uint64_t index) + /// DUP Zresult.D, Zdata.D[index] + /// + public static unsafe Vector DuplicateSelectedScalarToVector(Vector data, [ConstantExpected(Min = 0, Max = (byte)(7))] byte index) => DuplicateSelectedScalarToVector(data, index); + + /// + /// svint8_t svdup_lane[_s8](svint8_t data, uint8_t index) + /// DUP Zresult.B, Zdata.B[index] + /// + public static unsafe Vector DuplicateSelectedScalarToVector(Vector data, [ConstantExpected(Min = 0, Max = (byte)(63))] byte index) => DuplicateSelectedScalarToVector(data, index); + + /// + /// svfloat32_t svdup_lane[_f32](svfloat32_t data, uint32_t index) + /// DUP Zresult.S, Zdata.S[index] + /// + public static unsafe Vector DuplicateSelectedScalarToVector(Vector data, [ConstantExpected(Min = 0, Max = (byte)(15))] byte index) => DuplicateSelectedScalarToVector(data, index); + + /// + /// svuint16_t svdup_lane[_u16](svuint16_t data, uint16_t index) + /// DUP Zresult.H, Zdata.H[index] + /// + public static unsafe Vector DuplicateSelectedScalarToVector(Vector data, [ConstantExpected(Min = 0, Max = (byte)(31))] byte index) => DuplicateSelectedScalarToVector(data, index); + + /// + /// svuint32_t svdup_lane[_u32](svuint32_t data, uint32_t index) + /// DUP Zresult.S, Zdata.S[index] + /// + public static unsafe Vector DuplicateSelectedScalarToVector(Vector data, [ConstantExpected(Min = 0, Max = (byte)(15))] byte index) => DuplicateSelectedScalarToVector(data, index); + + /// + /// svuint64_t svdup_lane[_u64](svuint64_t data, uint64_t index) + /// DUP Zresult.D, Zdata.D[index] + /// + public static unsafe Vector DuplicateSelectedScalarToVector(Vector data, [ConstantExpected(Min = 0, Max = (byte)(7))] byte index) => DuplicateSelectedScalarToVector(data, index); + + /// /// svuint8_t svext[_u8](svuint8_t op1, svuint8_t op2, uint64_t imm3) /// EXT Ztied1.B, Ztied1.B, Zop2.B, #imm3 diff --git a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs index acc21ef80b49a1..166b6cc7ca8bb7 100644 --- a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs +++ b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs @@ -4391,6 +4391,17 @@ internal Arm64() { } public static System.Numerics.Vector DotProductBySelectedScalar(System.Numerics.Vector addend, System.Numerics.Vector left, System.Numerics.Vector right, [ConstantExpected] byte rightIndex) { throw null; } public static System.Numerics.Vector DotProductBySelectedScalar(System.Numerics.Vector addend, System.Numerics.Vector left, System.Numerics.Vector right, [ConstantExpected] byte rightIndex) { throw null; } + public static System.Numerics.Vector DuplicateSelectedScalarToVector(System.Numerics.Vector data, [ConstantExpected(Min = 0, Max = (byte)(63))] byte index) { throw null; } + public static System.Numerics.Vector DuplicateSelectedScalarToVector(System.Numerics.Vector data, [ConstantExpected(Min = 0, Max = (byte)(7))] byte index) { throw null; } + public static System.Numerics.Vector DuplicateSelectedScalarToVector(System.Numerics.Vector data, [ConstantExpected(Min = 0, Max = (byte)(31))] byte index) { throw null; } + public static System.Numerics.Vector DuplicateSelectedScalarToVector(System.Numerics.Vector data, [ConstantExpected(Min = 0, Max = (byte)(15))] byte index) { throw null; } + public static System.Numerics.Vector DuplicateSelectedScalarToVector(System.Numerics.Vector data, [ConstantExpected(Min = 0, Max = (byte)(7))] byte index) { throw null; } + public static System.Numerics.Vector DuplicateSelectedScalarToVector(System.Numerics.Vector data, [ConstantExpected(Min = 0, Max = (byte)(63))] byte index) { throw null; } + public static System.Numerics.Vector DuplicateSelectedScalarToVector(System.Numerics.Vector data, [ConstantExpected(Min = 0, Max = (byte)(15))] byte index) { throw null; } + public static System.Numerics.Vector DuplicateSelectedScalarToVector(System.Numerics.Vector data, [ConstantExpected(Min = 0, Max = (byte)(31))] byte index) { throw null; } + public static System.Numerics.Vector DuplicateSelectedScalarToVector(System.Numerics.Vector data, [ConstantExpected(Min = 0, Max = (byte)(15))] byte index) { throw null; } + public static System.Numerics.Vector DuplicateSelectedScalarToVector(System.Numerics.Vector data, [ConstantExpected(Min = 0, Max = (byte)(7))] byte index) { throw null; } + public static unsafe System.Numerics.Vector ExtractVector(System.Numerics.Vector upper, System.Numerics.Vector lower, [ConstantExpected] byte index) { throw null; } public static unsafe System.Numerics.Vector ExtractVector(System.Numerics.Vector upper, System.Numerics.Vector lower, [ConstantExpected] byte index) { throw null; } public static unsafe System.Numerics.Vector ExtractVector(System.Numerics.Vector upper, System.Numerics.Vector lower, [ConstantExpected] byte index) { throw null; } diff --git a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs index da93f2b6448b16..8b2801d39b7919 100644 --- a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs +++ b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs @@ -183,6 +183,7 @@ ("_SveBinaryOpTestTemplate.template", "SveVecBinOpConvertTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleTernVecOpTest_ValidationLogicForCndSel }), ("_SveBinaryMaskOpTestTemplate.template", "SveMaskVecBinOpConvertTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleVecOpTest_ValidationLogicForCndSel }), ("_SveImmBinaryOpTestTemplate.template", "SveVecImmBinOpTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleVecOpTest_ValidationLogicForCndSel }), + ("_SveImmUnaryOpTestTemplate.template", "SveVecImmUnOpTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleVecOpTest_ValidationLogicForCndSel }), ("_SveTernOpTestTemplate.template", "SveVecTernOpTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleTernVecOpTest_ValidationLogicForCndSel }), ("_SveTernOpFirstArgTestTemplate.template", "SveVecTernOpFirstArgTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleTernVecOpTest_ValidationLogicForCndSel }), ("_SveImmTernOpTestTemplate.template", "SveVecImmTernOpTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleTernVecOpTest_ValidationLogicForCndSel }), @@ -3179,6 +3180,17 @@ ("SveCreateTrueMaskTest.template", new Dictionary { ["TestName"] = "Sve_CreateTrueMaskUInt32", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateTrueMaskUInt32", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1Type"] = "SveMaskPattern"}), ("SveCreateTrueMaskTest.template", new Dictionary { ["TestName"] = "Sve_CreateTrueMaskUInt64", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateTrueMaskUInt64", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1Type"] = "SveMaskPattern"}), + ("SveVecImmUnOpTest.template", new Dictionary { ["TestName"] = "Sve_DuplicateSelectedScalarToVector_float", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "DuplicateSelectedScalarToVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSingle()", ["InvalidImm"] = "16", ["Imm"] = "TestLibrary.Generator.GetByte() % 16", ["ValidateIterResult"] = "result[i] != (imm < Op1ElementCount ? firstOp[imm] : 0)", ["GetIterResult"] = "(Single)(imm < Op1ElementCount ? firstOp[imm] : 0)"}), + ("SveVecImmUnOpTest.template", new Dictionary { ["TestName"] = "Sve_DuplicateSelectedScalarToVector_double", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "DuplicateSelectedScalarToVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetDouble()", ["InvalidImm"] = "8", ["Imm"] = "TestLibrary.Generator.GetByte() % 8", ["ValidateIterResult"] = "result[i] != (imm < Op1ElementCount ? firstOp[imm] : 0)", ["GetIterResult"] = "(Double)(imm < Op1ElementCount ? firstOp[imm] : 0)"}), + ("SveVecImmUnOpTest.template", new Dictionary { ["TestName"] = "Sve_DuplicateSelectedScalarToVector_sbyte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "DuplicateSelectedScalarToVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["InvalidImm"] = "64", ["Imm"] = "TestLibrary.Generator.GetByte() % 64", ["ValidateIterResult"] = "result[i] != (imm < Op1ElementCount ? firstOp[imm] : 0)", ["GetIterResult"] = "(SByte)(imm < Op1ElementCount ? firstOp[imm] : 0)"}), + ("SveVecImmUnOpTest.template", new Dictionary { ["TestName"] = "Sve_DuplicateSelectedScalarToVector_short", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "DuplicateSelectedScalarToVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["InvalidImm"] = "32", ["Imm"] = "TestLibrary.Generator.GetByte() % 32", ["ValidateIterResult"] = "result[i] != (imm < Op1ElementCount ? firstOp[imm] : 0)", ["GetIterResult"] = "(Int16)(imm < Op1ElementCount ? firstOp[imm] : 0)"}), + ("SveVecImmUnOpTest.template", new Dictionary { ["TestName"] = "Sve_DuplicateSelectedScalarToVector_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "DuplicateSelectedScalarToVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["InvalidImm"] = "16", ["Imm"] = "TestLibrary.Generator.GetByte() % 16", ["ValidateIterResult"] = "result[i] != (imm < Op1ElementCount ? firstOp[imm] : 0)", ["GetIterResult"] = "(Int32)(imm < Op1ElementCount ? firstOp[imm] : 0)"}), + ("SveVecImmUnOpTest.template", new Dictionary { ["TestName"] = "Sve_DuplicateSelectedScalarToVector_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "DuplicateSelectedScalarToVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt64()", ["InvalidImm"] = "8", ["Imm"] = "TestLibrary.Generator.GetByte() % 8", ["ValidateIterResult"] = "result[i] != (imm < Op1ElementCount ? firstOp[imm] : 0)", ["GetIterResult"] = "(Int64)(imm < Op1ElementCount ? firstOp[imm] : 0)"}), + ("SveVecImmUnOpTest.template", new Dictionary { ["TestName"] = "Sve_DuplicateSelectedScalarToVector_byte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "DuplicateSelectedScalarToVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Byte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetByte()", ["InvalidImm"] = "64", ["Imm"] = "TestLibrary.Generator.GetByte() % 64", ["ValidateIterResult"] = "result[i] != (imm < Op1ElementCount ? firstOp[imm] : 0)", ["GetIterResult"] = "(Byte)(imm < Op1ElementCount ? firstOp[imm] : 0)"}), + ("SveVecImmUnOpTest.template", new Dictionary { ["TestName"] = "Sve_DuplicateSelectedScalarToVector_ushort", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "DuplicateSelectedScalarToVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["InvalidImm"] = "32", ["Imm"] = "TestLibrary.Generator.GetByte() % 32", ["ValidateIterResult"] = "result[i] != (imm < Op1ElementCount ? firstOp[imm] : 0)", ["GetIterResult"] = "(UInt16)(imm < Op1ElementCount ? firstOp[imm] : 0)"}), + ("SveVecImmUnOpTest.template", new Dictionary { ["TestName"] = "Sve_DuplicateSelectedScalarToVector_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "DuplicateSelectedScalarToVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["InvalidImm"] = "16", ["Imm"] = "TestLibrary.Generator.GetByte() % 16", ["ValidateIterResult"] = "result[i] != (imm < Op1ElementCount ? firstOp[imm] : 0)", ["GetIterResult"] = "(UInt32)(imm < Op1ElementCount ? firstOp[imm] : 0)"}), + ("SveVecImmUnOpTest.template", new Dictionary { ["TestName"] = "Sve_DuplicateSelectedScalarToVector_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "DuplicateSelectedScalarToVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["InvalidImm"] = "8", ["Imm"] = "TestLibrary.Generator.GetByte() % 8", ["ValidateIterResult"] = "result[i] != (imm < Op1ElementCount ? firstOp[imm] : 0)", ["GetIterResult"] = "(UInt64)(imm < Op1ElementCount ? firstOp[imm] : 0)"}), + ("SveExtractVectorTest.template", new Dictionary { ["TestName"] = "SveExtractVector_Byte_1", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ExtractVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Byte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()", ["ElementIndex"] = "1", ["ValidateIterResult"] = "Helpers.ExtractVector(firstOp, secondOp, ElementIndex, i) != result[i]"}), ("SveExtractVectorTest.template", new Dictionary { ["TestName"] = "SveExtractVector_Double_1", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ExtractVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ElementIndex"] = "1", ["ValidateIterResult"] = "BitConverter.DoubleToInt64Bits(Helpers.ExtractVector(firstOp, secondOp, ElementIndex, i)) != BitConverter.DoubleToInt64Bits(result[i])"}), ("SveExtractVectorTest.template", new Dictionary { ["TestName"] = "SveExtractVector_Int16_1", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ExtractVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()", ["ElementIndex"] = "1", ["ValidateIterResult"] = "Helpers.ExtractVector(firstOp, secondOp, ElementIndex, i) != result[i]"}), diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveImmUnaryOpTestTemplate.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveImmUnaryOpTestTemplate.template new file mode 100644 index 00000000000000..dbc66907536445 --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveImmUnaryOpTestTemplate.template @@ -0,0 +1,402 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +/****************************************************************************** + * This file is auto-generated from a template file by the GenerateTests.csx * + * script in tests\src\JIT\HardwareIntrinsics.Arm\Shared. In order to make * + * changes, please update the corresponding template and run according to the * + * directions listed in the file. * + ******************************************************************************/ + +using System; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using Xunit; + +namespace JIT.HardwareIntrinsics.Arm +{ + public static partial class Program + { + [Fact] + public static void {TestName}() + { + var test = new {TemplateName}UnaryOpTest__{TestName}(); + + if (test.IsSupported) + { + // Validates basic functionality works, using Unsafe.Read + test.RunBasicScenario_UnsafeRead(); + + if ({LoadIsa}.IsSupported) + { + // Validates basic functionality works, using Load + test.RunBasicScenario_Load(); + } + + // Validates calling via reflection works, using Unsafe.Read + test.RunReflectionScenario_UnsafeRead(); + + // Validates passing a local works, using Unsafe.Read + test.RunLclVarScenario_UnsafeRead(); + + // Validates passing an instance member of a class works + test.RunClassFldScenario(); + + // Validates passing the field of a local struct works + test.RunStructLclFldScenario(); + + // Validates passing an instance member of a struct works + test.RunStructFldScenario(); + // Validates executing the test inside conditional, with op1 as falseValue + test.ConditionalSelect_Op1(); + // Validates executing the test inside conditional, with op3 as falseValue + test.ConditionalSelect_FalseOp(); + // Validates executing the test inside conditional, with op3 as zero + test.ConditionalSelect_ZeroOp(); + + // Validates basic functionality fails with an invalid imm, using Unsafe.ReadUnaligned + test.RunBasicScenario_UnsafeRead_InvalidImm(); + } + else + { + // Validates we throw on unsupported hardware + test.RunUnsupportedScenario(); + } + + if (!test.Succeeded) + { + throw new Exception("One or more scenarios did not complete as expected."); + } + } + } + + public sealed unsafe class {TemplateName}UnaryOpTest__{TestName} + { + private struct DataTable + { + private byte[] inArray; + private byte[] outArray; + + private GCHandle inHandle; + private GCHandle outHandle; + + private ulong alignment; + + public DataTable({Op1BaseType}[] inArray, {RetBaseType}[] outArray, int alignment) + { + int sizeOfinArray = inArray.Length * Unsafe.SizeOf<{Op1BaseType}>(); + int sizeOfoutArray = outArray.Length * Unsafe.SizeOf<{RetBaseType}>(); + if ((alignment != 64 && alignment != 16 && alignment != 8) || (alignment * 2) < sizeOfinArray || (alignment * 2) < sizeOfoutArray) + { + throw new ArgumentException("Invalid value of alignment"); + } + + this.inArray = new byte[alignment * 2]; + this.outArray = new byte[alignment * 2]; + + this.inHandle = GCHandle.Alloc(this.inArray, GCHandleType.Pinned); + this.outHandle = GCHandle.Alloc(this.outArray, GCHandleType.Pinned); + + this.alignment = (ulong)alignment; + + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArrayPtr), ref Unsafe.As<{Op1BaseType}, byte>(ref inArray[0]), (uint)sizeOfinArray); + } + + public void* inArrayPtr => Align((byte*)(inHandle.AddrOfPinnedObject().ToPointer()), alignment); + public void* outArrayPtr => Align((byte*)(outHandle.AddrOfPinnedObject().ToPointer()), alignment); + + public void Dispose() + { + inHandle.Free(); + outHandle.Free(); + } + + private static unsafe void* Align(byte* buffer, ulong expectedAlignment) + { + return (void*)(((ulong)buffer + expectedAlignment - 1) & ~(expectedAlignment - 1)); + } + } + + private struct TestStruct + { + public {Op1VectorType}<{Op1BaseType}> _fld; + public byte _imm; + + public static TestStruct Create() + { + var testStruct = new TestStruct(); + + for (var i = 0; i < Op1ElementCount; i++) { _data[i] = {NextValueOp1}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref testStruct._fld), ref Unsafe.As<{Op1BaseType}, byte>(ref _data[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + testStruct._imm = (byte)({Imm}); + + return testStruct; + } + + public void RunStructFldScenario({TemplateName}UnaryOpTest__{TestName} testClass) + { + var result = {Isa}.{Method}(_fld, _imm); + + Unsafe.Write(testClass._dataTable.outArrayPtr, result); + testClass.ValidateResult(_fld, _imm, testClass._dataTable.outArrayPtr); + } + } + + private static readonly int LargestVectorSize = {LargestVectorSize}; + + private static readonly int Op1ElementCount = Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>() / sizeof({Op1BaseType}); + private static readonly int RetElementCount = Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>() / sizeof({RetBaseType}); + private static readonly byte Imm = (byte)({Imm}); + + private static {Op1BaseType}[] _maskData = new {Op1BaseType}[Op1ElementCount]; + private static {Op1BaseType}[] _data = new {Op1BaseType}[Op1ElementCount]; + + private {Op1VectorType}<{Op1BaseType}> _mask; + private {Op1VectorType}<{Op1BaseType}> _fld; + private {Op1VectorType}<{Op1BaseType}> _falseFld; + + private DataTable _dataTable; + + public {TemplateName}UnaryOpTest__{TestName}() + { + Succeeded = true; + + for (var i = 0; i < Op1ElementCount; i++) { _maskData[i] = ({Op1BaseType})({NextValueOp1} % 2); } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref _mask), ref Unsafe.As<{Op1BaseType}, byte>(ref _maskData[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + for (var i = 0; i < Op1ElementCount; i++) { _data[i] = {NextValueOp1}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref _fld), ref Unsafe.As<{Op1BaseType}, byte>(ref _data[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + + for (var i = 0; i < Op1ElementCount; i++) { _data[i] = {NextValueOp1}; } + _dataTable = new DataTable(_data, new {RetBaseType}[RetElementCount], LargestVectorSize); + } + + public bool IsSupported => {Isa}.IsSupported; + + public bool Succeeded { get; set; } + + public void RunBasicScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_UnsafeRead)); + + var result = {Isa}.{Method}( + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArrayPtr), + Imm + ); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_dataTable.inArrayPtr, Imm, _dataTable.outArrayPtr); + } + + public void RunBasicScenario_UnsafeRead_InvalidImm() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_UnsafeRead_InvalidImm)); + + bool succeeded = false; + try + { + var result = {Isa}.{Method}( + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArrayPtr), + {InvalidImm} + ); + } + catch (ArgumentOutOfRangeException) + { + succeeded = true; + } + } + + public void RunBasicScenario_Load() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_Load)); + + {Op1VectorType}<{Op1BaseType}> loadMask = Sve.CreateTrueMask{RetBaseType}(SveMaskPattern.All); + + var result = {Isa}.{Method}( + {LoadIsa}.Load{Op1VectorType}(loadMask, ({Op1BaseType}*)(_dataTable.inArrayPtr)), + Imm + ); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_dataTable.inArrayPtr, Imm, _dataTable.outArrayPtr); + } + + public void RunReflectionScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunReflectionScenario_UnsafeRead)); + + var result = typeof({Isa}).GetMethod(nameof({Isa}.{Method}), new Type[] { typeof({Op1VectorType}<{Op1BaseType}>), typeof(byte) }) + .Invoke(null, new object[] { + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArrayPtr), + Imm + }); + + Unsafe.Write(_dataTable.outArrayPtr, ({RetVectorType}<{RetBaseType}>)(result)); + ValidateResult(_dataTable.inArrayPtr, Imm, _dataTable.outArrayPtr); + } + + public void RunLclVarScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunLclVarScenario_UnsafeRead)); + + var op = Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArrayPtr); + var result = {Isa}.{Method}(op, Imm); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(op, Imm, _dataTable.outArrayPtr); + } + + public void RunClassFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunClassFldScenario)); + + var result = {Isa}.{Method}(_fld, Imm); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_fld, Imm, _dataTable.outArrayPtr); + } + + public void RunStructLclFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunStructLclFldScenario)); + + var test = TestStruct.Create(); + var result = {Isa}.{Method}(test._fld, test._imm); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(test._fld, test._imm, _dataTable.outArrayPtr); + } + + public void RunStructFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunStructFldScenario)); + + var test = TestStruct.Create(); + test.RunStructFldScenario(this); + } + + public void ConditionalSelect_Op1() + { + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_mask"); + ConditionalSelectScenario(_mask, _fld, _fld); + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_zero"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.Zero, _fld, _fld); + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_all"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld, _fld); + } + + public void ConditionalSelect_FalseOp() + { + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_mask"); + ConditionalSelectScenario(_mask, _fld, _falseFld); + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_zero"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.Zero, _fld, _falseFld); + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_all"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld, _falseFld); + } + + public void ConditionalSelect_ZeroOp() + { + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_mask"); + ConditionalSelectScenario(_mask, _fld, {Op1VectorType}<{Op1BaseType}>.Zero); + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_zero"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.Zero, _fld, {Op1VectorType}<{Op1BaseType}>.Zero); + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_all"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld, {Op1VectorType}<{Op1BaseType}>.Zero); + } + + [method: MethodImpl(MethodImplOptions.AggressiveInlining)] + private void ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}> mask, {Op1VectorType}<{Op1BaseType}> op, {Op1VectorType}<{Op1BaseType}> falseOp) + { + var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op, Imm), falseOp); + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateConditionalSelectResult(mask, op, falseOp, Imm, _dataTable.outArrayPtr); + } + + public void RunUnsupportedScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunUnsupportedScenario)); + + bool succeeded = false; + + try + { + RunBasicScenario_UnsafeRead(); + } + catch (PlatformNotSupportedException) + { + succeeded = true; + } + + if (!succeeded) + { + Succeeded = false; + } + } + private void ValidateConditionalSelectResult({Op1VectorType}<{Op1BaseType}> maskOp, {Op1VectorType}<{Op1BaseType}> firstOp, {Op1VectorType}<{Op1BaseType}> falseOp, byte imm, void* output, [CallerMemberName] string method = "") + { + {Op1BaseType}[] mask = new {Op1BaseType}[Op1ElementCount]; + {Op1BaseType}[] first = new {Op1BaseType}[Op1ElementCount]; + {Op1BaseType}[] falseVal = new {Op1BaseType}[Op1ElementCount]; + {RetBaseType}[] result = new {RetBaseType}[RetElementCount]; + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref mask[0]), maskOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref first[0]), firstOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref falseVal[0]), falseOp); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref result[0]), ref Unsafe.AsRef(output), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + bool succeeded = true; + {TemplateValidationLogicForCndSel} + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1VectorType}<{Op1BaseType}>): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" mask: ({string.Join(", ", mask)})"); + TestLibrary.TestFramework.LogInformation($" first: ({string.Join(", ", first)})"); + TestLibrary.TestFramework.LogInformation($" falseOp: ({string.Join(", ", falseVal)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + Succeeded = false; + } + } + + private void ValidateResult({Op1VectorType}<{Op1BaseType}> op, byte imm, void* result, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray = new {Op1BaseType}[Op1ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray[0]), op); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateResult(inArray, imm, outArray, method); + } + + private void ValidateResult(void* op, byte imm, void* result, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray = new {Op1BaseType}[Op1ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray[0]), ref Unsafe.AsRef(op), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateResult(inArray, imm, outArray, method); + } + + private void ValidateResult({Op1BaseType}[] firstOp, byte imm, {RetBaseType}[] result, [CallerMemberName] string method = "") + { + bool succeeded = true; + + {TemplateValidationLogic} + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1VectorType}<{Op1BaseType}>, byte): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" firstOp: ({string.Join(", ", firstOp)})"); + TestLibrary.TestFramework.LogInformation($" imm: ({string.Join(", ", imm)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + } +}