diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 9a899443294b5d..9c87abe0c1a0f2 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -18290,7 +18290,8 @@ bool GenTree::canBeContained() const } else if (OperIsHWIntrinsic() && !isContainableHWIntrinsic()) { - return isEmbeddedMaskingCompatibleHWIntrinsic(); + return isEmbeddedMaskingCompatibleHWIntrinsic() || + HWIntrinsicInfo::SupportsContainment(AsHWIntrinsic()->GetHWIntrinsicId()); } return true; diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index 38a480ae77ef5a..ac78ae6d9cfd1d 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -542,9 +542,20 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) case 2: { - assert(instrIsRMW); + if (!instrIsRMW) + { + // Either this is VectorZero or ConvertVectorToMask(TrueMask, VectorZero) + assert(intrin.op3->IsVectorZero() || + (intrin.op3->OperIs(GT_HWINTRINSIC) && + intrin.op3->AsHWIntrinsic()->OperIsConvertVectorToMask() && + intrin.op3->AsHWIntrinsic()->Op(2)->IsVectorZero())); - if (intrin.op3->IsVectorZero()) + // Perform the actual "predicated" operation so that `embMaskOp1Reg` is the first operand + // and `embMaskOp2Reg` is the second operand. + GetEmitter()->emitIns_R_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, + embMaskOp2Reg, opt); + } + else if (intrin.op3->IsVectorZero()) { // If `falseReg` is zero, then move the first operand of `intrinEmbMask` in the // destination using /Z. diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index 600c1c7d9eaaaa..e4cf79f0ed06bd 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -18,6 +18,10 @@ // Sve HARDWARE_INTRINSIC(Sve, Abs, -1, -1, false, {INS_sve_abs, INS_invalid, INS_sve_abs, INS_invalid, INS_sve_abs, INS_invalid, INS_sve_abs, INS_invalid, INS_sve_fabs, INS_sve_fabs}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, AbsoluteCompareGreaterThan, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_facgt, INS_sve_facgt}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, AbsoluteCompareGreaterThanOrEqual, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_facge, INS_sve_facge}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, AbsoluteCompareLessThan, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_faclt, INS_sve_faclt}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, AbsoluteCompareLessThanOrEqual, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_facle, INS_sve_facle}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Sve, AbsoluteDifference, -1, -1, false, {INS_sve_sabd, INS_sve_uabd, INS_sve_sabd, INS_sve_uabd, INS_sve_sabd, INS_sve_uabd, INS_sve_sabd, INS_sve_uabd, INS_sve_fabd, INS_sve_fabd}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, Add, -1, -1, false, {INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_fadd, INS_sve_fadd}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, AddAcross, -1, 1, true, {INS_sve_saddv, INS_sve_uaddv, INS_sve_saddv, INS_sve_uaddv, INS_sve_saddv, INS_sve_uaddv, INS_sve_uaddv, INS_sve_uaddv, INS_sve_faddv, INS_sve_faddv}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) @@ -210,8 +214,8 @@ HARDWARE_INTRINSIC(Sve, ZipLow, // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Special intrinsics that are generated during importing or lowering -HARDWARE_INTRINSIC(Sve, ConvertMaskToVector, -1, 1, true, {INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov}, HW_Category_Helper, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation) -HARDWARE_INTRINSIC(Sve, ConvertVectorToMask, -1, 2, true, {INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne}, HW_Category_Helper, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, ConvertMaskToVector, -1, 1, true, {INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov}, HW_Category_Helper, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation| HW_Flag_SupportsContainment) +HARDWARE_INTRINSIC(Sve, ConvertVectorToMask, -1, 2, true, {INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne}, HW_Category_Helper, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_LowMaskedOperation| HW_Flag_SupportsContainment) HARDWARE_INTRINSIC(Sve, CreateTrueMaskAll, -1, -1, false, {INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue}, HW_Category_Helper, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Sve, StoreAndZipx2, -1, 3, true, {INS_sve_st2b, INS_sve_st2b, INS_sve_st2h, INS_sve_st2h, INS_sve_st2w, INS_sve_st2w, INS_sve_st2d, INS_sve_st2d, INS_sve_st2w, INS_sve_st2d}, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters) HARDWARE_INTRINSIC(Sve, StoreAndZipx3, -1, 3, true, {INS_sve_st3b, INS_sve_st3b, INS_sve_st3h, INS_sve_st3h, INS_sve_st3w, INS_sve_st3w, INS_sve_st3d, INS_sve_st3d, INS_sve_st3w, INS_sve_st3d}, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters) diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index 436c770065d3bf..8af185d12b3ebf 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -1303,17 +1303,32 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) { GenTree* user = use.User(); // Wrap the intrinsic in ConditionalSelect only if it is not already inside another ConditionalSelect - if (!user->OperIsHWIntrinsic() || (user->AsHWIntrinsic()->GetHWIntrinsicId() != NI_Sve_ConditionalSelect)) + // If it is inside ConditionalSelect, then make sure that it is the `mask` operation of it. + if (!user->OperIsHWIntrinsic(NI_Sve_ConditionalSelect) || + (HWIntrinsicInfo::ReturnsPerElementMask(node->GetHWIntrinsicId()))) { CorInfoType simdBaseJitType = node->GetSimdBaseJitType(); unsigned simdSize = node->GetSimdSize(); var_types simdType = Compiler::getSIMDTypeForSize(simdSize); GenTree* trueMask = comp->gtNewSimdAllTrueMaskNode(simdBaseJitType, simdSize); - GenTree* trueVal = node; GenTree* falseVal = comp->gtNewZeroConNode(simdType); + GenTree* trueVal = node; + var_types nodeType = simdType; + + if (HWIntrinsicInfo::ReturnsPerElementMask(node->GetHWIntrinsicId())) + { + nodeType = TYP_MASK; + + GenTree* trueMaskForOp1 = comp->gtNewSimdAllTrueMaskNode(simdBaseJitType, simdSize); + BlockRange().InsertBefore(node, trueMaskForOp1); + BlockRange().InsertBefore(node, falseVal); + + falseVal = comp->gtNewSimdHWIntrinsicNode(TYP_MASK, trueMaskForOp1, falseVal, + NI_Sve_ConvertVectorToMask, simdBaseJitType, simdSize); + } GenTreeHWIntrinsic* condSelNode = - comp->gtNewSimdHWIntrinsicNode(simdType, trueMask, trueVal, falseVal, NI_Sve_ConditionalSelect, + comp->gtNewSimdHWIntrinsicNode(nodeType, trueMask, trueVal, falseVal, NI_Sve_ConditionalSelect, simdBaseJitType, simdSize); BlockRange().InsertBefore(node, trueMask); @@ -3365,6 +3380,32 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) } // Handle op3 + if (op1->IsMaskAllBitsSet()) + { + bool isZeroValue = false; + if (op3->IsVectorZero()) + { + isZeroValue = true; + } + else if (op3->OperIsHWIntrinsic(NI_Sve_ConvertVectorToMask)) + { + GenTreeHWIntrinsic* gtVectorToMask = op3->AsHWIntrinsic(); + + if (gtVectorToMask->Op(1)->IsMaskAllBitsSet() && gtVectorToMask->Op(2)->IsVectorZero()) + { + isZeroValue = true; + } + } + + if (isZeroValue) + { + // When we are merging with zero, we can specialize + // and avoid instantiating the vector constant. + // Do this only if op1 was AllTrueMask + MakeSrcContained(node, op3); + } + } + if (op3->IsVectorZero() && op1->IsMaskAllBitsSet()) { // When we are merging with zero, we can specialize @@ -3410,6 +3451,22 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) MakeSrcContained(node, intrin.op3); } break; + case NI_Sve_ConvertVectorToMask: + assert(varTypeIsMask(intrin.op1)); + assert(varTypeIsSIMD(intrin.op2)); + if (intrin.op1->IsMaskAllBitsSet() && intrin.op2->IsVectorZero()) + { + MakeSrcContained(node, intrin.op2); + } + break; + + case NI_Sve_ConvertMaskToVector: + assert(varTypeIsMask(intrin.op1)); + if (intrin.op1->IsVectorZero()) + { + MakeSrcContained(node, intrin.op1); + } + break; default: unreached(); diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 0256b105a774bf..fefb9534fdcb88 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -1997,8 +1997,10 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou { SingleTypeRegSet candidates = lowVectorOperandNum == 2 ? lowVectorCandidates : RBM_NONE; - if (intrin.op2->gtType == TYP_MASK) + if (intrin.op2->OperIsHWIntrinsic(NI_Sve_ConvertVectorToMask)) { + // Have RBM_ALLMASK candidates only if op2 is VectorToMask + assert(intrin.op2->gtType == TYP_MASK); assert(lowVectorOperandNum != 2); candidates = RBM_ALLMASK.GetPredicateRegSet(); } diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs index 493afebaab801e..026a920371287f 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs @@ -157,6 +157,66 @@ internal Arm64() { } /// public static unsafe Vector AbsoluteDifference(Vector left, Vector right) { throw new PlatformNotSupportedException(); } + /// Absolute compare greater than + + /// + /// svbool_t svacgt[_f32](svbool_t pg, svfloat32_t op1, svfloat32_t op2) + /// FACGT Presult.S, Pg/Z, Zop1.S, Zop2.S + /// + public static unsafe Vector AbsoluteCompareGreaterThan(Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svacgt[_f64](svbool_t pg, svfloat64_t op1, svfloat64_t op2) + /// FACGT Presult.D, Pg/Z, Zop1.D, Zop2.D + /// + public static unsafe Vector AbsoluteCompareGreaterThan(Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + + /// Absolute compare greater than or equal to + + /// + /// svbool_t svacge[_f32](svbool_t pg, svfloat32_t op1, svfloat32_t op2) + /// FACGE Presult.S, Pg/Z, Zop1.S, Zop2.S + /// + public static unsafe Vector AbsoluteCompareGreaterThanOrEqual(Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svacge[_f64](svbool_t pg, svfloat64_t op1, svfloat64_t op2) + /// FACGE Presult.D, Pg/Z, Zop1.D, Zop2.D + /// + public static unsafe Vector AbsoluteCompareGreaterThanOrEqual(Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + + /// Absolute compare less than + + /// + /// svbool_t svaclt[_f32](svbool_t pg, svfloat32_t op1, svfloat32_t op2) + /// FACLT Presult.S, Pg/Z, Zop1.S, Zop2.S + /// + public static unsafe Vector AbsoluteCompareLessThan(Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svaclt[_f64](svbool_t pg, svfloat64_t op1, svfloat64_t op2) + /// FACLT Presult.D, Pg/Z, Zop1.D, Zop2.D + /// + public static unsafe Vector AbsoluteCompareLessThan(Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + + /// Absolute compare less than or equal to + + /// + /// svbool_t svacle[_f32](svbool_t pg, svfloat32_t op1, svfloat32_t op2) + /// FACLE Presult.S, Pg/Z, Zop1.S, Zop2.S + /// + public static unsafe Vector AbsoluteCompareLessThanOrEqual(Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svacle[_f64](svbool_t pg, svfloat64_t op1, svfloat64_t op2) + /// FACLE Presult.D, Pg/Z, Zop1.D, Zop2.D + /// + public static unsafe Vector AbsoluteCompareLessThanOrEqual(Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// Add : Add /// diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs index fba023cbb54c60..2cc5669ea730a8 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs @@ -185,6 +185,66 @@ internal Arm64() { } /// public static unsafe Vector AbsoluteDifference(Vector left, Vector right) => AbsoluteDifference(left, right); + /// Absolute compare greater than + + /// + /// svbool_t svacgt[_f32](svbool_t pg, svfloat32_t op1, svfloat32_t op2) + /// FACGT Presult.S, Pg/Z, Zop1.S, Zop2.S + /// + public static unsafe Vector AbsoluteCompareGreaterThan(Vector left, Vector right) => AbsoluteCompareGreaterThan(left, right); + + /// + /// svbool_t svacgt[_f64](svbool_t pg, svfloat64_t op1, svfloat64_t op2) + /// FACGT Presult.D, Pg/Z, Zop1.D, Zop2.D + /// + public static unsafe Vector AbsoluteCompareGreaterThan(Vector left, Vector right) => AbsoluteCompareGreaterThan(left, right); + + + /// Absolute compare greater than or equal to + + /// + /// svbool_t svacge[_f32](svbool_t pg, svfloat32_t op1, svfloat32_t op2) + /// FACGE Presult.S, Pg/Z, Zop1.S, Zop2.S + /// + public static unsafe Vector AbsoluteCompareGreaterThanOrEqual(Vector left, Vector right) => AbsoluteCompareGreaterThanOrEqual(left, right); + + /// + /// svbool_t svacge[_f64](svbool_t pg, svfloat64_t op1, svfloat64_t op2) + /// FACGE Presult.D, Pg/Z, Zop1.D, Zop2.D + /// + public static unsafe Vector AbsoluteCompareGreaterThanOrEqual(Vector left, Vector right) => AbsoluteCompareGreaterThanOrEqual(left, right); + + + /// Absolute compare less than + + /// + /// svbool_t svaclt[_f32](svbool_t pg, svfloat32_t op1, svfloat32_t op2) + /// FACLT Presult.S, Pg/Z, Zop1.S, Zop2.S + /// + public static unsafe Vector AbsoluteCompareLessThan(Vector left, Vector right) => AbsoluteCompareLessThan(left, right); + + /// + /// svbool_t svaclt[_f64](svbool_t pg, svfloat64_t op1, svfloat64_t op2) + /// FACLT Presult.D, Pg/Z, Zop1.D, Zop2.D + /// + public static unsafe Vector AbsoluteCompareLessThan(Vector left, Vector right) => AbsoluteCompareLessThan(left, right); + + + /// Absolute compare less than or equal to + + /// + /// svbool_t svacle[_f32](svbool_t pg, svfloat32_t op1, svfloat32_t op2) + /// FACLE Presult.S, Pg/Z, Zop1.S, Zop2.S + /// + public static unsafe Vector AbsoluteCompareLessThanOrEqual(Vector left, Vector right) => AbsoluteCompareLessThanOrEqual(left, right); + + /// + /// svbool_t svacle[_f64](svbool_t pg, svfloat64_t op1, svfloat64_t op2) + /// FACLE Presult.D, Pg/Z, Zop1.D, Zop2.D + /// + public static unsafe Vector AbsoluteCompareLessThanOrEqual(Vector left, Vector right) => AbsoluteCompareLessThanOrEqual(left, right); + + /// Add : Add /// diff --git a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs index 3e85d1af8d5839..a26a2c0a234af9 100644 --- a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs +++ b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs @@ -4202,6 +4202,15 @@ internal Arm64() { } public static System.Numerics.Vector Abs(System.Numerics.Vector value) { throw null; } public static System.Numerics.Vector Abs(System.Numerics.Vector value) { throw null; } + public static System.Numerics.Vector AbsoluteCompareGreaterThan(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector AbsoluteCompareGreaterThan(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector AbsoluteCompareGreaterThanOrEqual(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector AbsoluteCompareGreaterThanOrEqual(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector AbsoluteCompareLessThan(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector AbsoluteCompareLessThan(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector AbsoluteCompareLessThanOrEqual(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector AbsoluteCompareLessThanOrEqual(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector AbsoluteDifference(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } public static System.Numerics.Vector AbsoluteDifference(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } public static System.Numerics.Vector AbsoluteDifference(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } diff --git a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs index 6a1b0391cdaff0..e4298dc67762a2 100644 --- a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs +++ b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs @@ -78,6 +78,16 @@ } }"; +const string SimpleVecOpTest_ValidationLogicForCndSelMask = @"for (var i = 0; i < RetElementCount; i++) + { + {Op1BaseType} iterResult = ({GetIterResult} != 0) ? trueVal[i] : falseVal[i]; + if (iterResult != result[i]) + { + succeeded = false; + break; + } + }"; + const string SimpleVecOpTest_ValidationLogicForCndSelForNarrowing = @"for (var i = 0; i < Op1ElementCount; i++) { {Op1BaseType} iterResult = (mask[i] != 0) ? {GetIterResult} : falseVal[i]; @@ -180,6 +190,7 @@ ("_SveUnaryOpDifferentRetTypeTestTemplate.template", "SveSimpleVecOpDifferentRetTypeTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogicForNarrowing, ["TemplateValidationLogicForCndSel"] = SimpleVecOpTest_ValidationLogicForCndSelForNarrowing }), ("_SveBinaryOpTestTemplate.template", "SveVecBinOpTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleVecOpTest_ValidationLogicForCndSel }), ("_SveBinaryOpTestTemplate.template", "SveVecBinOpVecTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_VectorValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleVecOpTest_VectorValidationLogicForCndSel }), + ("_SveBinaryPredOpTestTemplate.template", "SveVecBinPredOpTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleVecOpTest_ValidationLogicForCndSel, ["TemplateValidationLogicForCndSelMask"] = SimpleVecOpTest_ValidationLogicForCndSelMask }), ("_SveBinaryOpTestTemplate.template", "SveVecBinOpConvertTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleTernVecOpTest_ValidationLogicForCndSel }), ("_SveBinaryMaskOpTestTemplate.template", "SveMaskVecBinOpConvertTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleVecOpTest_ValidationLogicForCndSel }), ("_SveImmBinaryOpTestTemplate.template", "SveVecImmBinOpTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleVecOpTest_ValidationLogicForCndSel }), @@ -2970,6 +2981,15 @@ ("SveSimpleVecOpTest.template", new Dictionary { ["TestName"] = "Sve_Abs_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Abs", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "-TestLibrary.Generator.GetInt32()", ["ValidateIterResult"] = "Helpers.Abs(firstOp[i]) != result[i]", ["GetIterResult"] = "(int)Helpers.Abs(leftOp[i])"}), ("SveSimpleVecOpTest.template", new Dictionary { ["TestName"] = "Sve_Abs_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Abs", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "-TestLibrary.Generator.GetInt64()", ["ValidateIterResult"] = "(long)Helpers.Abs(firstOp[i]) != (long)result[i]", ["GetIterResult"] = "(long)Helpers.Abs(leftOp[i])"}), + ("SveVecBinPredOpTest.template", new Dictionary {["TestName"] = "Sve_AbsoluteCompareGreaterThan_float", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "AbsoluteCompareGreaterThan", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateIterResult"] = "Helpers.SveAbsoluteCompareGreaterThan(left[i], right[i]) != result[i]", ["GetIterResult"] = "Helpers.SveAbsoluteCompareGreaterThan(leftOp[i], rightOp[i])"}), + ("SveVecBinPredOpTest.template", new Dictionary {["TestName"] = "Sve_AbsoluteCompareGreaterThan_double", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "AbsoluteCompareGreaterThan", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ValidateIterResult"] = "Helpers.SveAbsoluteCompareGreaterThan(left[i], right[i]) != result[i]", ["GetIterResult"] = "Helpers.SveAbsoluteCompareGreaterThan(leftOp[i], rightOp[i])"}), + ("SveVecBinPredOpTest.template", new Dictionary {["TestName"] = "Sve_AbsoluteCompareGreaterThanOrEqual_float", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "AbsoluteCompareGreaterThanOrEqual", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateIterResult"] = "Helpers.SveAbsoluteCompareGreaterThanOrEqual(left[i], right[i]) != result[i]", ["GetIterResult"] = "Helpers.SveAbsoluteCompareGreaterThanOrEqual(leftOp[i], rightOp[i])"}), + ("SveVecBinPredOpTest.template", new Dictionary {["TestName"] = "Sve_AbsoluteCompareGreaterThanOrEqual_double", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "AbsoluteCompareGreaterThanOrEqual", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ValidateIterResult"] = "Helpers.SveAbsoluteCompareGreaterThanOrEqual(left[i], right[i]) != result[i]", ["GetIterResult"] = "Helpers.SveAbsoluteCompareGreaterThanOrEqual(leftOp[i], rightOp[i])"}), + ("SveVecBinPredOpTest.template", new Dictionary {["TestName"] = "Sve_AbsoluteCompareLessThan_float", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "AbsoluteCompareLessThan", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateIterResult"] = "Helpers.SveAbsoluteCompareLessThan(left[i], right[i]) != result[i]", ["GetIterResult"] = "Helpers.SveAbsoluteCompareLessThan(leftOp[i], rightOp[i])"}), + ("SveVecBinPredOpTest.template", new Dictionary {["TestName"] = "Sve_AbsoluteCompareLessThan_double", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "AbsoluteCompareLessThan", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ValidateIterResult"] = "Helpers.SveAbsoluteCompareLessThan(left[i], right[i]) != result[i]", ["GetIterResult"] = "Helpers.SveAbsoluteCompareLessThan(leftOp[i], rightOp[i])"}), + ("SveVecBinPredOpTest.template", new Dictionary {["TestName"] = "Sve_AbsoluteCompareLessThanOrEqual_float", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "AbsoluteCompareLessThanOrEqual", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateIterResult"] = "Helpers.SveAbsoluteCompareLessThanOrEqual(left[i], right[i]) != result[i]", ["GetIterResult"] = "Helpers.SveAbsoluteCompareLessThanOrEqual(leftOp[i], rightOp[i])"}), + ("SveVecBinPredOpTest.template", new Dictionary {["TestName"] = "Sve_AbsoluteCompareLessThanOrEqual_double", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "AbsoluteCompareLessThanOrEqual", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ValidateIterResult"] = "Helpers.SveAbsoluteCompareLessThanOrEqual(left[i], right[i]) != result[i]", ["GetIterResult"] = "Helpers.SveAbsoluteCompareLessThanOrEqual(leftOp[i], rightOp[i])"}), + ("SveVecBinOpConvertTest.template", new Dictionary { ["TestName"] = "Sve_AbsoluteDifference_float", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "AbsoluteDifference", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateIterResult"] = "BitConverter.SingleToInt32Bits(Helpers.AbsoluteDifference(left[i], right[i])) != BitConverter.SingleToInt32Bits(result[i])", ["GetIterResult"] = "Helpers.AbsoluteDifference(left[i], right[i])", ["ConvertFunc"] = "BitConverter.SingleToInt32Bits"}), ("SveVecBinOpConvertTest.template", new Dictionary { ["TestName"] = "Sve_AbsoluteDifference_double", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "AbsoluteDifference", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ValidateIterResult"] = "BitConverter.DoubleToInt64Bits(Helpers.AbsoluteDifference(left[i], right[i])) != BitConverter.DoubleToInt64Bits(result[i])", ["GetIterResult"] = "Helpers.AbsoluteDifference(left[i], right[i])", ["ConvertFunc"] = "BitConverter.DoubleToInt64Bits"}), ("SveVecBinOpConvertTest.template", new Dictionary { ["TestName"] = "Sve_AbsoluteDifference_sbyte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "AbsoluteDifference", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "(sbyte)TestLibrary.Generator.GetSByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["ValidateIterResult"] = "Helpers.AbsoluteDifference(left[i], right[i]) != result[i]", ["GetIterResult"] = "(sbyte)Helpers.AbsoluteDifference(left[i], right[i])", ["ConvertFunc"] = ""}), diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs index f47938b15f927b..b9bd7c60dfffd6 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs @@ -1528,6 +1528,270 @@ public static float CompareTest(float left, float right) return BitConverter.Int32BitsToSingle(result); } + public static double SveAbsoluteCompareGreaterThan(double left, double right) + { + long result = 0; + + left = Math.Abs(left); + right = Math.Abs(right); + + if (left > right) + { + result = 1; + } + + return BitConverter.Int64BitsToDouble(result); + } + + public static float SveAbsoluteCompareGreaterThan(float left, float right) + { + int result = 0; + + left = Math.Abs(left); + right = Math.Abs(right); + + if (left > right) + { + result = 1; + } + + return BitConverter.Int32BitsToSingle(result); + } + + public static double SveAbsoluteCompareGreaterThanOrEqual(double left, double right) + { + long result = 0; + + left = Math.Abs(left); + right = Math.Abs(right); + + if (left >= right) + { + result = 1; + } + + return BitConverter.Int64BitsToDouble(result); + } + + public static float SveAbsoluteCompareGreaterThanOrEqual(float left, float right) + { + int result = 0; + + left = Math.Abs(left); + right = Math.Abs(right); + + if (left >= right) + { + result = 1; + } + + return BitConverter.Int32BitsToSingle(result); + } + + public static double SveAbsoluteCompareLessThan(double left, double right) + { + long result = 0; + + left = Math.Abs(left); + right = Math.Abs(right); + + if (left < right) + { + result = 1; + } + + return BitConverter.Int64BitsToDouble(result); + } + + public static float SveAbsoluteCompareLessThan(float left, float right) + { + int result = 0; + + left = Math.Abs(left); + right = Math.Abs(right); + + if (left < right) + { + result = 1; + } + + return BitConverter.Int32BitsToSingle(result); + } + + public static double SveAbsoluteCompareLessThanOrEqual(double left, double right) + { + long result = 0; + + left = Math.Abs(left); + right = Math.Abs(right); + + if (left <= right) + { + result = 1; + } + + return BitConverter.Int64BitsToDouble(result); + } + + public static float SveAbsoluteCompareLessThanOrEqual(float left, float right) + { + int result = 0; + + left = Math.Abs(left); + right = Math.Abs(right); + + if (left <= right) + { + result = 1; + } + + return BitConverter.Int32BitsToSingle(result); + } + + public static double SveCompareEqual(double left, double right) + { + long result = 0; + + if (left == right) + { + result = 1; + } + + return BitConverter.Int64BitsToDouble(result); + } + + public static float SveCompareEqual(float left, float right) + { + int result = 0; + + if (left == right) + { + result = 1; + } + + return BitConverter.Int32BitsToSingle(result); + } + + public static double SveCompareGreaterThan(double left, double right) + { + long result = 0; + + if (left > right) + { + result = 1; + } + + return BitConverter.Int64BitsToDouble(result); + } + + public static float SveCompareGreaterThan(float left, float right) + { + int result = 0; + + if (left > right) + { + result = 1; + } + + return BitConverter.Int32BitsToSingle(result); + } + + public static double SveCompareGreaterThanOrEqual(double left, double right) + { + long result = 0; + + if (left >= right) + { + result = 1; + } + + return BitConverter.Int64BitsToDouble(result); + } + + public static float SveCompareGreaterThanOrEqual(float left, float right) + { + int result = 0; + + if (left >= right) + { + result = 1; + } + + return BitConverter.Int32BitsToSingle(result); + } + + public static double SveCompareLessThan(double left, double right) + { + long result = 0; + + if (left < right) + { + result = 1; + } + + return BitConverter.Int64BitsToDouble(result); + } + + public static float SveCompareLessThan(float left, float right) + { + int result = 0; + + if (left < right) + { + result = 1; + } + + return BitConverter.Int32BitsToSingle(result); + } + + public static double SveCompareLessThanOrEqual(double left, double right) + { + long result = 0; + + if (left <= right) + { + result = 1; + } + + return BitConverter.Int64BitsToDouble(result); + } + + public static float SveCompareLessThanOrEqual(float left, float right) + { + int result = 0; + + if (left <= right) + { + result = 1; + } + + return BitConverter.Int32BitsToSingle(result); + } + + public static double SveCompareTest(double left, double right) + { + long result = 0; + + if ((BitConverter.DoubleToInt64Bits(left) & BitConverter.DoubleToInt64Bits(right)) != 0) + { + result = 1; + } + + return BitConverter.Int64BitsToDouble(result); + } + + public static float SveCompareTest(float left, float right) + { + int result = 0; + + if ((BitConverter.SingleToInt32Bits(left) & BitConverter.SingleToInt32Bits(right)) != 0) + { + result = 1; + } + + return BitConverter.Int32BitsToSingle(result); + } + public static byte Abs(sbyte value) => value < 0 ? (byte)-value : (byte)value; public static ushort Abs(short value) => value < 0 ? (ushort)-value : (ushort)value; diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveBinaryPredOpTestTemplate.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveBinaryPredOpTestTemplate.template new file mode 100644 index 00000000000000..3a3ca84d593b28 --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveBinaryPredOpTestTemplate.template @@ -0,0 +1,494 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +/****************************************************************************** + * This file is auto-generated from a template file by the GenerateTests.csx * + * script in tests\src\JIT\HardwareIntrinsics.Arm\Shared. In order to make * + * changes, please update the corresponding template and run according to the * + * directions listed in the file. * + ******************************************************************************/ + +using System; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using Xunit; + +namespace JIT.HardwareIntrinsics.Arm +{ + public static partial class Program + { + [Fact] + public static void {TestName}() + { + var test = new {TemplateName}BinaryOpTest__{TestName}(); + + if (test.IsSupported) + { + // Validates basic functionality works, using Unsafe.Read + test.RunBasicScenario_UnsafeRead(); + + if ({LoadIsa}.IsSupported) + { + // Validates basic functionality works, using Load + test.RunBasicScenario_Load(); + + // Validates passing as a mask, using load + test.RunLoadMask(); + } + + // Validates calling via reflection works, using Unsafe.Read + test.RunReflectionScenario_UnsafeRead(); + + // Validates passing a local works, using Unsafe.Read + test.RunLclVarScenario_UnsafeRead(); + + // Validates passing an instance member of a class works + test.RunClassFldScenario(); + + // Validates passing the field of a local struct works + test.RunStructLclFldScenario(); + + // Validates passing an instance member of a struct works + test.RunStructFldScenario(); + + // Validates executing the test inside conditional, with op1 as falseValue + test.ConditionalSelect_Op1(); + + // Validates executing the test inside conditional, with op2 as falseValue + test.ConditionalSelect_Op2(); + + // Validates executing the test inside conditional, with op3 as falseValue + test.ConditionalSelect_FalseOp(); + + // Validates executing the test inside conditional, with op3 as zero + test.ConditionalSelect_ZeroOp(); + + // Validates passing as a conditional works + test.ConditionalSelect_MethodMask(); + } + else + { + // Validates we throw on unsupported hardware + test.RunUnsupportedScenario(); + } + + if (!test.Succeeded) + { + throw new Exception("One or more scenarios did not complete as expected."); + } + } + } + + public sealed unsafe class {TemplateName}BinaryOpTest__{TestName} + { + private struct DataTable + { + private byte[] inArray1; + private byte[] inArray2; + private byte[] outArray; + + private GCHandle inHandle1; + private GCHandle inHandle2; + private GCHandle outHandle; + + private ulong alignment; + + public DataTable({Op1BaseType}[] inArray1, {Op2BaseType}[] inArray2, {RetBaseType}[] outArray, int alignment) + { + int sizeOfinArray1 = inArray1.Length * Unsafe.SizeOf<{Op1BaseType}>(); + int sizeOfinArray2 = inArray2.Length * Unsafe.SizeOf<{Op2BaseType}>(); + int sizeOfoutArray = outArray.Length * Unsafe.SizeOf<{RetBaseType}>(); + if ((alignment != 64 && alignment != 16 && alignment != 8) || (alignment * 2) < sizeOfinArray1 || (alignment * 2) < sizeOfinArray2 || (alignment * 2) < sizeOfoutArray) + { + throw new ArgumentException($"Invalid value of alignment: {alignment}, sizeOfinArray1: {sizeOfinArray1}, sizeOfinArray2: {sizeOfinArray2}, sizeOfoutArray: {sizeOfoutArray}"); + } + + this.inArray1 = new byte[alignment * 2]; + this.inArray2 = new byte[alignment * 2]; + this.outArray = new byte[alignment * 2]; + + this.inHandle1 = GCHandle.Alloc(this.inArray1, GCHandleType.Pinned); + this.inHandle2 = GCHandle.Alloc(this.inArray2, GCHandleType.Pinned); + this.outHandle = GCHandle.Alloc(this.outArray, GCHandleType.Pinned); + + this.alignment = (ulong)alignment; + + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray1Ptr), ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), (uint)sizeOfinArray1); + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray2Ptr), ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), (uint)sizeOfinArray2); + } + + public void* inArray1Ptr => Align((byte*)(inHandle1.AddrOfPinnedObject().ToPointer()), alignment); + public void* inArray2Ptr => Align((byte*)(inHandle2.AddrOfPinnedObject().ToPointer()), alignment); + public void* outArrayPtr => Align((byte*)(outHandle.AddrOfPinnedObject().ToPointer()), alignment); + + public void Dispose() + { + inHandle1.Free(); + inHandle2.Free(); + outHandle.Free(); + } + + private static unsafe void* Align(byte* buffer, ulong expectedAlignment) + { + return (void*)(((ulong)buffer + expectedAlignment - 1) & ~(expectedAlignment - 1)); + } + } + + private struct TestStruct + { + public {Op1VectorType}<{Op1BaseType}> _fld1; + public {Op2VectorType}<{Op2BaseType}> _fld2; + + public static TestStruct Create() + { + var testStruct = new TestStruct(); + + for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref testStruct._fld1), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + for (var i = 0; i < Op2ElementCount; i++) { _data2[i] = {NextValueOp2}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2VectorType}<{Op2BaseType}>, byte>(ref testStruct._fld2), ref Unsafe.As<{Op2BaseType}, byte>(ref _data2[0]), (uint)Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>()); + + return testStruct; + } + + public void RunStructFldScenario({TemplateName}BinaryOpTest__{TestName} testClass) + { + var result = {Isa}.{Method}(_fld1, _fld2); + + Unsafe.Write(testClass._dataTable.outArrayPtr, result); + testClass.ValidateResult(_fld1, _fld2, testClass._dataTable.outArrayPtr); + } + } + + private static readonly int LargestVectorSize = {LargestVectorSize}; + + private static readonly int Op1ElementCount = Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>() / sizeof({Op1BaseType}); + private static readonly int Op2ElementCount = Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>() / sizeof({Op2BaseType}); + private static readonly int RetElementCount = Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>() / sizeof({RetBaseType}); + + private static {Op1BaseType}[] _maskData = new {Op1BaseType}[Op1ElementCount]; + private static {Op1BaseType}[] _data1 = new {Op1BaseType}[Op1ElementCount]; + private static {Op2BaseType}[] _data2 = new {Op2BaseType}[Op2ElementCount]; + + private {Op1VectorType}<{Op1BaseType}> _mask; + private {Op1VectorType}<{Op1BaseType}> _fld1; + private {Op2VectorType}<{Op2BaseType}> _fld2; + private {Op2VectorType}<{Op2BaseType}> _falseFld; + + private DataTable _dataTable; + + public {TemplateName}BinaryOpTest__{TestName}() + { + Succeeded = true; + + for (var i = 0; i < Op1ElementCount; i++) { _maskData[i] = ({Op1BaseType})({NextValueOp1} % 2); } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref _mask), ref Unsafe.As<{Op1BaseType}, byte>(ref _maskData[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref _fld1), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + for (var i = 0; i < Op2ElementCount; i++) { _data2[i] = {NextValueOp2}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2VectorType}<{Op2BaseType}>, byte>(ref _fld2), ref Unsafe.As<{Op2BaseType}, byte>(ref _data2[0]), (uint)Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2VectorType}<{Op2BaseType}>, byte>(ref _falseFld), ref Unsafe.As<{Op2BaseType}, byte>(ref _data2[0]), (uint)Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>()); + + for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } + for (var i = 0; i < Op2ElementCount; i++) { _data2[i] = {NextValueOp2}; } + _dataTable = new DataTable(_data1, _data2, new {RetBaseType}[RetElementCount], LargestVectorSize); + } + + public bool IsSupported => {Isa}.IsSupported; + + public bool Succeeded { get; set; } + + public void RunBasicScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_UnsafeRead)); + + var result = {Isa}.{Method}( + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr), + Unsafe.Read<{Op2VectorType}<{Op2BaseType}>>(_dataTable.inArray2Ptr) + ); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.outArrayPtr); + } + + public void RunBasicScenario_Load() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_Load)); + + {Op1VectorType}<{Op1BaseType}> loadMask = Sve.CreateTrueMask{RetBaseType}(SveMaskPattern.All); + + var result = {Isa}.{Method}( + {LoadIsa}.Load{Op1VectorType}(loadMask, ({Op1BaseType}*)(_dataTable.inArray1Ptr)), + {LoadIsa}.Load{Op2VectorType}(loadMask, ({Op2BaseType}*)(_dataTable.inArray2Ptr)) + ); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.outArrayPtr); + } + + public void RunLoadMask() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunLoadMask)); + + {Op1VectorType}<{Op1BaseType}> loadMask = {Isa}.{Method}(_fld1, _fld2); + + var result = {LoadIsa}.Load{Op1VectorType}(loadMask, ({Op1BaseType}*)(_dataTable.inArray1Ptr)); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateConditionalSelectMaskResult(_fld1, _fld2, Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr), {Op1VectorType}<{Op1BaseType}>.Zero, _dataTable.outArrayPtr); + } + + public void RunReflectionScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunReflectionScenario_UnsafeRead)); + + var result = typeof({Isa}).GetMethod(nameof({Isa}.{Method}), new Type[] { typeof({Op1VectorType}<{Op1BaseType}>), typeof({Op2VectorType}<{Op2BaseType}>) }) + .Invoke(null, new object[] { + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr), + Unsafe.Read<{Op2VectorType}<{Op2BaseType}>>(_dataTable.inArray2Ptr) + }); + + Unsafe.Write(_dataTable.outArrayPtr, ({RetVectorType}<{RetBaseType}>)(result)); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.outArrayPtr); + } + + public void RunLclVarScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunLclVarScenario_UnsafeRead)); + + var op1 = Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr); + var op2 = Unsafe.Read<{Op2VectorType}<{Op2BaseType}>>(_dataTable.inArray2Ptr); + var result = {Isa}.{Method}(op1, op2); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(op1, op2, _dataTable.outArrayPtr); + } + + public void RunClassFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunClassFldScenario)); + + var result = {Isa}.{Method}(_fld1, _fld2); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_fld1, _fld2, _dataTable.outArrayPtr); + } + + public void RunStructLclFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunStructLclFldScenario)); + + var test = TestStruct.Create(); + var result = {Isa}.{Method}(test._fld1, test._fld2); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(test._fld1, test._fld2, _dataTable.outArrayPtr); + } + + public void RunStructFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunStructFldScenario)); + + var test = TestStruct.Create(); + test.RunStructFldScenario(this); + } + + public void ConditionalSelect_Op1() + { + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_mask"); + ConditionalSelectScenario(_mask, _fld1, _fld2, _fld1); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_zero"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _fld1); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_all"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld1); + } + + public void ConditionalSelect_Op2() + { + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op2_mask"); + ConditionalSelectScenario(_mask, _fld1, _fld2, _fld2); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op2_zero"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _fld2); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op2_all"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld2); + } + + public void ConditionalSelect_FalseOp() + { + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_mask"); + ConditionalSelectScenario(_mask, _fld1, _fld2, _falseFld); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_zero"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _falseFld); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_all"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _falseFld); + } + + public void ConditionalSelect_ZeroOp() + { + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_mask"); + ConditionalSelectScenario(_mask, _fld1, _fld2, {Op1VectorType}<{Op1BaseType}>.Zero); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_zero"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, {Op1VectorType}<{Op1BaseType}>.Zero); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_all"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, {Op1VectorType}<{Op1BaseType}>.Zero); + } + + [method: MethodImpl(MethodImplOptions.AggressiveInlining)] + private void ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op1VectorType}<{Op1BaseType}> op2, {Op1VectorType}<{Op1BaseType}> falseOp) + { + var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2), falseOp); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateConditionalSelectResult(mask, op1, op2, falseOp, _dataTable.outArrayPtr); + } + + public void ConditionalSelect_MethodMask() + { + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_MethodMask"); + var result = Sve.ConditionalSelect({Isa}.{Method}(_fld1, _fld2), _fld1, _fld2); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateConditionalSelectMaskResult(_fld1, _fld2, _fld1, _fld2, _dataTable.outArrayPtr); + } + + public void RunUnsupportedScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunUnsupportedScenario)); + + bool succeeded = false; + + try + { + RunBasicScenario_UnsafeRead(); + } + catch (PlatformNotSupportedException) + { + succeeded = true; + } + + if (!succeeded) + { + Succeeded = false; + } + } + + private void ValidateConditionalSelectResult({Op1VectorType}<{Op1BaseType}> maskOp, {Op1VectorType}<{Op1BaseType}> leftOp, {Op1VectorType}<{Op1BaseType}> rightOp, {Op1VectorType}<{Op1BaseType}> falseOp, void* output, [CallerMemberName] string method = "") + { + {Op1BaseType}[] mask = new {Op1BaseType}[Op1ElementCount]; + {Op1BaseType}[] left = new {Op1BaseType}[Op1ElementCount]; + {Op1BaseType}[] right = new {Op1BaseType}[Op1ElementCount]; + {Op1BaseType}[] falseVal = new {Op1BaseType}[Op1ElementCount]; + {RetBaseType}[] result = new {RetBaseType}[RetElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref mask[0]), maskOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref left[0]), leftOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref right[0]), rightOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref falseVal[0]), falseOp); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref result[0]), ref Unsafe.AsRef(output), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + bool succeeded = true; + + {TemplateValidationLogicForCndSel} + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1VectorType}<{Op1BaseType}>, {Op2VectorType}<{Op2BaseType}>): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" mask: ({string.Join(", ", mask)})"); + TestLibrary.TestFramework.LogInformation($" left: ({string.Join(", ", left)})"); + TestLibrary.TestFramework.LogInformation($" right: ({string.Join(", ", right)})"); + TestLibrary.TestFramework.LogInformation($" falseOp: ({string.Join(", ", falseVal)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + + private void ValidateConditionalSelectMaskResult({Op1VectorType}<{Op1BaseType}> leftOp, {Op1VectorType}<{Op1BaseType}> rightOp, {Op1VectorType}<{Op1BaseType}> trueOp, {Op1VectorType}<{Op1BaseType}> falseOp, void* output, [CallerMemberName] string method = "") + { + {Op1BaseType}[] left = new {Op1BaseType}[Op1ElementCount]; + {Op1BaseType}[] right = new {Op1BaseType}[Op1ElementCount]; + {Op1BaseType}[] trueVal = new {Op1BaseType}[Op1ElementCount]; + {Op1BaseType}[] falseVal = new {Op1BaseType}[Op1ElementCount]; + {RetBaseType}[] result = new {RetBaseType}[RetElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref left[0]), leftOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref right[0]), rightOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref trueVal[0]), trueOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref falseVal[0]), falseOp); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref result[0]), ref Unsafe.AsRef(output), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + bool succeeded = true; + + {TemplateValidationLogicForCndSelMask} + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1VectorType}<{Op1BaseType}>, {Op2VectorType}<{Op2BaseType}>): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" left: ({string.Join(", ", left)})"); + TestLibrary.TestFramework.LogInformation($" right: ({string.Join(", ", right)})"); + TestLibrary.TestFramework.LogInformation($" trueOp: ({string.Join(", ", trueVal)})"); + TestLibrary.TestFramework.LogInformation($" falseOp: ({string.Join(", ", falseVal)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + + private void ValidateResult({Op1VectorType}<{Op1BaseType}> op1, {Op2VectorType}<{Op2BaseType}> op2, void* result, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; + {Op2BaseType}[] inArray2 = new {Op2BaseType}[Op2ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), op1); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), op2); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateResult(inArray1, inArray2, outArray, method); + } + + private void ValidateResult(void* op1, void* op2, void* result, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; + {Op2BaseType}[] inArray2 = new {Op2BaseType}[Op2ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), ref Unsafe.AsRef(op1), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), ref Unsafe.AsRef(op2), (uint)Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateResult(inArray1, inArray2, outArray, method); + } + + private void ValidateResult({Op1BaseType}[] left, {Op2BaseType}[] right, {RetBaseType}[] result, [CallerMemberName] string method = "") + { + bool succeeded = true; + + {TemplateValidationLogic} + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1VectorType}<{Op1BaseType}>, {Op2VectorType}<{Op2BaseType}>): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" left: ({string.Join(", ", left)})"); + TestLibrary.TestFramework.LogInformation($" right: ({string.Join(", ", right)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + } +}