From ae30cbaf993d757eec8bfc3420b152e1a7bc4af0 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Thu, 1 Aug 2024 09:11:05 -0700 Subject: [PATCH] Allow explicitly implemented ISimdVector APIs to be handled as intrinsic (#104983) * Allow explicitly implemented ISimdVector APIs to be handled as intrinsic * Apply formatting patch --- src/coreclr/jit/importercalls.cpp | 32 ++++++++ .../tools/Common/JitInterface/CorInfoImpl.cs | 38 ++++++++- .../src/System/Numerics/Vector_1.cs | 80 ++++++++++++++++++- .../System/Runtime/Intrinsics/Vector128_1.cs | 80 ++++++++++++++++++- .../System/Runtime/Intrinsics/Vector256_1.cs | 80 ++++++++++++++++++- .../System/Runtime/Intrinsics/Vector512_1.cs | 80 ++++++++++++++++++- src/mono/mono/mini/interp/transform-simd.c | 61 +++++++++++--- src/mono/mono/mini/intrinsics.c | 20 ++++- src/mono/mono/mini/simd-intrinsics.c | 66 ++++++++++++--- 9 files changed, 509 insertions(+), 28 deletions(-) diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 566fdceb8fdf8..bd885040308b1 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -10312,6 +10312,19 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method) else { #ifdef FEATURE_HW_INTRINSICS + if (strncmp(methodName, + "System.Runtime.Intrinsics.ISimdVector APIs to still be expanded where + // possible but, they all prefix the qualified name of the interface first, so we'll check + // for that and skip the prefix before trying to resolve the method. + + if (strncmp(methodName + 70, ",T>.", 7) == 0) + { + methodName += 77; + } + } + CORINFO_SIG_INFO sig; info.compCompHnd->getMethodSig(method, &sig); @@ -10530,6 +10543,25 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method) #error Unsupported platform #endif + if (strncmp(methodName, + "System.Runtime.Intrinsics.ISimdVector APIs to still be expanded where + // possible but, they all prefix the qualified name of the interface first, so we'll check + // for that and skip the prefix before trying to resolve the method. + + if (strncmp(methodName + 70, "64,T>.", 9) == 0) + { + methodName += 79; + } + else if ((strncmp(methodName + 70, "128,T>.", 10) == 0) || + (strncmp(methodName + 70, "256,T>.", 10) == 0) || + (strncmp(methodName + 70, "512,T>.", 10) == 0)) + { + methodName += 80; + } + } + if ((namespaceName[0] == '\0') || (strcmp(namespaceName, platformNamespaceName) == 0)) { CORINFO_SIG_INFO sig; diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs b/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs index 91146e192c102..d0ff9b5e6450b 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs @@ -4384,12 +4384,44 @@ private bool notifyInstructionSetUsage(InstructionSet instructionSet, bool suppo // By policy we code review all changes into corelib, such that failing to use an instruction // set is not a reason to not support usage of it. Except for functions which check if a given // feature is supported or hardware accelerated. - if (!isMethodDefinedInCoreLib() || - MethodBeingCompiled.Name == "get_IsSupported" || - MethodBeingCompiled.Name == "get_IsHardwareAccelerated") + if (!isMethodDefinedInCoreLib()) { _actualInstructionSetUnsupported.AddInstructionSet(instructionSet); } + else + { + ReadOnlySpan methodName = MethodBeingCompiled.Name.AsSpan(); + + if (methodName.StartsWith("System.Runtime.Intrinsics.ISimdVector APIs to still be expanded where possible + // but, they all prefix the qualified name of the interface first, so we'll check for that and + // skip the prefix before trying to resolve the method. + + ReadOnlySpan partialMethodName = methodName.Slice(70); + + if (partialMethodName.StartsWith(",T>.")) + { + methodName = partialMethodName.Slice(7); + } + else if (partialMethodName.StartsWith("64,T>.")) + { + methodName = partialMethodName.Slice(9); + } + else if (partialMethodName.StartsWith("128,T>.") || + partialMethodName.StartsWith("256,T>.") || + partialMethodName.StartsWith("512,T>.")) + { + methodName = partialMethodName.Slice(10); + } + } + + if (methodName.Equals("get_IsSupported", StringComparison.Ordinal) || + methodName.Equals("get_IsHardwareAccelerated", StringComparison.Ordinal)) + { + _actualInstructionSetUnsupported.AddInstructionSet(instructionSet); + } + } } return supportEnabled; } diff --git a/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector_1.cs b/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector_1.cs index dd59d3bf07885..e6cb7e1d4d4fa 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector_1.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector_1.cs @@ -825,36 +825,50 @@ public bool TryCopyTo(Span destination) static int ISimdVector, T>.Alignment => Vector.Alignment; /// - static bool ISimdVector, T>.IsHardwareAccelerated => Vector.IsHardwareAccelerated; + static bool ISimdVector, T>.IsHardwareAccelerated + { + [Intrinsic] + get => Vector.IsHardwareAccelerated; + } /// + [Intrinsic] static Vector ISimdVector, T>.Abs(Vector vector) => Vector.Abs(vector); /// + [Intrinsic] static Vector ISimdVector, T>.Add(Vector left, Vector right) => left + right; /// + [Intrinsic] static Vector ISimdVector, T>.AndNot(Vector left, Vector right) => Vector.AndNot(left, right); /// + [Intrinsic] static Vector ISimdVector, T>.BitwiseAnd(Vector left, Vector right) => left & right; /// + [Intrinsic] static Vector ISimdVector, T>.BitwiseOr(Vector left, Vector right) => left | right; /// + [Intrinsic] static Vector ISimdVector, T>.Ceiling(Vector vector) => Vector.Ceiling(vector); /// + [Intrinsic] static Vector ISimdVector, T>.Clamp(Vector value, Vector min, Vector max) => Vector.Clamp(value, min, max); /// + [Intrinsic] static Vector ISimdVector, T>.ClampNative(Vector value, Vector min, Vector max) => Vector.ClampNative(value, min, max); /// + [Intrinsic] static Vector ISimdVector, T>.ConditionalSelect(Vector condition, Vector left, Vector right) => Vector.ConditionalSelect(condition, left, right); /// + [Intrinsic] static Vector ISimdVector, T>.CopySign(Vector value, Vector sign) => Vector.CopySign(value, sign); /// @@ -867,6 +881,7 @@ public bool TryCopyTo(Span destination) static void ISimdVector, T>.CopyTo(Vector vector, Span destination) => vector.CopyTo(destination); /// + [Intrinsic] static Vector ISimdVector, T>.Create(T value) => Vector.Create(value); /// @@ -879,180 +894,238 @@ public bool TryCopyTo(Span destination) static Vector ISimdVector, T>.Create(ReadOnlySpan values) => Vector.Create(values); /// + [Intrinsic] static Vector ISimdVector, T>.CreateScalar(T value) => Vector.CreateScalar(value); /// + [Intrinsic] static Vector ISimdVector, T>.CreateScalarUnsafe(T value) => Vector.CreateScalarUnsafe(value); /// + [Intrinsic] static Vector ISimdVector, T>.Divide(Vector left, Vector right) => left / right; /// + [Intrinsic] static Vector ISimdVector, T>.Divide(Vector left, T right) => left / right; /// + [Intrinsic] static T ISimdVector, T>.Dot(Vector left, Vector right) => Vector.Dot(left, right); /// + [Intrinsic] static Vector ISimdVector, T>.Equals(Vector left, Vector right) => Vector.Equals(left, right); /// + [Intrinsic] static bool ISimdVector, T>.EqualsAll(Vector left, Vector right) => left == right; /// + [Intrinsic] static bool ISimdVector, T>.EqualsAny(Vector left, Vector right) => Vector.EqualsAny(left, right); /// + [Intrinsic] static Vector ISimdVector, T>.Floor(Vector vector) => Vector.Floor(vector); /// + [Intrinsic] static T ISimdVector, T>.GetElement(Vector vector, int index) => vector.GetElement(index); /// + [Intrinsic] static Vector ISimdVector, T>.GreaterThan(Vector left, Vector right) => Vector.GreaterThan(left, right); /// + [Intrinsic] static bool ISimdVector, T>.GreaterThanAll(Vector left, Vector right) => Vector.GreaterThanAll(left, right); /// + [Intrinsic] static bool ISimdVector, T>.GreaterThanAny(Vector left, Vector right) => Vector.GreaterThanAny(left, right); /// + [Intrinsic] static Vector ISimdVector, T>.GreaterThanOrEqual(Vector left, Vector right) => Vector.GreaterThanOrEqual(left, right); /// + [Intrinsic] static bool ISimdVector, T>.GreaterThanOrEqualAll(Vector left, Vector right) => Vector.GreaterThanOrEqualAll(left, right); /// + [Intrinsic] static bool ISimdVector, T>.GreaterThanOrEqualAny(Vector left, Vector right) => Vector.GreaterThanOrEqualAny(left, right); /// + [Intrinsic] static Vector ISimdVector, T>.LessThan(Vector left, Vector right) => Vector.LessThan(left, right); /// + [Intrinsic] static bool ISimdVector, T>.LessThanAll(Vector left, Vector right) => Vector.LessThanAll(left, right); /// + [Intrinsic] static bool ISimdVector, T>.LessThanAny(Vector left, Vector right) => Vector.LessThanAny(left, right); /// + [Intrinsic] static Vector ISimdVector, T>.LessThanOrEqual(Vector left, Vector right) => Vector.LessThanOrEqual(left, right); /// + [Intrinsic] static bool ISimdVector, T>.LessThanOrEqualAll(Vector left, Vector right) => Vector.LessThanOrEqualAll(left, right); /// + [Intrinsic] static bool ISimdVector, T>.LessThanOrEqualAny(Vector left, Vector right) => Vector.LessThanOrEqualAny(left, right); /// + [Intrinsic] static Vector ISimdVector, T>.Load(T* source) => Vector.Load(source); /// + [Intrinsic] static Vector ISimdVector, T>.LoadAligned(T* source) => Vector.LoadAligned(source); /// + [Intrinsic] static Vector ISimdVector, T>.LoadAlignedNonTemporal(T* source) => Vector.LoadAlignedNonTemporal(source); /// + [Intrinsic] static Vector ISimdVector, T>.LoadUnsafe(ref readonly T source) => Vector.LoadUnsafe(in source); /// + [Intrinsic] static Vector ISimdVector, T>.LoadUnsafe(ref readonly T source, nuint elementOffset) => Vector.LoadUnsafe(in source, elementOffset); /// + [Intrinsic] static Vector ISimdVector, T>.Max(Vector left, Vector right) => Vector.Max(left, right); /// + [Intrinsic] static Vector ISimdVector, T>.MaxMagnitude(Vector left, Vector right) => Vector.MaxMagnitude(left, right); /// + [Intrinsic] static Vector ISimdVector, T>.MaxMagnitudeNumber(Vector left, Vector right) => Vector.MaxMagnitudeNumber(left, right); /// + [Intrinsic] static Vector ISimdVector, T>.MaxNative(Vector left, Vector right) => Vector.MaxNative(left, right); /// + [Intrinsic] static Vector ISimdVector, T>.MaxNumber(Vector left, Vector right) => Vector.MaxNumber(left, right); /// + [Intrinsic] static Vector ISimdVector, T>.Min(Vector left, Vector right) => Vector.Min(left, right); /// + [Intrinsic] static Vector ISimdVector, T>.MinMagnitude(Vector left, Vector right) => Vector.MinMagnitude(left, right); /// + [Intrinsic] static Vector ISimdVector, T>.MinMagnitudeNumber(Vector left, Vector right) => Vector.MinMagnitudeNumber(left, right); /// + [Intrinsic] static Vector ISimdVector, T>.MinNative(Vector left, Vector right) => Vector.MinNative(left, right); /// + [Intrinsic] static Vector ISimdVector, T>.MinNumber(Vector left, Vector right) => Vector.MinNumber(left, right); /// + [Intrinsic] static Vector ISimdVector, T>.Multiply(Vector left, Vector right) => left * right; /// + [Intrinsic] static Vector ISimdVector, T>.Multiply(Vector left, T right) => left * right; /// + [Intrinsic] static Vector ISimdVector, T>.MultiplyAddEstimate(Vector left, Vector right, Vector addend) => Vector.MultiplyAddEstimate(left, right, addend); /// + [Intrinsic] static Vector ISimdVector, T>.Negate(Vector vector) => -vector; /// + [Intrinsic] static Vector ISimdVector, T>.OnesComplement(Vector vector) => ~vector; /// + [Intrinsic] static Vector ISimdVector, T>.Round(Vector vector) => Vector.Round(vector); /// + [Intrinsic] static Vector ISimdVector, T>.ShiftLeft(Vector vector, int shiftCount) => vector << shiftCount; /// + [Intrinsic] static Vector ISimdVector, T>.ShiftRightArithmetic(Vector vector, int shiftCount) => vector >> shiftCount; /// + [Intrinsic] static Vector ISimdVector, T>.ShiftRightLogical(Vector vector, int shiftCount) => vector >>> shiftCount; /// + [Intrinsic] static Vector ISimdVector, T>.Sqrt(Vector vector) => Vector.SquareRoot(vector); /// + [Intrinsic] static void ISimdVector, T>.Store(Vector source, T* destination) => source.Store(destination); /// + [Intrinsic] static void ISimdVector, T>.StoreAligned(Vector source, T* destination) => source.StoreAligned(destination); /// + [Intrinsic] static void ISimdVector, T>.StoreAlignedNonTemporal(Vector source, T* destination) => source.StoreAlignedNonTemporal(destination); /// + [Intrinsic] static void ISimdVector, T>.StoreUnsafe(Vector vector, ref T destination) => vector.StoreUnsafe(ref destination); /// + [Intrinsic] static void ISimdVector, T>.StoreUnsafe(Vector vector, ref T destination, nuint elementOffset) => vector.StoreUnsafe(ref destination, elementOffset); /// + [Intrinsic] static Vector ISimdVector, T>.Subtract(Vector left, Vector right) => left - right; /// + [Intrinsic] static T ISimdVector, T>.Sum(Vector vector) => Vector.Sum(vector); /// + [Intrinsic] static T ISimdVector, T>.ToScalar(Vector vector) => vector.ToScalar(); /// + [Intrinsic] static Vector ISimdVector, T>.Truncate(Vector vector) => Vector.Truncate(vector); /// static bool ISimdVector, T>.TryCopyTo(Vector vector, Span destination) => vector.TryCopyTo(destination); /// + [Intrinsic] static Vector ISimdVector, T>.WithElement(Vector vector, int index, T value) => vector.WithElement(index, value); /// + [Intrinsic] static Vector ISimdVector, T>.Xor(Vector left, Vector right) => left ^ right; // @@ -1083,14 +1156,19 @@ static int ISimdVector, T>.IndexOfLastMatch(Vector vector) } } + [Intrinsic] static Vector ISimdVector, T>.IsNaN(Vector vector) => Vector.IsNaN(vector); + [Intrinsic] static Vector ISimdVector, T>.IsNegative(Vector vector) => Vector.IsNegative(vector); + [Intrinsic] static Vector ISimdVector, T>.IsPositive(Vector vector) => Vector.IsPositive(vector); + [Intrinsic] static Vector ISimdVector, T>.IsPositiveInfinity(Vector vector) => Vector.IsPositiveInfinity(vector); + [Intrinsic] static Vector ISimdVector, T>.IsZero(Vector vector) => Vector.IsZero(vector); } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128_1.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128_1.cs index 1ec3556834821..9228778af2988 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128_1.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128_1.cs @@ -472,36 +472,50 @@ private string ToString([StringSyntax(StringSyntaxAttribute.NumericFormat)] stri static int ISimdVector, T>.Alignment => Vector128.Alignment; /// - static bool ISimdVector, T>.IsHardwareAccelerated => Vector128.IsHardwareAccelerated; + static bool ISimdVector, T>.IsHardwareAccelerated + { + [Intrinsic] + get => Vector128.IsHardwareAccelerated; + } /// + [Intrinsic] static Vector128 ISimdVector, T>.Abs(Vector128 vector) => Vector128.Abs(vector); /// + [Intrinsic] static Vector128 ISimdVector, T>.Add(Vector128 left, Vector128 right) => left + right; /// + [Intrinsic] static Vector128 ISimdVector, T>.AndNot(Vector128 left, Vector128 right) => Vector128.AndNot(left, right); /// + [Intrinsic] static Vector128 ISimdVector, T>.BitwiseAnd(Vector128 left, Vector128 right) => left & right; /// + [Intrinsic] static Vector128 ISimdVector, T>.BitwiseOr(Vector128 left, Vector128 right) => left | right; /// + [Intrinsic] static Vector128 ISimdVector, T>.Ceiling(Vector128 vector) => Vector128.Ceiling(vector); /// + [Intrinsic] static Vector128 ISimdVector, T>.Clamp(Vector128 value, Vector128 min, Vector128 max) => Vector128.Clamp(value, min, max); /// + [Intrinsic] static Vector128 ISimdVector, T>.ClampNative(Vector128 value, Vector128 min, Vector128 max) => Vector128.ClampNative(value, min, max); /// + [Intrinsic] static Vector128 ISimdVector, T>.ConditionalSelect(Vector128 condition, Vector128 left, Vector128 right) => Vector128.ConditionalSelect(condition, left, right); /// + [Intrinsic] static Vector128 ISimdVector, T>.CopySign(Vector128 value, Vector128 sign) => Vector128.CopySign(value, sign); /// @@ -514,6 +528,7 @@ private string ToString([StringSyntax(StringSyntaxAttribute.NumericFormat)] stri static void ISimdVector, T>.CopyTo(Vector128 vector, Span destination) => vector.CopyTo(destination); /// + [Intrinsic] static Vector128 ISimdVector, T>.Create(T value) => Vector128.Create(value); /// @@ -526,180 +541,238 @@ private string ToString([StringSyntax(StringSyntaxAttribute.NumericFormat)] stri static Vector128 ISimdVector, T>.Create(ReadOnlySpan values) => Vector128.Create(values); /// + [Intrinsic] static Vector128 ISimdVector, T>.CreateScalar(T value) => Vector128.CreateScalar(value); /// + [Intrinsic] static Vector128 ISimdVector, T>.CreateScalarUnsafe(T value) => Vector128.CreateScalarUnsafe(value); /// + [Intrinsic] static Vector128 ISimdVector, T>.Divide(Vector128 left, Vector128 right) => left / right; /// + [Intrinsic] static Vector128 ISimdVector, T>.Divide(Vector128 left, T right) => left / right; /// + [Intrinsic] static T ISimdVector, T>.Dot(Vector128 left, Vector128 right) => Vector128.Dot(left, right); /// + [Intrinsic] static Vector128 ISimdVector, T>.Equals(Vector128 left, Vector128 right) => Vector128.Equals(left, right); /// + [Intrinsic] static bool ISimdVector, T>.EqualsAll(Vector128 left, Vector128 right) => left == right; /// + [Intrinsic] static bool ISimdVector, T>.EqualsAny(Vector128 left, Vector128 right) => Vector128.EqualsAny(left, right); /// + [Intrinsic] static Vector128 ISimdVector, T>.Floor(Vector128 vector) => Vector128.Floor(vector); /// + [Intrinsic] static T ISimdVector, T>.GetElement(Vector128 vector, int index) => vector.GetElement(index); /// + [Intrinsic] static Vector128 ISimdVector, T>.GreaterThan(Vector128 left, Vector128 right) => Vector128.GreaterThan(left, right); /// + [Intrinsic] static bool ISimdVector, T>.GreaterThanAll(Vector128 left, Vector128 right) => Vector128.GreaterThanAll(left, right); /// + [Intrinsic] static bool ISimdVector, T>.GreaterThanAny(Vector128 left, Vector128 right) => Vector128.GreaterThanAny(left, right); /// + [Intrinsic] static Vector128 ISimdVector, T>.GreaterThanOrEqual(Vector128 left, Vector128 right) => Vector128.GreaterThanOrEqual(left, right); /// + [Intrinsic] static bool ISimdVector, T>.GreaterThanOrEqualAll(Vector128 left, Vector128 right) => Vector128.GreaterThanOrEqualAll(left, right); /// + [Intrinsic] static bool ISimdVector, T>.GreaterThanOrEqualAny(Vector128 left, Vector128 right) => Vector128.GreaterThanOrEqualAny(left, right); /// + [Intrinsic] static Vector128 ISimdVector, T>.LessThan(Vector128 left, Vector128 right) => Vector128.LessThan(left, right); /// + [Intrinsic] static bool ISimdVector, T>.LessThanAll(Vector128 left, Vector128 right) => Vector128.LessThanAll(left, right); /// + [Intrinsic] static bool ISimdVector, T>.LessThanAny(Vector128 left, Vector128 right) => Vector128.LessThanAny(left, right); /// + [Intrinsic] static Vector128 ISimdVector, T>.LessThanOrEqual(Vector128 left, Vector128 right) => Vector128.LessThanOrEqual(left, right); /// + [Intrinsic] static bool ISimdVector, T>.LessThanOrEqualAll(Vector128 left, Vector128 right) => Vector128.LessThanOrEqualAll(left, right); /// + [Intrinsic] static bool ISimdVector, T>.LessThanOrEqualAny(Vector128 left, Vector128 right) => Vector128.LessThanOrEqualAny(left, right); /// + [Intrinsic] static Vector128 ISimdVector, T>.Load(T* source) => Vector128.Load(source); /// + [Intrinsic] static Vector128 ISimdVector, T>.LoadAligned(T* source) => Vector128.LoadAligned(source); /// + [Intrinsic] static Vector128 ISimdVector, T>.LoadAlignedNonTemporal(T* source) => Vector128.LoadAlignedNonTemporal(source); /// + [Intrinsic] static Vector128 ISimdVector, T>.LoadUnsafe(ref readonly T source) => Vector128.LoadUnsafe(in source); /// + [Intrinsic] static Vector128 ISimdVector, T>.LoadUnsafe(ref readonly T source, nuint elementOffset) => Vector128.LoadUnsafe(in source, elementOffset); /// + [Intrinsic] static Vector128 ISimdVector, T>.Max(Vector128 left, Vector128 right) => Vector128.Max(left, right); /// + [Intrinsic] static Vector128 ISimdVector, T>.MaxMagnitude(Vector128 left, Vector128 right) => Vector128.MaxMagnitude(left, right); /// + [Intrinsic] static Vector128 ISimdVector, T>.MaxMagnitudeNumber(Vector128 left, Vector128 right) => Vector128.MaxMagnitudeNumber(left, right); /// + [Intrinsic] static Vector128 ISimdVector, T>.MaxNative(Vector128 left, Vector128 right) => Vector128.MaxNative(left, right); /// + [Intrinsic] static Vector128 ISimdVector, T>.MaxNumber(Vector128 left, Vector128 right) => Vector128.MaxNumber(left, right); /// + [Intrinsic] static Vector128 ISimdVector, T>.Min(Vector128 left, Vector128 right) => Vector128.Min(left, right); /// + [Intrinsic] static Vector128 ISimdVector, T>.MinMagnitude(Vector128 left, Vector128 right) => Vector128.MinMagnitude(left, right); /// + [Intrinsic] static Vector128 ISimdVector, T>.MinMagnitudeNumber(Vector128 left, Vector128 right) => Vector128.MinMagnitudeNumber(left, right); /// + [Intrinsic] static Vector128 ISimdVector, T>.MinNative(Vector128 left, Vector128 right) => Vector128.MinNative(left, right); /// + [Intrinsic] static Vector128 ISimdVector, T>.MinNumber(Vector128 left, Vector128 right) => Vector128.MinNumber(left, right); /// + [Intrinsic] static Vector128 ISimdVector, T>.Multiply(Vector128 left, Vector128 right) => left * right; /// + [Intrinsic] static Vector128 ISimdVector, T>.Multiply(Vector128 left, T right) => left * right; /// + [Intrinsic] static Vector128 ISimdVector, T>.MultiplyAddEstimate(Vector128 left, Vector128 right, Vector128 addend) => Vector128.MultiplyAddEstimate(left, right, addend); /// + [Intrinsic] static Vector128 ISimdVector, T>.Negate(Vector128 vector) => -vector; /// + [Intrinsic] static Vector128 ISimdVector, T>.OnesComplement(Vector128 vector) => ~vector; /// + [Intrinsic] static Vector128 ISimdVector, T>.Round(Vector128 vector) => Vector128.Round(vector); /// + [Intrinsic] static Vector128 ISimdVector, T>.ShiftLeft(Vector128 vector, int shiftCount) => vector << shiftCount; /// + [Intrinsic] static Vector128 ISimdVector, T>.ShiftRightArithmetic(Vector128 vector, int shiftCount) => vector >> shiftCount; /// + [Intrinsic] static Vector128 ISimdVector, T>.ShiftRightLogical(Vector128 vector, int shiftCount) => vector >>> shiftCount; /// + [Intrinsic] static Vector128 ISimdVector, T>.Sqrt(Vector128 vector) => Vector128.Sqrt(vector); /// + [Intrinsic] static void ISimdVector, T>.Store(Vector128 source, T* destination) => source.Store(destination); /// + [Intrinsic] static void ISimdVector, T>.StoreAligned(Vector128 source, T* destination) => source.StoreAligned(destination); /// + [Intrinsic] static void ISimdVector, T>.StoreAlignedNonTemporal(Vector128 source, T* destination) => source.StoreAlignedNonTemporal(destination); /// + [Intrinsic] static void ISimdVector, T>.StoreUnsafe(Vector128 vector, ref T destination) => vector.StoreUnsafe(ref destination); /// + [Intrinsic] static void ISimdVector, T>.StoreUnsafe(Vector128 vector, ref T destination, nuint elementOffset) => vector.StoreUnsafe(ref destination, elementOffset); /// + [Intrinsic] static Vector128 ISimdVector, T>.Subtract(Vector128 left, Vector128 right) => left - right; /// + [Intrinsic] static T ISimdVector, T>.Sum(Vector128 vector) => Vector128.Sum(vector); /// + [Intrinsic] static T ISimdVector, T>.ToScalar(Vector128 vector) => vector.ToScalar(); /// + [Intrinsic] static Vector128 ISimdVector, T>.Truncate(Vector128 vector) => Vector128.Truncate(vector); /// static bool ISimdVector, T>.TryCopyTo(Vector128 vector, Span destination) => vector.TryCopyTo(destination); /// + [Intrinsic] static Vector128 ISimdVector, T>.WithElement(Vector128 vector, int index, T value) => vector.WithElement(index, value); /// + [Intrinsic] static Vector128 ISimdVector, T>.Xor(Vector128 left, Vector128 right) => left ^ right; // @@ -716,14 +789,19 @@ static int ISimdVector, T>.IndexOfLastMatch(Vector128 vector) return 31 - BitOperations.LeadingZeroCount(mask); // 31 = 32 (bits in Int32) - 1 (indexing from zero) } + [Intrinsic] static Vector128 ISimdVector, T>.IsNaN(Vector128 vector) => Vector128.IsNaN(vector); + [Intrinsic] static Vector128 ISimdVector, T>.IsNegative(Vector128 vector) => Vector128.IsNegative(vector); + [Intrinsic] static Vector128 ISimdVector, T>.IsPositive(Vector128 vector) => Vector128.IsPositive(vector); + [Intrinsic] static Vector128 ISimdVector, T>.IsPositiveInfinity(Vector128 vector) => Vector128.IsPositiveInfinity(vector); + [Intrinsic] static Vector128 ISimdVector, T>.IsZero(Vector128 vector) => Vector128.IsZero(vector); } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256_1.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256_1.cs index a0d97807ac037..52df8ffcaf4f2 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256_1.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256_1.cs @@ -461,36 +461,50 @@ private string ToString([StringSyntax(StringSyntaxAttribute.NumericFormat)] stri static int ISimdVector, T>.Alignment => Vector256.Alignment; /// - static bool ISimdVector, T>.IsHardwareAccelerated => Vector256.IsHardwareAccelerated; + static bool ISimdVector, T>.IsHardwareAccelerated + { + [Intrinsic] + get => Vector256.IsHardwareAccelerated; + } /// + [Intrinsic] static Vector256 ISimdVector, T>.Abs(Vector256 vector) => Vector256.Abs(vector); /// + [Intrinsic] static Vector256 ISimdVector, T>.Add(Vector256 left, Vector256 right) => left + right; /// + [Intrinsic] static Vector256 ISimdVector, T>.AndNot(Vector256 left, Vector256 right) => Vector256.AndNot(left, right); /// + [Intrinsic] static Vector256 ISimdVector, T>.BitwiseAnd(Vector256 left, Vector256 right) => left & right; /// + [Intrinsic] static Vector256 ISimdVector, T>.BitwiseOr(Vector256 left, Vector256 right) => left | right; /// + [Intrinsic] static Vector256 ISimdVector, T>.Ceiling(Vector256 vector) => Vector256.Ceiling(vector); /// + [Intrinsic] static Vector256 ISimdVector, T>.Clamp(Vector256 value, Vector256 min, Vector256 max) => Vector256.Clamp(value, min, max); /// + [Intrinsic] static Vector256 ISimdVector, T>.ClampNative(Vector256 value, Vector256 min, Vector256 max) => Vector256.ClampNative(value, min, max); /// + [Intrinsic] static Vector256 ISimdVector, T>.ConditionalSelect(Vector256 condition, Vector256 left, Vector256 right) => Vector256.ConditionalSelect(condition, left, right); /// + [Intrinsic] static Vector256 ISimdVector, T>.CopySign(Vector256 value, Vector256 sign) => Vector256.CopySign(value, sign); /// @@ -503,6 +517,7 @@ private string ToString([StringSyntax(StringSyntaxAttribute.NumericFormat)] stri static void ISimdVector, T>.CopyTo(Vector256 vector, Span destination) => vector.CopyTo(destination); /// + [Intrinsic] static Vector256 ISimdVector, T>.Create(T value) => Vector256.Create(value); /// @@ -515,180 +530,238 @@ private string ToString([StringSyntax(StringSyntaxAttribute.NumericFormat)] stri static Vector256 ISimdVector, T>.Create(ReadOnlySpan values) => Vector256.Create(values); /// + [Intrinsic] static Vector256 ISimdVector, T>.CreateScalar(T value) => Vector256.CreateScalar(value); /// + [Intrinsic] static Vector256 ISimdVector, T>.CreateScalarUnsafe(T value) => Vector256.CreateScalarUnsafe(value); /// + [Intrinsic] static Vector256 ISimdVector, T>.Divide(Vector256 left, Vector256 right) => left / right; /// + [Intrinsic] static Vector256 ISimdVector, T>.Divide(Vector256 left, T right) => left / right; /// + [Intrinsic] static T ISimdVector, T>.Dot(Vector256 left, Vector256 right) => Vector256.Dot(left, right); /// + [Intrinsic] static Vector256 ISimdVector, T>.Equals(Vector256 left, Vector256 right) => Vector256.Equals(left, right); /// + [Intrinsic] static bool ISimdVector, T>.EqualsAll(Vector256 left, Vector256 right) => left == right; /// + [Intrinsic] static bool ISimdVector, T>.EqualsAny(Vector256 left, Vector256 right) => Vector256.EqualsAny(left, right); /// + [Intrinsic] static Vector256 ISimdVector, T>.Floor(Vector256 vector) => Vector256.Floor(vector); /// + [Intrinsic] static T ISimdVector, T>.GetElement(Vector256 vector, int index) => vector.GetElement(index); /// + [Intrinsic] static Vector256 ISimdVector, T>.GreaterThan(Vector256 left, Vector256 right) => Vector256.GreaterThan(left, right); /// + [Intrinsic] static bool ISimdVector, T>.GreaterThanAll(Vector256 left, Vector256 right) => Vector256.GreaterThanAll(left, right); /// + [Intrinsic] static bool ISimdVector, T>.GreaterThanAny(Vector256 left, Vector256 right) => Vector256.GreaterThanAny(left, right); /// + [Intrinsic] static Vector256 ISimdVector, T>.GreaterThanOrEqual(Vector256 left, Vector256 right) => Vector256.GreaterThanOrEqual(left, right); /// + [Intrinsic] static bool ISimdVector, T>.GreaterThanOrEqualAll(Vector256 left, Vector256 right) => Vector256.GreaterThanOrEqualAll(left, right); /// + [Intrinsic] static bool ISimdVector, T>.GreaterThanOrEqualAny(Vector256 left, Vector256 right) => Vector256.GreaterThanOrEqualAny(left, right); /// + [Intrinsic] static Vector256 ISimdVector, T>.LessThan(Vector256 left, Vector256 right) => Vector256.LessThan(left, right); /// + [Intrinsic] static bool ISimdVector, T>.LessThanAll(Vector256 left, Vector256 right) => Vector256.LessThanAll(left, right); /// + [Intrinsic] static bool ISimdVector, T>.LessThanAny(Vector256 left, Vector256 right) => Vector256.LessThanAny(left, right); /// + [Intrinsic] static Vector256 ISimdVector, T>.LessThanOrEqual(Vector256 left, Vector256 right) => Vector256.LessThanOrEqual(left, right); /// + [Intrinsic] static bool ISimdVector, T>.LessThanOrEqualAll(Vector256 left, Vector256 right) => Vector256.LessThanOrEqualAll(left, right); /// + [Intrinsic] static bool ISimdVector, T>.LessThanOrEqualAny(Vector256 left, Vector256 right) => Vector256.LessThanOrEqualAny(left, right); /// + [Intrinsic] static Vector256 ISimdVector, T>.Load(T* source) => Vector256.Load(source); /// + [Intrinsic] static Vector256 ISimdVector, T>.LoadAligned(T* source) => Vector256.LoadAligned(source); /// + [Intrinsic] static Vector256 ISimdVector, T>.LoadAlignedNonTemporal(T* source) => Vector256.LoadAlignedNonTemporal(source); /// + [Intrinsic] static Vector256 ISimdVector, T>.LoadUnsafe(ref readonly T source) => Vector256.LoadUnsafe(in source); /// + [Intrinsic] static Vector256 ISimdVector, T>.LoadUnsafe(ref readonly T source, nuint elementOffset) => Vector256.LoadUnsafe(in source, elementOffset); /// + [Intrinsic] static Vector256 ISimdVector, T>.Max(Vector256 left, Vector256 right) => Vector256.Max(left, right); /// + [Intrinsic] static Vector256 ISimdVector, T>.MaxMagnitude(Vector256 left, Vector256 right) => Vector256.MaxMagnitude(left, right); /// + [Intrinsic] static Vector256 ISimdVector, T>.MaxMagnitudeNumber(Vector256 left, Vector256 right) => Vector256.MaxMagnitudeNumber(left, right); /// + [Intrinsic] static Vector256 ISimdVector, T>.MaxNative(Vector256 left, Vector256 right) => Vector256.MaxNative(left, right); /// + [Intrinsic] static Vector256 ISimdVector, T>.MaxNumber(Vector256 left, Vector256 right) => Vector256.MaxNumber(left, right); /// + [Intrinsic] static Vector256 ISimdVector, T>.Min(Vector256 left, Vector256 right) => Vector256.Min(left, right); /// + [Intrinsic] static Vector256 ISimdVector, T>.MinMagnitude(Vector256 left, Vector256 right) => Vector256.MinMagnitude(left, right); /// + [Intrinsic] static Vector256 ISimdVector, T>.MinMagnitudeNumber(Vector256 left, Vector256 right) => Vector256.MinMagnitudeNumber(left, right); /// + [Intrinsic] static Vector256 ISimdVector, T>.MinNative(Vector256 left, Vector256 right) => Vector256.MinNative(left, right); /// + [Intrinsic] static Vector256 ISimdVector, T>.MinNumber(Vector256 left, Vector256 right) => Vector256.MinNumber(left, right); /// + [Intrinsic] static Vector256 ISimdVector, T>.Multiply(Vector256 left, Vector256 right) => left * right; /// + [Intrinsic] static Vector256 ISimdVector, T>.Multiply(Vector256 left, T right) => left * right; /// + [Intrinsic] static Vector256 ISimdVector, T>.MultiplyAddEstimate(Vector256 left, Vector256 right, Vector256 addend) => Vector256.MultiplyAddEstimate(left, right, addend); /// + [Intrinsic] static Vector256 ISimdVector, T>.Negate(Vector256 vector) => -vector; /// + [Intrinsic] static Vector256 ISimdVector, T>.OnesComplement(Vector256 vector) => ~vector; /// + [Intrinsic] static Vector256 ISimdVector, T>.Round(Vector256 vector) => Vector256.Round(vector); /// + [Intrinsic] static Vector256 ISimdVector, T>.ShiftLeft(Vector256 vector, int shiftCount) => vector << shiftCount; /// + [Intrinsic] static Vector256 ISimdVector, T>.ShiftRightArithmetic(Vector256 vector, int shiftCount) => vector >> shiftCount; /// + [Intrinsic] static Vector256 ISimdVector, T>.ShiftRightLogical(Vector256 vector, int shiftCount) => vector >>> shiftCount; /// + [Intrinsic] static Vector256 ISimdVector, T>.Sqrt(Vector256 vector) => Vector256.Sqrt(vector); /// + [Intrinsic] static void ISimdVector, T>.Store(Vector256 source, T* destination) => source.Store(destination); /// + [Intrinsic] static void ISimdVector, T>.StoreAligned(Vector256 source, T* destination) => source.StoreAligned(destination); /// + [Intrinsic] static void ISimdVector, T>.StoreAlignedNonTemporal(Vector256 source, T* destination) => source.StoreAlignedNonTemporal(destination); /// + [Intrinsic] static void ISimdVector, T>.StoreUnsafe(Vector256 vector, ref T destination) => vector.StoreUnsafe(ref destination); /// + [Intrinsic] static void ISimdVector, T>.StoreUnsafe(Vector256 vector, ref T destination, nuint elementOffset) => vector.StoreUnsafe(ref destination, elementOffset); /// + [Intrinsic] static Vector256 ISimdVector, T>.Subtract(Vector256 left, Vector256 right) => left - right; /// + [Intrinsic] static T ISimdVector, T>.Sum(Vector256 vector) => Vector256.Sum(vector); /// + [Intrinsic] static T ISimdVector, T>.ToScalar(Vector256 vector) => vector.ToScalar(); /// + [Intrinsic] static Vector256 ISimdVector, T>.Truncate(Vector256 vector) => Vector256.Truncate(vector); /// static bool ISimdVector, T>.TryCopyTo(Vector256 vector, Span destination) => vector.TryCopyTo(destination); /// + [Intrinsic] static Vector256 ISimdVector, T>.WithElement(Vector256 vector, int index, T value) => vector.WithElement(index, value); /// + [Intrinsic] static Vector256 ISimdVector, T>.Xor(Vector256 left, Vector256 right) => left ^ right; // @@ -705,14 +778,19 @@ static int ISimdVector, T>.IndexOfLastMatch(Vector256 vector) return 31 - BitOperations.LeadingZeroCount(mask); // 31 = 32 (bits in Int32) - 1 (indexing from zero) } + [Intrinsic] static Vector256 ISimdVector, T>.IsNaN(Vector256 vector) => Vector256.IsNaN(vector); + [Intrinsic] static Vector256 ISimdVector, T>.IsNegative(Vector256 vector) => Vector256.IsNegative(vector); + [Intrinsic] static Vector256 ISimdVector, T>.IsPositive(Vector256 vector) => Vector256.IsPositive(vector); + [Intrinsic] static Vector256 ISimdVector, T>.IsPositiveInfinity(Vector256 vector) => Vector256.IsPositiveInfinity(vector); + [Intrinsic] static Vector256 ISimdVector, T>.IsZero(Vector256 vector) => Vector256.IsZero(vector); } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512_1.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512_1.cs index 667d170fea491..e70dca3d3d6f3 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512_1.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512_1.cs @@ -461,36 +461,50 @@ private string ToString([StringSyntax(StringSyntaxAttribute.NumericFormat)] stri static int ISimdVector, T>.Alignment => Vector512.Alignment; /// - static bool ISimdVector, T>.IsHardwareAccelerated => Vector512.IsHardwareAccelerated; + static bool ISimdVector, T>.IsHardwareAccelerated + { + [Intrinsic] + get => Vector512.IsHardwareAccelerated; + } /// + [Intrinsic] static Vector512 ISimdVector, T>.Abs(Vector512 vector) => Vector512.Abs(vector); /// + [Intrinsic] static Vector512 ISimdVector, T>.Add(Vector512 left, Vector512 right) => left + right; /// + [Intrinsic] static Vector512 ISimdVector, T>.AndNot(Vector512 left, Vector512 right) => Vector512.AndNot(left, right); /// + [Intrinsic] static Vector512 ISimdVector, T>.BitwiseAnd(Vector512 left, Vector512 right) => left & right; /// + [Intrinsic] static Vector512 ISimdVector, T>.BitwiseOr(Vector512 left, Vector512 right) => left | right; /// + [Intrinsic] static Vector512 ISimdVector, T>.Ceiling(Vector512 vector) => Vector512.Ceiling(vector); /// + [Intrinsic] static Vector512 ISimdVector, T>.Clamp(Vector512 value, Vector512 min, Vector512 max) => Vector512.Clamp(value, min, max); /// + [Intrinsic] static Vector512 ISimdVector, T>.ClampNative(Vector512 value, Vector512 min, Vector512 max) => Vector512.ClampNative(value, min, max); /// + [Intrinsic] static Vector512 ISimdVector, T>.ConditionalSelect(Vector512 condition, Vector512 left, Vector512 right) => Vector512.ConditionalSelect(condition, left, right); /// + [Intrinsic] static Vector512 ISimdVector, T>.CopySign(Vector512 value, Vector512 sign) => Vector512.CopySign(value, sign); /// @@ -503,6 +517,7 @@ private string ToString([StringSyntax(StringSyntaxAttribute.NumericFormat)] stri static void ISimdVector, T>.CopyTo(Vector512 vector, Span destination) => vector.CopyTo(destination); /// + [Intrinsic] static Vector512 ISimdVector, T>.Create(T value) => Vector512.Create(value); /// @@ -515,180 +530,238 @@ private string ToString([StringSyntax(StringSyntaxAttribute.NumericFormat)] stri static Vector512 ISimdVector, T>.Create(ReadOnlySpan values) => Vector512.Create(values); /// + [Intrinsic] static Vector512 ISimdVector, T>.CreateScalar(T value) => Vector512.CreateScalar(value); /// + [Intrinsic] static Vector512 ISimdVector, T>.CreateScalarUnsafe(T value) => Vector512.CreateScalarUnsafe(value); /// + [Intrinsic] static Vector512 ISimdVector, T>.Divide(Vector512 left, Vector512 right) => left / right; /// + [Intrinsic] static Vector512 ISimdVector, T>.Divide(Vector512 left, T right) => left / right; /// + [Intrinsic] static T ISimdVector, T>.Dot(Vector512 left, Vector512 right) => Vector512.Dot(left, right); /// + [Intrinsic] static Vector512 ISimdVector, T>.Equals(Vector512 left, Vector512 right) => Vector512.Equals(left, right); /// + [Intrinsic] static bool ISimdVector, T>.EqualsAll(Vector512 left, Vector512 right) => left == right; /// + [Intrinsic] static bool ISimdVector, T>.EqualsAny(Vector512 left, Vector512 right) => Vector512.EqualsAny(left, right); /// + [Intrinsic] static Vector512 ISimdVector, T>.Floor(Vector512 vector) => Vector512.Floor(vector); /// + [Intrinsic] static T ISimdVector, T>.GetElement(Vector512 vector, int index) => vector.GetElement(index); /// + [Intrinsic] static Vector512 ISimdVector, T>.GreaterThan(Vector512 left, Vector512 right) => Vector512.GreaterThan(left, right); /// + [Intrinsic] static bool ISimdVector, T>.GreaterThanAll(Vector512 left, Vector512 right) => Vector512.GreaterThanAll(left, right); /// + [Intrinsic] static bool ISimdVector, T>.GreaterThanAny(Vector512 left, Vector512 right) => Vector512.GreaterThanAny(left, right); /// + [Intrinsic] static Vector512 ISimdVector, T>.GreaterThanOrEqual(Vector512 left, Vector512 right) => Vector512.GreaterThanOrEqual(left, right); /// + [Intrinsic] static bool ISimdVector, T>.GreaterThanOrEqualAll(Vector512 left, Vector512 right) => Vector512.GreaterThanOrEqualAll(left, right); /// + [Intrinsic] static bool ISimdVector, T>.GreaterThanOrEqualAny(Vector512 left, Vector512 right) => Vector512.GreaterThanOrEqualAny(left, right); /// + [Intrinsic] static Vector512 ISimdVector, T>.LessThan(Vector512 left, Vector512 right) => Vector512.LessThan(left, right); /// + [Intrinsic] static bool ISimdVector, T>.LessThanAll(Vector512 left, Vector512 right) => Vector512.LessThanAll(left, right); /// + [Intrinsic] static bool ISimdVector, T>.LessThanAny(Vector512 left, Vector512 right) => Vector512.LessThanAny(left, right); /// + [Intrinsic] static Vector512 ISimdVector, T>.LessThanOrEqual(Vector512 left, Vector512 right) => Vector512.LessThanOrEqual(left, right); /// + [Intrinsic] static bool ISimdVector, T>.LessThanOrEqualAll(Vector512 left, Vector512 right) => Vector512.LessThanOrEqualAll(left, right); /// + [Intrinsic] static bool ISimdVector, T>.LessThanOrEqualAny(Vector512 left, Vector512 right) => Vector512.LessThanOrEqualAny(left, right); /// + [Intrinsic] static Vector512 ISimdVector, T>.Load(T* source) => Vector512.Load(source); /// + [Intrinsic] static Vector512 ISimdVector, T>.LoadAligned(T* source) => Vector512.LoadAligned(source); /// + [Intrinsic] static Vector512 ISimdVector, T>.LoadAlignedNonTemporal(T* source) => Vector512.LoadAlignedNonTemporal(source); /// + [Intrinsic] static Vector512 ISimdVector, T>.LoadUnsafe(ref readonly T source) => Vector512.LoadUnsafe(in source); /// + [Intrinsic] static Vector512 ISimdVector, T>.LoadUnsafe(ref readonly T source, nuint elementOffset) => Vector512.LoadUnsafe(in source, elementOffset); /// + [Intrinsic] static Vector512 ISimdVector, T>.Max(Vector512 left, Vector512 right) => Vector512.Max(left, right); /// + [Intrinsic] static Vector512 ISimdVector, T>.MaxMagnitude(Vector512 left, Vector512 right) => Vector512.MaxMagnitude(left, right); /// + [Intrinsic] static Vector512 ISimdVector, T>.MaxMagnitudeNumber(Vector512 left, Vector512 right) => Vector512.MaxMagnitudeNumber(left, right); /// + [Intrinsic] static Vector512 ISimdVector, T>.MaxNative(Vector512 left, Vector512 right) => Vector512.MaxNative(left, right); /// + [Intrinsic] static Vector512 ISimdVector, T>.MaxNumber(Vector512 left, Vector512 right) => Vector512.MaxNumber(left, right); /// + [Intrinsic] static Vector512 ISimdVector, T>.Min(Vector512 left, Vector512 right) => Vector512.Min(left, right); /// + [Intrinsic] static Vector512 ISimdVector, T>.MinMagnitude(Vector512 left, Vector512 right) => Vector512.MinMagnitude(left, right); /// + [Intrinsic] static Vector512 ISimdVector, T>.MinMagnitudeNumber(Vector512 left, Vector512 right) => Vector512.MinMagnitudeNumber(left, right); /// + [Intrinsic] static Vector512 ISimdVector, T>.MinNative(Vector512 left, Vector512 right) => Vector512.MinNative(left, right); /// + [Intrinsic] static Vector512 ISimdVector, T>.MinNumber(Vector512 left, Vector512 right) => Vector512.MinNumber(left, right); /// + [Intrinsic] static Vector512 ISimdVector, T>.Multiply(Vector512 left, Vector512 right) => left * right; /// + [Intrinsic] static Vector512 ISimdVector, T>.Multiply(Vector512 left, T right) => left * right; /// + [Intrinsic] static Vector512 ISimdVector, T>.MultiplyAddEstimate(Vector512 left, Vector512 right, Vector512 addend) => Vector512.MultiplyAddEstimate(left, right, addend); /// + [Intrinsic] static Vector512 ISimdVector, T>.Negate(Vector512 vector) => -vector; /// + [Intrinsic] static Vector512 ISimdVector, T>.OnesComplement(Vector512 vector) => ~vector; /// + [Intrinsic] static Vector512 ISimdVector, T>.Round(Vector512 vector) => Vector512.Round(vector); /// + [Intrinsic] static Vector512 ISimdVector, T>.ShiftLeft(Vector512 vector, int shiftCount) => vector << shiftCount; /// + [Intrinsic] static Vector512 ISimdVector, T>.ShiftRightArithmetic(Vector512 vector, int shiftCount) => vector >> shiftCount; /// + [Intrinsic] static Vector512 ISimdVector, T>.ShiftRightLogical(Vector512 vector, int shiftCount) => vector >>> shiftCount; /// + [Intrinsic] static Vector512 ISimdVector, T>.Sqrt(Vector512 vector) => Vector512.Sqrt(vector); /// + [Intrinsic] static void ISimdVector, T>.Store(Vector512 source, T* destination) => source.Store(destination); /// + [Intrinsic] static void ISimdVector, T>.StoreAligned(Vector512 source, T* destination) => source.StoreAligned(destination); /// + [Intrinsic] static void ISimdVector, T>.StoreAlignedNonTemporal(Vector512 source, T* destination) => source.StoreAlignedNonTemporal(destination); /// + [Intrinsic] static void ISimdVector, T>.StoreUnsafe(Vector512 vector, ref T destination) => vector.StoreUnsafe(ref destination); /// + [Intrinsic] static void ISimdVector, T>.StoreUnsafe(Vector512 vector, ref T destination, nuint elementOffset) => vector.StoreUnsafe(ref destination, elementOffset); /// + [Intrinsic] static Vector512 ISimdVector, T>.Subtract(Vector512 left, Vector512 right) => left - right; /// + [Intrinsic] static T ISimdVector, T>.Sum(Vector512 vector) => Vector512.Sum(vector); /// + [Intrinsic] static T ISimdVector, T>.ToScalar(Vector512 vector) => vector.ToScalar(); /// + [Intrinsic] static Vector512 ISimdVector, T>.Truncate(Vector512 vector) => Vector512.Truncate(vector); /// static bool ISimdVector, T>.TryCopyTo(Vector512 vector, Span destination) => vector.TryCopyTo(destination); /// + [Intrinsic] static Vector512 ISimdVector, T>.WithElement(Vector512 vector, int index, T value) => vector.WithElement(index, value); /// + [Intrinsic] static Vector512 ISimdVector, T>.Xor(Vector512 left, Vector512 right) => left ^ right; // @@ -705,14 +778,19 @@ static int ISimdVector, T>.IndexOfLastMatch(Vector512 vector) return 63 - BitOperations.LeadingZeroCount(mask); // 63 = 64 (bits in Int64) - 1 (indexing from zero) } + [Intrinsic] static Vector512 ISimdVector, T>.IsNaN(Vector512 vector) => Vector512.IsNaN(vector); + [Intrinsic] static Vector512 ISimdVector, T>.IsNegative(Vector512 vector) => Vector512.IsNegative(vector); + [Intrinsic] static Vector512 ISimdVector, T>.IsPositive(Vector512 vector) => Vector512.IsPositive(vector); + [Intrinsic] static Vector512 ISimdVector, T>.IsPositiveInfinity(Vector512 vector) => Vector512.IsPositiveInfinity(vector); + [Intrinsic] static Vector512 ISimdVector, T>.IsZero(Vector512 vector) => Vector512.IsZero(vector); } } diff --git a/src/mono/mono/mini/interp/transform-simd.c b/src/mono/mono/mini/interp/transform-simd.c index 2896390bd96e5..c1d6b81019d02 100644 --- a/src/mono/mono/mini/interp/transform-simd.c +++ b/src/mono/mono/mini/interp/transform-simd.c @@ -42,9 +42,9 @@ simd_intrinsic_compare_by_name (const void *key, const void *value) } static int -lookup_intrins (guint16 *intrinsics, int size, MonoMethod *cmethod) +lookup_intrins (guint16 *intrinsics, int size, const char *cmethod_name) { - guint16 *result = mono_binary_search (cmethod->name, intrinsics, size / sizeof (guint16), sizeof (guint16), &simd_intrinsic_compare_by_name); + guint16 *result = mono_binary_search (cmethod_name, intrinsics, size / sizeof (guint16), sizeof (guint16), &simd_intrinsic_compare_by_name); if (result == NULL) return -1; @@ -414,7 +414,19 @@ emit_vector_create (TransformData *td, MonoMethodSignature *csignature, MonoClas static gboolean emit_sri_vector128 (TransformData *td, MonoMethod *cmethod, MonoMethodSignature *csignature) { - int id = lookup_intrins (sri_vector128_methods, sizeof (sri_vector128_methods), cmethod); + const char *cmethod_name = cmethod->name; + + if (strncmp(cmethod_name, "System.Runtime.Intrinsics.ISimdVector APIs to still be expanded where possible + // but, they all prefix the qualified name of the interface first, so we'll check for that and + // skip the prefix before trying to resolve the method. + + if (strncmp(cmethod_name + 70, "128,T>.", 10) == 0) { + cmethod_name += 80; + } + } + + int id = lookup_intrins (sri_vector128_methods, sizeof (sri_vector128_methods), cmethod_name); if (id == -1) return FALSE; @@ -614,9 +626,28 @@ emit_sri_vector128 (TransformData *td, MonoMethod *cmethod, MonoMethodSignature static gboolean emit_sri_vector128_t (TransformData *td, MonoMethod *cmethod, MonoMethodSignature *csignature) { - int id = lookup_intrins (sri_vector128_t_methods, sizeof (sri_vector128_t_methods), cmethod); - if (id == -1) - return FALSE; + const char *cmethod_name = cmethod->name; + bool explicitly_implemented = false; + + if (strncmp(cmethod_name, "System.Runtime.Intrinsics.ISimdVector APIs to still be expanded where possible + // but, they all prefix the qualified name of the interface first, so we'll check for that and + // skip the prefix before trying to resolve the method. + + if ((strncmp(cmethod_name + 70, "128,T>.", 10) == 0)) { + cmethod_name += 80; + explicitly_implemented = true; + } + } + + int id = lookup_intrins (sri_vector128_t_methods, sizeof (sri_vector128_t_methods), cmethod->name); + if (id == -1) { + if (explicitly_implemented) { + return emit_sri_vector128 (td, cmethod, csignature); + } else { + return FALSE; + } + } gint16 simd_opcode = -1; gint16 simd_intrins = -1; @@ -646,7 +677,19 @@ emit_sri_vector128_t (TransformData *td, MonoMethod *cmethod, MonoMethodSignatur static gboolean emit_sn_vector_t (TransformData *td, MonoMethod *cmethod, MonoMethodSignature *csignature, gboolean newobj) { - int id = lookup_intrins (sn_vector_t_methods, sizeof (sn_vector_t_methods), cmethod); + const char *cmethod_name = cmethod->name; + + if (strncmp(cmethod_name, "System.Runtime.Intrinsics.ISimdVector APIs to still be expanded where possible + // but, they all prefix the qualified name of the interface first, so we'll check for that and + // skip the prefix before trying to resolve the method. + + if (strncmp(cmethod_name + 70, ",T>.", 7) == 0) { + cmethod_name += 77; + } + } + + int id = lookup_intrins (sn_vector_t_methods, sizeof (sn_vector_t_methods), cmethod_name); if (id == -1) return FALSE; @@ -691,7 +734,7 @@ emit_sn_vector_t (TransformData *td, MonoMethod *cmethod, MonoMethodSignature *c static gboolean emit_sn_vector4 (TransformData *td, MonoMethod *cmethod, MonoMethodSignature *csignature, gboolean newobj) { - int id = lookup_intrins (sn_vector_t_methods, sizeof (sn_vector_t_methods), cmethod); + int id = lookup_intrins (sn_vector_t_methods, sizeof (sn_vector_t_methods), cmethod->name); if (id == -1) return FALSE; @@ -943,7 +986,7 @@ lookup_packedsimd_intrinsic (const char *name, MonoType *arg1) static gboolean emit_sri_packedsimd (TransformData *td, MonoMethod *cmethod, MonoMethodSignature *csignature) { - int id = lookup_intrins (sri_packedsimd_methods, sizeof (sri_packedsimd_methods), cmethod); + int id = lookup_intrins (sri_packedsimd_methods, sizeof (sri_packedsimd_methods), cmethod->name); // We don't early-out for an unrecognized method, we will generate an NIY later MonoClass *vector_klass = mono_class_from_mono_type_internal (csignature->ret); diff --git a/src/mono/mono/mini/intrinsics.c b/src/mono/mono/mini/intrinsics.c index 48e76d34f849e..b994a01c48745 100644 --- a/src/mono/mono/mini/intrinsics.c +++ b/src/mono/mono/mini/intrinsics.c @@ -2272,7 +2272,25 @@ mini_emit_inst_for_method (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSign if (in_corlib && ((!strcmp ("System.Numerics", cmethod_klass_name_space) && !strcmp ("Vector", cmethod_klass_name)) || !strncmp ("System.Runtime.Intrinsics", cmethod_klass_name_space, 25))) { - if (!strcmp (cmethod->name, "get_IsHardwareAccelerated")) { + const char* cmethod_name = cmethod->name; + + if (strncmp(cmethod_name, "System.Runtime.Intrinsics.ISimdVector APIs to still be expanded where possible + // but, they all prefix the qualified name of the interface first, so we'll check for that and + // skip the prefix before trying to resolve the method. + + if (strncmp(cmethod_name + 70, ",T>.", 7) == 0) { + cmethod_name += 77; + } else if (strncmp(cmethod_name + 70, "64,T>.", 9) == 0) { + cmethod_name += 79; + } else if ((strncmp(cmethod_name + 70, "128,T>.", 10) == 0) || + (strncmp(cmethod_name + 70, "256,T>.", 10) == 0) || + (strncmp(cmethod_name + 70, "512,T>.", 10) == 0)) { + cmethod_name += 80; + } + } + + if (!strcmp (cmethod_name, "get_IsHardwareAccelerated")) { EMIT_NEW_ICONST (cfg, ins, 0); ins->type = STACK_I4; return ins; diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index b387323b19312..47b1df99798cd 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -138,9 +138,9 @@ simd_intrinsic_info_compare_by_name (const void *key, const void *value) } static int -lookup_intrins (guint16 *intrinsics, int size, MonoMethod *cmethod) +lookup_intrins (guint16 *intrinsics, int size, const char *cmethod_name) { - const guint16 *result = (const guint16 *)mono_binary_search (cmethod->name, intrinsics, size / sizeof (guint16), sizeof (guint16), &simd_intrinsic_compare_by_name); + const guint16 *result = (const guint16 *)mono_binary_search (cmethod_name, intrinsics, size / sizeof (guint16), sizeof (guint16), &simd_intrinsic_compare_by_name); if (result == NULL) return -1; @@ -149,7 +149,7 @@ lookup_intrins (guint16 *intrinsics, int size, MonoMethod *cmethod) } static SimdIntrinsic* -lookup_intrins_info (SimdIntrinsic *intrinsics, int size, MonoMethod *cmethod) +lookup_intrins_info (SimdIntrinsic *intrinsics, int size, const char *cmethod_name) { #if 0 for (int i = 0; i < (size / sizeof (SimdIntrinsic)) - 1; ++i) { @@ -167,7 +167,7 @@ lookup_intrins_info (SimdIntrinsic *intrinsics, int size, MonoMethod *cmethod) } } #endif - return (SimdIntrinsic *)mono_binary_search (cmethod->name, intrinsics, size / sizeof (SimdIntrinsic), sizeof (SimdIntrinsic), &simd_intrinsic_info_compare_by_name); + return (SimdIntrinsic *)mono_binary_search (cmethod_name, intrinsics, size / sizeof (SimdIntrinsic), sizeof (SimdIntrinsic), &simd_intrinsic_info_compare_by_name); } static gboolean @@ -1074,7 +1074,7 @@ emit_hardware_intrinsics ( const SimdIntrinsic *intrinsics = intrin_group->intrinsics; int intrinsics_size = intrin_group->intrinsics_size; MonoCPUFeatures feature = intrin_group->feature; - const SimdIntrinsic *info = lookup_intrins_info ((SimdIntrinsic *) intrinsics, intrinsics_size, cmethod); + const SimdIntrinsic *info = lookup_intrins_info ((SimdIntrinsic *) intrinsics, intrinsics_size, cmethod->name); { if (!info) goto support_probe_complete; @@ -1922,7 +1922,25 @@ emit_dot (MonoCompile *cfg, MonoClass *klass, MonoType *vector_type, MonoTypeEnu static MonoInst* emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsig, MonoInst **args) { - int id = lookup_intrins (sri_vector_methods, sizeof (sri_vector_methods), cmethod); + const char *cmethod_name = cmethod->name; + + if (strncmp(cmethod_name, "System.Runtime.Intrinsics.ISimdVector APIs to still be expanded where possible + // but, they all prefix the qualified name of the interface first, so we'll check for that and + // skip the prefix before trying to resolve the method. + + if (strncmp(cmethod_name + 70, ",T>.", 7) == 0) { + cmethod_name += 77; + } else if (strncmp(cmethod_name + 70, "64,T>.", 9) == 0) { + cmethod_name += 79; + } else if ((strncmp(cmethod_name + 70, "128,T>.", 10) == 0) || + (strncmp(cmethod_name + 70, "256,T>.", 10) == 0) || + (strncmp(cmethod_name + 70, "512,T>.", 10) == 0)) { + cmethod_name += 80; + } + } + + int id = lookup_intrins (sri_vector_methods, sizeof (sri_vector_methods), cmethod_name); if (id == -1) { //check_no_intrinsic_cattr (cmethod); return NULL; @@ -3169,10 +3187,36 @@ static guint16 sri_vector_t_methods [] = { static MonoInst* emit_sri_vector_t (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsig, MonoInst **args) { - int id = lookup_intrins (sri_vector_t_methods, sizeof (sri_vector_t_methods), cmethod); + const char *cmethod_name = cmethod->name; + bool explicitly_implemented = false; + + if (strncmp(cmethod_name, "System.Runtime.Intrinsics.ISimdVector APIs to still be expanded where possible + // but, they all prefix the qualified name of the interface first, so we'll check for that and + // skip the prefix before trying to resolve the method. + + if (strncmp(cmethod_name + 70, ",T>.", 7) == 0) { + cmethod_name += 77; + explicitly_implemented = true; + } else if (strncmp(cmethod_name + 70, "64,T>.", 9) == 0) { + cmethod_name += 79; + explicitly_implemented = true; + } else if ((strncmp(cmethod_name + 70, "128,T>.", 10) == 0) || + (strncmp(cmethod_name + 70, "256,T>.", 10) == 0) || + (strncmp(cmethod_name + 70, "512,T>.", 10) == 0)) { + cmethod_name += 80; + explicitly_implemented = true; + } + } + + int id = lookup_intrins (sri_vector_t_methods, sizeof (sri_vector_t_methods), cmethod_name); if (id == -1) { - //check_no_intrinsic_cattr (cmethod); - return NULL; + if (explicitly_implemented) { + return emit_sri_vector (cfg, cmethod, fsig, args); + } else { + //check_no_intrinsic_cattr (cmethod); + return NULL; + } } MonoClass *klass = cmethod->klass; @@ -3430,7 +3474,7 @@ emit_vector_2_3_4 (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *f MonoClass *klass; MonoType *type, *etype; - id = lookup_intrins (vector_2_3_4_methods, sizeof (vector_2_3_4_methods), cmethod); + id = lookup_intrins (vector_2_3_4_methods, sizeof (vector_2_3_4_methods), cmethod->name); if (id == -1) { // https://github.com/dotnet/runtime/issues/81961 // check_no_intrinsic_cattr (cmethod); @@ -5982,7 +6026,7 @@ emit_wasm_zero_count (MonoCompile *cfg, MonoMethodSignature *fsig, MonoInst **ar static MonoInst* emit_wasm_bitoperations_intrinsics (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsig, MonoInst **args) { - int id = lookup_intrins (bitoperations_methods, sizeof (bitoperations_methods), cmethod); + int id = lookup_intrins (bitoperations_methods, sizeof (bitoperations_methods), cmethod->name); if (id == -1) { return NULL; }