diff --git a/src/coreclr/inc/corjit.h b/src/coreclr/inc/corjit.h index 122fcd0d7b0ba..ac44b8a9c8f3d 100644 --- a/src/coreclr/inc/corjit.h +++ b/src/coreclr/inc/corjit.h @@ -368,6 +368,7 @@ class ICorJitInfo : public ICorDynamicInfo FourByte = 1, EightByte = 2, TypeHandle = 3, + MethodHandle = 4, // Mask of all schema data types MarshalMask = 0xF, @@ -385,9 +386,10 @@ class ICorJitInfo : public ICorDynamicInfo Done = None, // All instrumentation schemas must end with a record which is "Done" BasicBlockIntCount = (DescriptorMin * 1) | FourByte, // basic block counter using unsigned 4 byte int BasicBlockLongCount = (DescriptorMin * 1) | EightByte, // basic block counter using unsigned 8 byte int - TypeHandleHistogramIntCount = (DescriptorMin * 2) | FourByte | AlignPointer, // 4 byte counter that is part of a type histogram. Aligned to match ClassProfile32's alignment. - TypeHandleHistogramLongCount = (DescriptorMin * 2) | EightByte, // 8 byte counter that is part of a type histogram - TypeHandleHistogramTypeHandle = (DescriptorMin * 3) | TypeHandle, // TypeHandle that is part of a type histogram + HandleHistogramIntCount = (DescriptorMin * 2) | FourByte | AlignPointer, // 4 byte counter that is part of a type histogram. Aligned to match ClassProfile32's alignment. + HandleHistogramLongCount = (DescriptorMin * 2) | EightByte, // 8 byte counter that is part of a type histogram + HandleHistogramTypes = (DescriptorMin * 3) | TypeHandle, // Histogram of type handles + HandleHistogramMethods = (DescriptorMin * 3) | MethodHandle, // Histogram of method handles Version = (DescriptorMin * 4) | None, // Version is encoded in the Other field of the schema NumRuns = (DescriptorMin * 5) | None, // Number of runs is encoded in the Other field of the schema EdgeIntCount = (DescriptorMin * 6) | FourByte, // edge counter using unsigned 4 byte int @@ -416,12 +418,12 @@ class ICorJitInfo : public ICorDynamicInfo }; #define DEFAULT_UNKNOWN_TYPEHANDLE 1 -#define UNKNOWN_TYPEHANDLE_MIN 1 -#define UNKNOWN_TYPEHANDLE_MAX 33 +#define UNKNOWN_HANDLE_MIN 1 +#define UNKNOWN_HANDLE_MAX 33 - static inline bool IsUnknownTypeHandle(intptr_t typeHandle) + static inline bool IsUnknownHandle(intptr_t typeHandle) { - return ((typeHandle >= UNKNOWN_TYPEHANDLE_MIN) && (typeHandle <= UNKNOWN_TYPEHANDLE_MAX)); + return ((typeHandle >= UNKNOWN_HANDLE_MIN) && (typeHandle <= UNKNOWN_HANDLE_MAX)); } // get profile information to be used for optimizing a current method. The format diff --git a/src/coreclr/inc/pgo_formatprocessing.h b/src/coreclr/inc/pgo_formatprocessing.h index 18fe5601c0d28..175d7d8786ba5 100644 --- a/src/coreclr/inc/pgo_formatprocessing.h +++ b/src/coreclr/inc/pgo_formatprocessing.h @@ -8,15 +8,7 @@ #ifdef FEATURE_PGO -inline bool AddTypeHandleToUnknownTypeHandleMask(INT_PTR typeHandle, uint32_t *unknownTypeHandleMask) -{ - uint32_t bitMask = (uint32_t)(1 << (typeHandle - UNKNOWN_TYPEHANDLE_MIN)); - bool result = (bitMask & *unknownTypeHandleMask) == 0; - *unknownTypeHandleMask |= bitMask; - return result; -} - -inline INT_PTR HashToPgoUnknownTypeHandle(uint32_t hash) +inline INT_PTR HashToPgoUnknownHandle(uint32_t hash) { // Map from a 32bit hash to the 32 different unknown type handle values return (hash & 0x1F) + 1; @@ -53,6 +45,7 @@ inline uint32_t InstrumentationKindToSize(ICorJitInfo::PgoInstrumentationKind ki case ICorJitInfo::PgoInstrumentationKind::EightByte: return 8; case ICorJitInfo::PgoInstrumentationKind::TypeHandle: + case ICorJitInfo::PgoInstrumentationKind::MethodHandle: return TARGET_POINTER_SIZE; default: _ASSERTE(FALSE); @@ -242,6 +235,7 @@ bool ReadInstrumentationData(const uint8_t *pByte, size_t cbDataMax, SchemaAndDa bool done = false; int64_t lastDataValue = 0; int64_t lastTypeDataValue = 0; + int64_t lastMethodDataValue = 0; int32_t dataCountToRead = 0; ReadCompressedInts(pByte, cbDataMax, [&](int64_t curValue) @@ -267,8 +261,16 @@ bool ReadInstrumentationData(const uint8_t *pByte, size_t cbDataMax, SchemaAndDa return false; } break; + case ICorJitInfo::PgoInstrumentationKind::MethodHandle: + lastMethodDataValue += curValue; + + if (!handler(schemaHandler.GetSchema(), lastMethodDataValue, schemaHandler.GetSchema().Count - dataCountToRead)) + { + return false; + } + break; default: - assert(false); + assert(!"Unexpected PGO instrumentation data type"); break; } dataCountToRead--; diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 10305239a81d5..a30c0c6113c91 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -6264,6 +6264,7 @@ class Compiler UINT32 fgPgoBlockCounts; UINT32 fgPgoEdgeCounts; UINT32 fgPgoClassProfiles; + UINT32 fgPgoMethodProfiles; unsigned fgPgoInlineePgo; unsigned fgPgoInlineeNoPgo; unsigned fgPgoInlineeNoPgoSingleBlock; diff --git a/src/coreclr/jit/fgbasic.cpp b/src/coreclr/jit/fgbasic.cpp index 632978641b9ee..e8881e283a44c 100644 --- a/src/coreclr/jit/fgbasic.cpp +++ b/src/coreclr/jit/fgbasic.cpp @@ -180,6 +180,7 @@ void Compiler::fgInit() fgPgoBlockCounts = 0; fgPgoEdgeCounts = 0; fgPgoClassProfiles = 0; + fgPgoMethodProfiles = 0; fgPgoInlineePgo = 0; fgPgoInlineeNoPgo = 0; fgPgoInlineeNoPgoSingleBlock = 0; diff --git a/src/coreclr/jit/fgprofile.cpp b/src/coreclr/jit/fgprofile.cpp index 25de6f11e0721..a5914e3696532 100644 --- a/src/coreclr/jit/fgprofile.cpp +++ b/src/coreclr/jit/fgprofile.cpp @@ -1496,15 +1496,15 @@ class BuildClassProbeSchemaGen } schemaElem.InstrumentationKind = JitConfig.JitCollect64BitCounts() - ? ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramLongCount - : ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramIntCount; + ? ICorJitInfo::PgoInstrumentationKind::HandleHistogramLongCount + : ICorJitInfo::PgoInstrumentationKind::HandleHistogramIntCount; schemaElem.ILOffset = (int32_t)call->gtClassProfileCandidateInfo->ilOffset; schemaElem.Offset = 0; m_schema.push_back(schemaElem); // Re-using ILOffset and Other fields from schema item for TypeHandleHistogramCount - schemaElem.InstrumentationKind = ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramTypeHandle; + schemaElem.InstrumentationKind = ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes; schemaElem.Count = ICorJitInfo::ClassProfile32::SIZE; m_schema.push_back(schemaElem); @@ -1551,9 +1551,9 @@ class ClassProbeInserter // assert(m_schema[*m_currentSchemaIndex].ILOffset == (int32_t)call->gtClassProfileCandidateInfo->ilOffset); bool is32 = m_schema[*m_currentSchemaIndex].InstrumentationKind == - ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramIntCount; + ICorJitInfo::PgoInstrumentationKind::HandleHistogramIntCount; bool is64 = m_schema[*m_currentSchemaIndex].InstrumentationKind == - ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramLongCount; + ICorJitInfo::PgoInstrumentationKind::HandleHistogramLongCount; assert(is32 || is64); // Figure out where the table is located. @@ -2050,12 +2050,32 @@ PhaseStatus Compiler::fgIncorporateProfileData() fgPgoEdgeCounts++; break; - case ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramIntCount: - case ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramLongCount: case ICorJitInfo::PgoInstrumentationKind::GetLikelyClass: fgPgoClassProfiles++; break; + case ICorJitInfo::PgoInstrumentationKind::HandleHistogramIntCount: + case ICorJitInfo::PgoInstrumentationKind::HandleHistogramLongCount: + if (iSchema + 1 < fgPgoSchemaCount) + { + if (fgPgoSchema[iSchema + 1].InstrumentationKind == + ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes) + { + fgPgoClassProfiles++; + iSchema++; + break; + } + if (fgPgoSchema[iSchema + 1].InstrumentationKind == + ICorJitInfo::PgoInstrumentationKind::HandleHistogramMethods) + { + fgPgoMethodProfiles++; + iSchema++; + break; + } + } + + __fallthrough; + default: JITDUMP("Unknown PGO record type 0x%x in schema entry %u (offset 0x%x count 0x%x other 0x%x)\n", fgPgoSchema[iSchema].InstrumentationKind, iSchema, fgPgoSchema[iSchema].ILOffset, @@ -2070,8 +2090,9 @@ PhaseStatus Compiler::fgIncorporateProfileData() fgNumProfileRuns = 1; } - JITDUMP("Profile summary: %d runs, %d block probes, %d edge probes, %d class profiles, %d other records\n", - fgNumProfileRuns, fgPgoBlockCounts, fgPgoEdgeCounts, fgPgoClassProfiles, otherRecords); + JITDUMP("Profile summary: %d runs, %d block probes, %d edge probes, %d class profiles, %d method profiles, %d " + "other records\n", + fgNumProfileRuns, fgPgoBlockCounts, fgPgoEdgeCounts, fgPgoClassProfiles, fgPgoMethodProfiles, otherRecords); const bool haveBlockCounts = fgPgoBlockCounts > 0; const bool haveEdgeCounts = fgPgoEdgeCounts > 0; diff --git a/src/coreclr/jit/likelyclass.cpp b/src/coreclr/jit/likelyclass.cpp index fc6028c0ac4ef..632c9ce8b847b 100644 --- a/src/coreclr/jit/likelyclass.cpp +++ b/src/coreclr/jit/likelyclass.cpp @@ -157,7 +157,7 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord* (schema[i].Count == 1)) { INT_PTR result = *(INT_PTR*)(pInstrumentationData + schema[i].Offset); - if (ICorJitInfo::IsUnknownTypeHandle(result)) + if (ICorJitInfo::IsUnknownHandle(result)) { return 0; } @@ -168,11 +168,11 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord* } const bool isHistogramCount = - (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramIntCount) || - (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramLongCount); + (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramIntCount) || + (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramLongCount); if (isHistogramCount && (schema[i].Count == 1) && ((i + 1) < countSchemaItems) && - (schema[i + 1].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramTypeHandle)) + (schema[i + 1].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes)) { // Form a histogram // @@ -191,7 +191,7 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord* { LikelyClassHistogramEntry const hist0 = h.HistogramEntryAt(0); // Fast path for monomorphic cases - if (ICorJitInfo::IsUnknownTypeHandle(hist0.m_mt)) + if (ICorJitInfo::IsUnknownHandle(hist0.m_mt)) { return 0; } @@ -205,12 +205,12 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord* LikelyClassHistogramEntry const hist0 = h.HistogramEntryAt(0); LikelyClassHistogramEntry const hist1 = h.HistogramEntryAt(1); // Fast path for two classes - if ((hist0.m_count >= hist1.m_count) && !ICorJitInfo::IsUnknownTypeHandle(hist0.m_mt)) + if ((hist0.m_count >= hist1.m_count) && !ICorJitInfo::IsUnknownHandle(hist0.m_mt)) { pLikelyClasses[0].likelihood = (100 * hist0.m_count) / h.m_totalCount; pLikelyClasses[0].clsHandle = (CORINFO_CLASS_HANDLE)hist0.m_mt; - if ((maxLikelyClasses > 1) && !ICorJitInfo::IsUnknownTypeHandle(hist1.m_mt)) + if ((maxLikelyClasses > 1) && !ICorJitInfo::IsUnknownHandle(hist1.m_mt)) { pLikelyClasses[1].likelihood = (100 * hist1.m_count) / h.m_totalCount; pLikelyClasses[1].clsHandle = (CORINFO_CLASS_HANDLE)hist1.m_mt; @@ -219,12 +219,12 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord* return 1; } - if (!ICorJitInfo::IsUnknownTypeHandle(hist1.m_mt)) + if (!ICorJitInfo::IsUnknownHandle(hist1.m_mt)) { pLikelyClasses[0].likelihood = (100 * hist1.m_count) / h.m_totalCount; pLikelyClasses[0].clsHandle = (CORINFO_CLASS_HANDLE)hist1.m_mt; - if ((maxLikelyClasses > 1) && !ICorJitInfo::IsUnknownTypeHandle(hist0.m_mt)) + if ((maxLikelyClasses > 1) && !ICorJitInfo::IsUnknownHandle(hist0.m_mt)) { pLikelyClasses[1].likelihood = (100 * hist0.m_count) / h.m_totalCount; pLikelyClasses[1].clsHandle = (CORINFO_CLASS_HANDLE)hist0.m_mt; @@ -244,7 +244,7 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord* for (unsigned m = 0; m < h.countHistogramElements; m++) { LikelyClassHistogramEntry const hist = h.HistogramEntryAt(m); - if (!ICorJitInfo::IsUnknownTypeHandle(hist.m_mt)) + if (!ICorJitInfo::IsUnknownHandle(hist.m_mt)) { sortedEntries[knownHandles++] = hist; } @@ -311,7 +311,7 @@ CORINFO_CLASS_HANDLE Compiler::getRandomClass(ICorJitInfo::PgoInstrumentationSch (schema[i].Count == 1)) { INT_PTR result = *(INT_PTR*)(pInstrumentationData + schema[i].Offset); - if (ICorJitInfo::IsUnknownTypeHandle(result)) + if (ICorJitInfo::IsUnknownHandle(result)) { return NO_CLASS_HANDLE; } @@ -322,11 +322,11 @@ CORINFO_CLASS_HANDLE Compiler::getRandomClass(ICorJitInfo::PgoInstrumentationSch } bool isHistogramCount = - (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramIntCount) || - (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramLongCount); + (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramIntCount) || + (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramLongCount); if (isHistogramCount && (schema[i].Count == 1) && ((i + 1) < countSchemaItems) && - (schema[i + 1].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramTypeHandle)) + (schema[i + 1].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes)) { // Form a histogram // @@ -342,7 +342,7 @@ CORINFO_CLASS_HANDLE Compiler::getRandomClass(ICorJitInfo::PgoInstrumentationSch unsigned randomEntryIndex = random->Next(0, h.countHistogramElements); LikelyClassHistogramEntry randomEntry = h.HistogramEntryAt(randomEntryIndex); - if (ICorJitInfo::IsUnknownTypeHandle(randomEntry.m_mt)) + if (ICorJitInfo::IsUnknownHandle(randomEntry.m_mt)) { return NO_CLASS_HANDLE; } diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs b/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs index 8af0179381d0c..aee2d053426d0 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs @@ -195,8 +195,7 @@ public static IEnumerable ConvertTypeHandleHistogramsToCompactTyp bool hasTypeHistogram = false; foreach (var elem in pgoData) { - if (elem.InstrumentationKind == PgoInstrumentationKind.TypeHandleHistogramIntCount || - elem.InstrumentationKind == PgoInstrumentationKind.TypeHandleHistogramLongCount) + if (elem.InstrumentationKind == PgoInstrumentationKind.HandleHistogramTypes) { // found histogram hasTypeHistogram = true; @@ -218,10 +217,12 @@ public static IEnumerable ConvertTypeHandleHistogramsToCompactTyp ComputeJitPgoInstrumentationSchema(LocalObjectToHandle, pgoData, out var nativeSchema, out var instrumentationData); - for (int i = 0; i < (pgoData.Length); i++) + for (int i = 0; i < pgoData.Length; i++) { - if (pgoData[i].InstrumentationKind == PgoInstrumentationKind.TypeHandleHistogramIntCount || - pgoData[i].InstrumentationKind == PgoInstrumentationKind.TypeHandleHistogramLongCount) + if ((i + 1 < pgoData.Length) && + (pgoData[i].InstrumentationKind == PgoInstrumentationKind.HandleHistogramIntCount || + pgoData[i].InstrumentationKind == PgoInstrumentationKind.HandleHistogramLongCount) && + (pgoData[i + 1].InstrumentationKind == PgoInstrumentationKind.HandleHistogramTypes)) { PgoSchemaElem? newElem = ComputeLikelyClass(i, handleToObject, nativeSchema, instrumentationData, compilationModuleGroup); if (newElem.HasValue) @@ -3863,6 +3864,10 @@ public static void ComputeJitPgoInstrumentationSchema(Func objec { ptrVal = (IntPtr)objectToHandle(typeVal.AsType); } + else if (typeVal.AsMethod != null) + { + ptrVal = (IntPtr)objectToHandle(typeVal.AsMethod); + } else { // The "Unknown types are the values from 1-33 diff --git a/src/coreclr/tools/Common/Pgo/PgoFormat.cs b/src/coreclr/tools/Common/Pgo/PgoFormat.cs index a7de96a03fd0f..a4e23286be4c6 100644 --- a/src/coreclr/tools/Common/Pgo/PgoFormat.cs +++ b/src/coreclr/tools/Common/Pgo/PgoFormat.cs @@ -21,6 +21,7 @@ public enum PgoInstrumentationKind FourByte = 1, EightByte = 2, TypeHandle = 3, + MethodHandle = 4, // Mask of all schema data types MarshalMask = 0xF, @@ -39,9 +40,10 @@ public enum PgoInstrumentationKind Done = None, // All instrumentation schemas must end with a record which is "Done" BasicBlockIntCount = (DescriptorMin * 1) | FourByte, // basic block counter using unsigned 4 byte int BasicBlockLongCount = (DescriptorMin * 1) | EightByte, // basic block counter using unsigned 8 byte int - TypeHandleHistogramIntCount = (DescriptorMin * 2) | FourByte | AlignPointer, // 4 byte counter that is part of a type histogram. Aligned to match ClassProfile32's alignment. - TypeHandleHistogramLongCount = (DescriptorMin * 2) | EightByte, // 8 byte counter that is part of a type histogram - TypeHandleHistogramTypeHandle = (DescriptorMin * 3) | TypeHandle, // TypeHandle that is part of a type histogram + HandleHistogramIntCount = (DescriptorMin * 2) | FourByte | AlignPointer, // 4 byte counter that is part of a type histogram. Aligned to match ClassProfile32's alignment. + HandleHistogramLongCount = (DescriptorMin * 2) | EightByte, // 8 byte counter that is part of a type histogram + HandleHistogramTypes = (DescriptorMin * 3) | TypeHandle, // TypeHandle that is part of a type histogram + HandleHistogramMethods = (DescriptorMin * 3) | MethodHandle, // TypeHandle that is part of a type histogram Version = (DescriptorMin * 4) | None, // Version is encoded in the Other field of the schema NumRuns = (DescriptorMin * 5) | None, // Number of runs is encoded in the Other field of the schema EdgeIntCount = (DescriptorMin * 6) | FourByte, // edge counter using unsigned 4 byte int @@ -49,14 +51,16 @@ public enum PgoInstrumentationKind GetLikelyClass = (DescriptorMin * 7) | TypeHandle, // Compressed get likely class data } - public interface IPgoSchemaDataLoader + public interface IPgoSchemaDataLoader { TType TypeFromLong(long input); + TMethod MethodFromLong(long input); } - public interface IPgoEncodedValueEmitter + public interface IPgoEncodedValueEmitter { void EmitType(TType type, TType previousValue); + void EmitMethod(TMethod method, TMethod previousValue); void EmitLong(long value, long previousValue); bool EmitDone(); } @@ -77,6 +81,8 @@ public struct PgoSchemaElem public bool DataHeldInDataLong => (Count == 1 && (((InstrumentationKind & PgoInstrumentationKind.MarshalMask) == PgoInstrumentationKind.FourByte) || ((InstrumentationKind & PgoInstrumentationKind.MarshalMask) == PgoInstrumentationKind.EightByte))); + + public override string ToString() => $"{InstrumentationKind} @ {ILOffset} Count = {Count} DataLong = {DataLong}"; } // Flags stored in 'Other' field of TypeHandleHistogram*Count entries. @@ -230,13 +236,14 @@ public static IEnumerable PgoEncodedCompressedLongGenerator(IEnumerable ParsePgoData(IPgoSchemaDataLoader dataProvider, IEnumerable inputDataStream, bool longsAreCompressed) + public static IEnumerable ParsePgoData(IPgoSchemaDataLoader dataProvider, IEnumerable inputDataStream, bool longsAreCompressed) { int dataCountToRead = 0; PgoSchemaElem curSchema = default(PgoSchemaElem); InstrumentationDataProcessingState processingState = InstrumentationDataProcessingState.UpdateProcessMaskFlag; long lastDataValue = 0; long lastTypeValue = 0; + long lastMethodValue = 0; foreach (long value in inputDataStream) { @@ -278,6 +285,14 @@ public static IEnumerable ParsePgoData(IPgoSchemaDataLoade lastTypeValue = value; ((TType[])curSchema.DataObject)[dataIndex] = dataProvider.TypeFromLong(lastTypeValue); break; + + case PgoInstrumentationKind.MethodHandle: + if (longsAreCompressed) + lastMethodValue += value; + else + lastMethodValue = value; + ((TMethod[])curSchema.DataObject)[dataIndex] = dataProvider.MethodFromLong(lastMethodValue); + break; } } dataCountToRead--; @@ -364,6 +379,10 @@ public static IEnumerable ParsePgoData(IPgoSchemaDataLoade curSchema.DataObject = new TType[curSchema.Count]; dataCountToRead = curSchema.Count; break; + case PgoInstrumentationKind.MethodHandle: + curSchema.DataObject = new TMethod[curSchema.Count]; + dataCountToRead = curSchema.Count; + break; default: throw new Exception("Unknown Type"); } @@ -373,10 +392,11 @@ public static IEnumerable ParsePgoData(IPgoSchemaDataLoade throw new Exception("Partial Instrumentation Data"); } - public static void EncodePgoData(IEnumerable schemas, IPgoEncodedValueEmitter valueEmitter, bool emitAllElementsUnconditionally) + public static void EncodePgoData(IEnumerable schemas, IPgoEncodedValueEmitter valueEmitter, bool emitAllElementsUnconditionally) { PgoSchemaElem prevSchema = default(PgoSchemaElem); TType prevEmittedType = default(TType); + TMethod prevEmittedMethod = default(TMethod); long prevEmittedIntData = 0; foreach (PgoSchemaElem schema in schemas) @@ -422,7 +442,8 @@ public static void EncodePgoData(IEnumerable schemas, IPgo for (int i = 0; i < schema.Count; i++) { - switch (schema.InstrumentationKind & PgoInstrumentationKind.MarshalMask) + PgoInstrumentationKind marshal = schema.InstrumentationKind & PgoInstrumentationKind.MarshalMask; + switch (marshal) { case PgoInstrumentationKind.None: break; @@ -463,6 +484,15 @@ public static void EncodePgoData(IEnumerable schemas, IPgo prevEmittedType = typeToEmit; break; } + case PgoInstrumentationKind.MethodHandle: + { + TMethod methodToEmit = ((TMethod[])schema.DataObject)[i]; + valueEmitter.EmitMethod(methodToEmit, prevEmittedMethod); + prevEmittedMethod = methodToEmit; + break; + } + default: + throw new ArgumentException("Unknown schema marshal " + marshal); } } @@ -512,7 +542,7 @@ public bool Equals(PgoSchemaElem x, PgoSchemaElem y) int IEqualityComparer.GetHashCode(PgoSchemaElem obj) => obj.ILOffset ^ ((int)(obj.InstrumentationKind & PgoInstrumentationKind.DescriptorMask) << 20); } - public static PgoSchemaElem[] Merge(ReadOnlySpan schemasToMerge) + public static PgoSchemaElem[] Merge(ReadOnlySpan schemasToMerge) { { // The merging algorithm will sort the schema data by iloffset, then by InstrumentationKind @@ -560,8 +590,8 @@ void MergeInSchemaElem(Dictionary dataMerger, PgoS case PgoInstrumentationKind.BasicBlockLongCount: case PgoInstrumentationKind.EdgeIntCount: case PgoInstrumentationKind.EdgeLongCount: - case PgoInstrumentationKind.TypeHandleHistogramIntCount: - case PgoInstrumentationKind.TypeHandleHistogramLongCount: + case PgoInstrumentationKind.HandleHistogramIntCount: + case PgoInstrumentationKind.HandleHistogramLongCount: if ((existingSchemaItem.Count != 1) || (schema.Count != 1)) { throw new Exception("Unable to merge pgo data. Invalid format"); @@ -569,7 +599,7 @@ void MergeInSchemaElem(Dictionary dataMerger, PgoS mergedElem.DataLong = existingSchemaItem.DataLong + schema.DataLong; break; - case PgoInstrumentationKind.TypeHandleHistogramTypeHandle: + case PgoInstrumentationKind.HandleHistogramTypes: { mergedElem.Count = existingSchemaItem.Count + schema.Count; TType[] newMergedTypeArray = new TType[mergedElem.Count]; @@ -586,6 +616,23 @@ void MergeInSchemaElem(Dictionary dataMerger, PgoS break; } + case PgoInstrumentationKind.HandleHistogramMethods: + { + mergedElem.Count = existingSchemaItem.Count + schema.Count; + TMethod[] newMergedMethodArray = new TMethod[mergedElem.Count]; + mergedElem.DataObject = newMergedMethodArray; + int i = 0; + foreach (TMethod meth in (TMethod[])existingSchemaItem.DataObject) + { + newMergedMethodArray[i++] = meth; + } + foreach (TMethod meth in (TMethod[])schema.DataObject) + { + newMergedMethodArray[i++] = meth; + } + break; + } + case PgoInstrumentationKind.Version: { mergedElem.Other = Math.Max(existingSchemaItem.Other, schema.Other); diff --git a/src/coreclr/tools/Common/Pgo/TypeSystemEntityOrUnknown.cs b/src/coreclr/tools/Common/Pgo/TypeSystemEntityOrUnknown.cs index e8355b9b4d0a9..6eabe55cf7cc5 100644 --- a/src/coreclr/tools/Common/Pgo/TypeSystemEntityOrUnknown.cs +++ b/src/coreclr/tools/Common/Pgo/TypeSystemEntityOrUnknown.cs @@ -18,6 +18,11 @@ public TypeSystemEntityOrUnknown(TypeDesc type) _data = type; } + public TypeSystemEntityOrUnknown(MethodDesc method) + { + _data = method; + } + readonly object _data; public TypeDesc AsType => _data as TypeDesc; public MethodDesc AsMethod => _data as MethodDesc; diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/InstrumentationDataTableNode.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/InstrumentationDataTableNode.cs index 90883bf0b073d..c73888a49c357 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/InstrumentationDataTableNode.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/InstrumentationDataTableNode.cs @@ -38,7 +38,7 @@ public void Initialize(ReadyToRunSymbolNodeFactory symbolNodeFactory) _symbolNodeFactory = symbolNodeFactory; } - class PgoValueEmitter : IPgoEncodedValueEmitter + class PgoValueEmitter : IPgoEncodedValueEmitter { public PgoValueEmitter(CompilationModuleGroup compilationGroup, ReadyToRunSymbolNodeFactory factory, bool actuallyCaptureOutput) { @@ -52,14 +52,18 @@ public void Clear() _longs.Clear(); _imports.Clear(); _typeConversions.Clear(); + _methodConversions.Clear(); _unknownTypesFound = 0; + _unknownMethodsFound = 0; } public IReadOnlyList ReferencedImports => _imports; List _longs = new List(); List _imports = new List(); Dictionary _typeConversions = new Dictionary(); + Dictionary _methodConversions = new Dictionary(); int _unknownTypesFound = 0; + int _unknownMethodsFound = 0; CompilationModuleGroup _compilationGroup; ReadyToRunSymbolNodeFactory _symbolFactory; bool _actuallyCaptureOutput; @@ -84,18 +88,23 @@ public void EmitType(TypeSystemEntityOrUnknown type, TypeSystemEntityOrUnknown p EmitLong(TypeToInt(type), TypeToInt(previousValue)); } - private int TypeToInt(TypeSystemEntityOrUnknown type) + public void EmitMethod(TypeSystemEntityOrUnknown method, TypeSystemEntityOrUnknown previousValue) { - if (type.IsNull || (type.AsType == null && type.AsUnknown == 0)) + EmitLong(MethodToInt(method), MethodToInt(previousValue)); + } + + private int TypeToInt(TypeSystemEntityOrUnknown handle) + { + if (handle.IsNull || (handle.AsType == null && handle.AsUnknown == 0)) return 0; - if (_typeConversions.TryGetValue(type, out int computedInt)) + if (_typeConversions.TryGetValue(handle, out int computedInt)) { return computedInt; } - if (type.AsType != null && _compilationGroup.VersionsWithTypeReference(type.AsType)) + if (handle.AsType != null && _compilationGroup.VersionsWithTypeReference(handle.AsType)) { - Import typeHandleImport = (Import)_symbolFactory.CreateReadyToRunHelper(ReadyToRunHelperId.TypeHandle, type.AsType); + Import typeHandleImport = (Import)_symbolFactory.CreateReadyToRunHelper(ReadyToRunHelperId.TypeHandle, handle.AsType); _imports.Add(typeHandleImport); if (_actuallyCaptureOutput) @@ -120,7 +129,51 @@ private int TypeToInt(TypeSystemEntityOrUnknown type) { computedInt = ((++_unknownTypesFound) << 4) | 0xF; } - _typeConversions.Add(type, computedInt); + _typeConversions.Add(handle, computedInt); + return computedInt; + } + + private int MethodToInt(TypeSystemEntityOrUnknown handle) + { + if (handle.IsNull || (handle.AsMethod == null && handle.AsUnknown == 0)) + return 0; + + if (_methodConversions.TryGetValue(handle, out int computedInt)) + { + return computedInt; + } + if (handle.AsMethod != null && _compilationGroup.VersionsWithMethodBody(handle.AsMethod)) + { + EcmaMethod typicalMethod = (EcmaMethod)handle.AsMethod.GetTypicalMethodDefinition(); + ModuleToken moduleToken = new ModuleToken(typicalMethod.Module, typicalMethod.Handle); + + MethodWithToken tok = new MethodWithToken(handle.AsMethod, moduleToken, constrainedType: null, unboxing: false, context: null); + Import methodHandleImport = (Import)_symbolFactory.CreateReadyToRunHelper(ReadyToRunHelperId.MethodHandle, tok); + _imports.Add(methodHandleImport); + + if (_actuallyCaptureOutput) + { + if (methodHandleImport.Table.IndexFromBeginningOfArray >= 0xF) + { + // The current implementation of this table only allows for 15 different + // import tables to be used. This is probably enough for long term + // but this code will throw if we use more import tables and attempt + // to encode pgo data + throw new Exception("Unexpected high index for table import"); + } + + computedInt = (methodHandleImport.IndexFromBeginningOfArray << 4) | methodHandleImport.Table.IndexFromBeginningOfArray; + } + else + { + computedInt = _imports.Count << 1; + } + } + else + { + computedInt = ((++_unknownMethodsFound) << 4) | 0xF; + } + _methodConversions.Add(handle, computedInt); return computedInt; } } diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ProfileData.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ProfileData.cs index 5220e13f813ee..dc5641a83366d 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ProfileData.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ProfileData.cs @@ -111,7 +111,7 @@ public static void MergeProfileData(ref bool partialNgen, Dictionary(schemaElemMergerArray); + mergedSchemaData = PgoProcessor.Merge(schemaElemMergerArray); } mergedProfileData[data.Method] = new MethodProfileData(data.Method, dataToMerge.Flags | data.Flags, data.ExclusiveWeight + dataToMerge.ExclusiveWeight, mergedCallWeights, dataToMerge.ScenarioMask | data.ScenarioMask, mergedSchemaData); } diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/IBC/MIbcProfileParser.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/IBC/MIbcProfileParser.cs index 2f302d2edd677..f0d7c5afc4100 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/IBC/MIbcProfileParser.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/IBC/MIbcProfileParser.cs @@ -22,7 +22,7 @@ namespace ILCompiler.IBC { static class MIbcProfileParser { - private class MetadataLoaderForPgoData : IPgoSchemaDataLoader + private class MetadataLoaderForPgoData : IPgoSchemaDataLoader { private readonly EcmaMethodIL _ilBody; @@ -30,12 +30,12 @@ public MetadataLoaderForPgoData(EcmaMethodIL ilBody) { _ilBody = ilBody; } - TypeSystemEntityOrUnknown IPgoSchemaDataLoader.TypeFromLong(long token) + TypeSystemEntityOrUnknown IPgoSchemaDataLoader.TypeFromLong(long token) { try { if (token == 0) - return new TypeSystemEntityOrUnknown(null); + return new TypeSystemEntityOrUnknown((TypeDesc)null); if ((token & 0xFF000000) == 0) { // token type is 0, therefore it can't be a type @@ -53,6 +53,29 @@ TypeSystemEntityOrUnknown IPgoSchemaDataLoader.TypeFr return new TypeSystemEntityOrUnknown((int)token); } } + TypeSystemEntityOrUnknown IPgoSchemaDataLoader.MethodFromLong(long token) + { + try + { + if (token == 0) + return new TypeSystemEntityOrUnknown((MethodDesc)null); + if ((token & 0xFF000000) == 0) + { + // token type is 0, therefore it can't be a method + return new TypeSystemEntityOrUnknown((int)token); + } + MethodDesc foundMethod = _ilBody.GetObject((int)token, NotFoundBehavior.ReturnNull) as MethodDesc; + if (foundMethod == null) + { + return new TypeSystemEntityOrUnknown((int)token & 0x00FFFFFF); + } + return new TypeSystemEntityOrUnknown(foundMethod); + } + catch + { + return new TypeSystemEntityOrUnknown((int)token); + } + } } public static PEReader OpenMibcAsPEReader(string filename) @@ -413,7 +436,7 @@ static IEnumerable ReadMIbcGroup(TypeSystemContext tsc, EcmaM { instrumentationDataLongs.Add(2); // MarshalMask 2 (Type) instrumentationDataLongs.Add(0); // PgoInstrumentationKind.Done (0) - pgoSchemaData = PgoProcessor.ParsePgoData(metadataLoader, instrumentationDataLongs, false).ToArray(); + pgoSchemaData = PgoProcessor.ParsePgoData(metadataLoader, instrumentationDataLongs, false).ToArray(); } state = MibcGroupParseState.LookingForOptionalData; break; diff --git a/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/PgoInfo.cs b/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/PgoInfo.cs index d07b5496a970a..aff9b42f027e7 100644 --- a/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/PgoInfo.cs +++ b/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/PgoInfo.cs @@ -31,7 +31,7 @@ public PgoInfo(PgoInfoKey key, ReadyToRunReader r2rReader, int pgoFormatVersion, PgoSchemaElem[] _pgoData; int _size; - class PgoDataLoader : IPgoSchemaDataLoader + class PgoDataLoader : IPgoSchemaDataLoader { ReadyToRunReader _r2rReader; SignatureFormattingOptions _formatOptions; @@ -42,7 +42,7 @@ public PgoDataLoader(ReadyToRunReader r2rReader, SignatureFormattingOptions form _r2rReader = r2rReader; } - string IPgoSchemaDataLoader.TypeFromLong(long input) + string IPgoSchemaDataLoader.TypeFromLong(long input) { int tableIndex = checked((int)(input & 0xF)); int fixupIndex = checked((int)(input >> 4)); @@ -55,6 +55,20 @@ string IPgoSchemaDataLoader.TypeFromLong(long input) return _r2rReader.ImportSections[tableIndex].Entries[fixupIndex].Signature.ToString(_formatOptions); } } + + string IPgoSchemaDataLoader.MethodFromLong(long input) + { + int tableIndex = checked((int)(input & 0xF)); + int fixupIndex = checked((int)(input >> 4)); + if (tableIndex == 0xF) + { + return $"Unknown method {fixupIndex}"; + } + else + { + return _r2rReader.ImportSections[tableIndex].Entries[fixupIndex].Signature.ToString(_formatOptions); + } + } } void EnsurePgoData() @@ -72,7 +86,7 @@ void EnsurePgoData() SignatureFormattingOptions formattingOptions = new SignatureFormattingOptions(); - _pgoData = PgoProcessor.ParsePgoData(new PgoDataLoader(_r2rReader, formattingOptions), compressedIntParser, true).ToArray(); + _pgoData = PgoProcessor.ParsePgoData(new PgoDataLoader(_r2rReader, formattingOptions), compressedIntParser, true).ToArray(); _size = compressedIntParser.Offset - Offset; } } diff --git a/src/coreclr/tools/dotnet-pgo/MibcEmitter.cs b/src/coreclr/tools/dotnet-pgo/MibcEmitter.cs index 997f6110c96be..a498bfd9e21d5 100644 --- a/src/coreclr/tools/dotnet-pgo/MibcEmitter.cs +++ b/src/coreclr/tools/dotnet-pgo/MibcEmitter.cs @@ -32,7 +32,7 @@ namespace Microsoft.Diagnostics.Tools.Pgo { static class MibcEmitter { - class MIbcGroup : IPgoEncodedValueEmitter + class MIbcGroup : IPgoEncodedValueEmitter { private static int s_emitCount = 0; @@ -100,7 +100,7 @@ public void AddProcessedMethodData(MethodProfileData processedMethodData) if (processedMethodData.SchemaData != null) { _il.LoadString(_emitter.GetUserStringHandle("InstrumentationDataStart")); - PgoProcessor.EncodePgoData(processedMethodData.SchemaData, this, true); + PgoProcessor.EncodePgoData(processedMethodData.SchemaData, this, true); } _il.OpCode(ILOpCode.Pop); } @@ -121,13 +121,13 @@ public MethodDefinitionHandle EmitMethod() return _emitter.AddGlobalMethod(methodName, _il, 8); } - bool IPgoEncodedValueEmitter.EmitDone() + bool IPgoEncodedValueEmitter.EmitDone() { _il.LoadString(_emitter.GetUserStringHandle("InstrumentationDataEnd")); return true; } - void IPgoEncodedValueEmitter.EmitLong(long value, long previousValue) + void IPgoEncodedValueEmitter.EmitLong(long value, long previousValue) { if ((value <= int.MaxValue) && (value >= int.MinValue)) { @@ -139,7 +139,7 @@ void IPgoEncodedValueEmitter.EmitLong(long value, lon } } - void IPgoEncodedValueEmitter.EmitType(TypeSystemEntityOrUnknown type, TypeSystemEntityOrUnknown previousValue) + void IPgoEncodedValueEmitter.EmitType(TypeSystemEntityOrUnknown type, TypeSystemEntityOrUnknown previousValue) { if (type.AsType != null) { @@ -153,6 +153,20 @@ void IPgoEncodedValueEmitter.EmitType(TypeSystemEntit } } + void IPgoEncodedValueEmitter.EmitMethod(TypeSystemEntityOrUnknown method, TypeSystemEntityOrUnknown previousValue) + { + if (method.AsMethod != null) + { + _il.OpCode(ILOpCode.Ldtoken); + _il.Token(_emitter.GetMethodRef(method.AsMethod)); + } + else + { + + _il.LoadConstantI4(method.AsUnknown & 0x00FFFFFF); + } + } + } private static string GetTypeDefiningAssembly(TypeDesc type) diff --git a/src/coreclr/tools/dotnet-pgo/Program.cs b/src/coreclr/tools/dotnet-pgo/Program.cs index c210adca92148..a9bfc1db4664a 100644 --- a/src/coreclr/tools/dotnet-pgo/Program.cs +++ b/src/coreclr/tools/dotnet-pgo/Program.cs @@ -55,7 +55,7 @@ class MethodChunks public int LastChunk = -1; } - class PgoDataLoader : IPgoSchemaDataLoader + class PgoDataLoader : IPgoSchemaDataLoader { private TraceRuntimeDescToTypeSystemDesc _idParser; @@ -84,6 +84,27 @@ public TypeSystemEntityOrUnknown TypeFromLong(long input) // Unknown type, apply unique value, but keep the upper byte zeroed so that it can be distinguished from a token return new TypeSystemEntityOrUnknown(System.HashCode.Combine(input) & 0x7FFFFF | 0x800000); } + + public TypeSystemEntityOrUnknown MethodFromLong(long input) + { + if (input == 0) + return new TypeSystemEntityOrUnknown(0); + + MethodDesc method = null; + + try + { + method = _idParser.ResolveMethodID(input, false); + } + catch + { } + if (method != null) + { + return new TypeSystemEntityOrUnknown(method); + } + // Unknown type, apply unique value, but keep the upper byte zeroed so that it can be distinguished from a token + return new TypeSystemEntityOrUnknown(System.HashCode.Combine(input) & 0x7FFFFF | 0x800000); + } } struct ProcessedMethodData @@ -576,7 +597,7 @@ void PrintBar(string label, ref int curIndex, Func include, bool f List typeHandleHistogramCallSites = prof1.SchemaData.Concat(prof2.SchemaData) - .Where(e => e.InstrumentationKind == PgoInstrumentationKind.GetLikelyClass || e.InstrumentationKind == PgoInstrumentationKind.TypeHandleHistogramTypeHandle) + .Where(e => e.InstrumentationKind == PgoInstrumentationKind.GetLikelyClass || e.InstrumentationKind == PgoInstrumentationKind.HandleHistogramTypes) .Select(e => e.ILOffset) .Distinct() .ToList(); @@ -781,8 +802,8 @@ static void PrintMibcStats(ProfileData data) PrintOutput($"# Methods with 64-bit block counts: {profiledMethods.Count(spd => spd.SchemaData.Any(elem => elem.InstrumentationKind == PgoInstrumentationKind.BasicBlockLongCount))}"); PrintOutput($"# Methods with 32-bit edge counts: {profiledMethods.Count(spd => spd.SchemaData.Any(elem => elem.InstrumentationKind == PgoInstrumentationKind.EdgeIntCount))}"); PrintOutput($"# Methods with 64-bit edge counts: {profiledMethods.Count(spd => spd.SchemaData.Any(elem => elem.InstrumentationKind == PgoInstrumentationKind.EdgeLongCount))}"); - int numTypeHandleHistograms = profiledMethods.Sum(spd => spd.SchemaData.Count(elem => elem.InstrumentationKind == PgoInstrumentationKind.TypeHandleHistogramTypeHandle)); - int methodsWithTypeHandleHistograms = profiledMethods.Count(spd => spd.SchemaData.Any(elem => elem.InstrumentationKind == PgoInstrumentationKind.TypeHandleHistogramTypeHandle)); + int numTypeHandleHistograms = profiledMethods.Sum(spd => spd.SchemaData.Count(elem => elem.InstrumentationKind == PgoInstrumentationKind.HandleHistogramTypes)); + int methodsWithTypeHandleHistograms = profiledMethods.Count(spd => spd.SchemaData.Any(elem => elem.InstrumentationKind == PgoInstrumentationKind.HandleHistogramTypes)); PrintOutput($"# Type handle histograms: {numTypeHandleHistograms} in {methodsWithTypeHandleHistograms} methods"); int numGetLikelyClass = profiledMethods.Sum(spd => spd.SchemaData.Count(elem => elem.InstrumentationKind == PgoInstrumentationKind.GetLikelyClass)); int methodsWithGetLikelyClass = profiledMethods.Count(spd => spd.SchemaData.Any(elem => elem.InstrumentationKind == PgoInstrumentationKind.GetLikelyClass)); @@ -793,7 +814,7 @@ static void PrintMibcStats(ProfileData data) { var sites = mpd.SchemaData - .Where(e => e.InstrumentationKind == PgoInstrumentationKind.TypeHandleHistogramTypeHandle || e.InstrumentationKind == PgoInstrumentationKind.GetLikelyClass) + .Where(e => e.InstrumentationKind == PgoInstrumentationKind.HandleHistogramTypes || e.InstrumentationKind == PgoInstrumentationKind.GetLikelyClass) .Select(e => e.ILOffset) .Distinct(); @@ -826,7 +847,7 @@ static bool CountersSumToZero(MethodProfileData data) PrintCallsitesByLikelyClassesChart(profiledMethods .SelectMany(m => m.SchemaData) - .Where(sd => sd.InstrumentationKind == PgoInstrumentationKind.TypeHandleHistogramTypeHandle) + .Where(sd => sd.InstrumentationKind == PgoInstrumentationKind.HandleHistogramTypes) .Select(GetUniqueClassesSeen) .ToArray()); @@ -841,7 +862,7 @@ static int GetUniqueClassesSeen(PgoSchemaElem se) PrintLikelihoodHistogram(profiledMethods .SelectMany(m => m.SchemaData) - .Where(sd => sd.InstrumentationKind == PgoInstrumentationKind.TypeHandleHistogramTypeHandle) + .Where(sd => sd.InstrumentationKind == PgoInstrumentationKind.HandleHistogramTypes) .Select(GetLikelihoodOfMostPopularType) .ToArray()); @@ -890,10 +911,10 @@ static bool IsUnknownTypeHandle(int handle) } bool isHistogramCount = - elem.InstrumentationKind == PgoInstrumentationKind.TypeHandleHistogramIntCount || - elem.InstrumentationKind == PgoInstrumentationKind.TypeHandleHistogramLongCount; + elem.InstrumentationKind == PgoInstrumentationKind.HandleHistogramIntCount || + elem.InstrumentationKind == PgoInstrumentationKind.HandleHistogramLongCount; - if (isHistogramCount && elem.Count == 1 && i + 1 < schema.Length && schema[i + 1].InstrumentationKind == PgoInstrumentationKind.TypeHandleHistogramTypeHandle) + if (isHistogramCount && elem.Count == 1 && i + 1 < schema.Length && schema[i + 1].InstrumentationKind == PgoInstrumentationKind.HandleHistogramTypes) { var handles = (TypeSystemEntityOrUnknown[])schema[i + 1].DataObject; var histogram = handles.Where(e => !e.IsNull).GroupBy(e => e).ToList(); @@ -1572,7 +1593,7 @@ void AddToInstrumentationData(int eventClrInstanceId, long methodID, int methodF } var intDecompressor = new PgoProcessor.PgoEncodedCompressedIntParser(instrumentationData, 0); - methodData.InstrumentationData = PgoProcessor.ParsePgoData(pgoDataLoader, intDecompressor, true).ToArray(); + methodData.InstrumentationData = PgoProcessor.ParsePgoData(pgoDataLoader, intDecompressor, true).ToArray(); } else { diff --git a/src/coreclr/tools/superpmi/mcs/verbdumpmap.cpp b/src/coreclr/tools/superpmi/mcs/verbdumpmap.cpp index c612b13f80de6..1cfbc598dbee1 100644 --- a/src/coreclr/tools/superpmi/mcs/verbdumpmap.cpp +++ b/src/coreclr/tools/superpmi/mcs/verbdumpmap.cpp @@ -91,9 +91,10 @@ void DumpMap(int index, MethodContext* mc) // Add in the "fake" pgo flags bool hasEdgeProfile = false; bool hasClassProfile = false; + bool hasMethodProfile = false; bool hasLikelyClass = false; ICorJitInfo::PgoSource pgoSource = ICorJitInfo::PgoSource::Unknown; - if (mc->hasPgoData(hasEdgeProfile, hasClassProfile, hasLikelyClass, pgoSource)) + if (mc->hasPgoData(hasEdgeProfile, hasClassProfile, hasMethodProfile, hasLikelyClass, pgoSource)) { rawFlags |= 1ULL << (EXTRA_JIT_FLAGS::HAS_PGO); @@ -107,6 +108,11 @@ void DumpMap(int index, MethodContext* mc) rawFlags |= 1ULL << (EXTRA_JIT_FLAGS::HAS_CLASS_PROFILE); } + if (hasMethodProfile) + { + rawFlags |= 1ULL << (EXTRA_JIT_FLAGS::HAS_METHOD_PROFILE); + } + if (hasLikelyClass) { rawFlags |= 1ULL << (EXTRA_JIT_FLAGS::HAS_LIKELY_CLASS); diff --git a/src/coreclr/tools/superpmi/mcs/verbjitflags.cpp b/src/coreclr/tools/superpmi/mcs/verbjitflags.cpp index eaf208508b7dd..a3e31a1f73062 100644 --- a/src/coreclr/tools/superpmi/mcs/verbjitflags.cpp +++ b/src/coreclr/tools/superpmi/mcs/verbjitflags.cpp @@ -29,9 +29,10 @@ int verbJitFlags::DoWork(const char* nameOfInput) // bool hasEdgeProfile = false; bool hasClassProfile = false; + bool hasMethodProfile = false; bool hasLikelyClass = false; ICorJitInfo::PgoSource pgoSource = ICorJitInfo::PgoSource::Unknown; - if (mc->hasPgoData(hasEdgeProfile, hasClassProfile, hasLikelyClass, pgoSource)) + if (mc->hasPgoData(hasEdgeProfile, hasClassProfile, hasMethodProfile, hasLikelyClass, pgoSource)) { rawFlags |= 1ULL << (EXTRA_JIT_FLAGS::HAS_PGO); @@ -45,6 +46,11 @@ int verbJitFlags::DoWork(const char* nameOfInput) rawFlags |= 1ULL << (EXTRA_JIT_FLAGS::HAS_CLASS_PROFILE); } + if (hasMethodProfile) + { + rawFlags |= 1ULL << (EXTRA_JIT_FLAGS::HAS_METHOD_PROFILE); + } + if (hasLikelyClass) { rawFlags |= 1ULL << (EXTRA_JIT_FLAGS::HAS_LIKELY_CLASS); diff --git a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp index 7cca8b9adc482..a684eb5d40571 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp @@ -5587,13 +5587,14 @@ void MethodContext::dmpGetPgoInstrumentationResults(DWORDLONG key, const Agnosti case ICorJitInfo::PgoInstrumentationKind::EdgeLongCount: printf("E %llu", *(uint64_t*)(pInstrumentationData + pBuf[i].Offset)); break; - case ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramIntCount: + case ICorJitInfo::PgoInstrumentationKind::HandleHistogramIntCount: printf("T %u", *(unsigned*)(pInstrumentationData + pBuf[i].Offset)); break; - case ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramLongCount: + case ICorJitInfo::PgoInstrumentationKind::HandleHistogramLongCount: printf("T %llu", *(uint64_t*)(pInstrumentationData + pBuf[i].Offset)); break; - case ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramTypeHandle: + case ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes: + case ICorJitInfo::PgoInstrumentationKind::HandleHistogramMethods: for (unsigned int j = 0; j < pBuf[i].Count; j++) { printf("[%u] %016llX ", j, CastHandle(*(uintptr_t*)(pInstrumentationData + pBuf[i].Offset + j * sizeof(uintptr_t)))); @@ -7065,10 +7066,11 @@ int MethodContext::dumpMD5HashToBuffer(BYTE* pBuffer, int bufLen, char* hash, in return m_hash.HashBuffer(pBuffer, bufLen, hash, hashLen); } -bool MethodContext::hasPgoData(bool& hasEdgeProfile, bool& hasClassProfile, bool& hasLikelyClass, ICorJitInfo::PgoSource& pgoSource) +bool MethodContext::hasPgoData(bool& hasEdgeProfile, bool& hasClassProfile, bool& hasMethodProfile, bool& hasLikelyClass, ICorJitInfo::PgoSource& pgoSource) { hasEdgeProfile = false; hasClassProfile = false; + hasMethodProfile = false; hasLikelyClass = false; // Obtain the Method Info structure for this method @@ -7091,8 +7093,8 @@ bool MethodContext::hasPgoData(bool& hasEdgeProfile, bool& hasClassProfile, bool { hasEdgeProfile |= (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::EdgeIntCount); hasEdgeProfile |= (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::EdgeLongCount); - hasClassProfile |= (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramIntCount); - hasClassProfile |= (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramLongCount); + hasClassProfile |= (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes); + hasMethodProfile |= (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramMethods); hasLikelyClass |= (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::GetLikelyClass); if (hasEdgeProfile && hasClassProfile && hasLikelyClass) diff --git a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h index 589d6d85b2c65..81c098d292483 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h +++ b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h @@ -52,7 +52,8 @@ enum EXTRA_JIT_FLAGS HAS_CLASS_PROFILE = 61, HAS_LIKELY_CLASS = 60, HAS_STATIC_PROFILE = 59, - HAS_DYNAMIC_PROFILE = 58 + HAS_DYNAMIC_PROFILE = 58, + HAS_METHOD_PROFILE = 57, }; // Asserts to catch changes in corjit flags definitions. @@ -105,7 +106,7 @@ class MethodContext int dumpMethodIdentityInfoToBuffer(char* buff, int len, bool ignoreMethodName = false, CORINFO_METHOD_INFO* optInfo = nullptr, unsigned optFlags = 0); int dumpMethodMD5HashToBuffer(char* buff, int len, bool ignoreMethodName = false, CORINFO_METHOD_INFO* optInfo = nullptr, unsigned optFlags = 0); - bool hasPgoData(bool& hasEdgeProfile, bool& hasClassProfile, bool& hasLikelyClass, ICorJitInfo::PgoSource& pgoSource); + bool hasPgoData(bool& hasEdgeProfile, bool& hasClassProfile, bool& hasMethodProfile, bool& hasLikelyClass, ICorJitInfo::PgoSource& pgoSource); void recGlobalContext(const MethodContext& other); diff --git a/src/coreclr/tools/superpmi/superpmi-shared/spmidumphelper.cpp b/src/coreclr/tools/superpmi/superpmi-shared/spmidumphelper.cpp index ad4aee9fcd945..1d1d4d53b1a84 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/spmidumphelper.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shared/spmidumphelper.cpp @@ -283,6 +283,7 @@ std::string SpmiDumpHelper::DumpJitFlags(unsigned long long flags) AddFlagNumeric(HAS_PGO, EXTRA_JIT_FLAGS::HAS_PGO); AddFlagNumeric(HAS_EDGE_PROFILE, EXTRA_JIT_FLAGS::HAS_EDGE_PROFILE); AddFlagNumeric(HAS_CLASS_PROFILE, EXTRA_JIT_FLAGS::HAS_CLASS_PROFILE); + AddFlagNumeric(HAS_METHOD_PROFILE, EXTRA_JIT_FLAGS::HAS_METHOD_PROFILE); AddFlagNumeric(HAS_LIKELY_CLASS, EXTRA_JIT_FLAGS::HAS_LIKELY_CLASS); AddFlagNumeric(HAS_STATIC_PROFILE, EXTRA_JIT_FLAGS::HAS_STATIC_PROFILE); AddFlagNumeric(HAS_DYNAMIC_PROFILE, EXTRA_JIT_FLAGS::HAS_DYNAMIC_PROFILE); diff --git a/src/coreclr/vm/pgo.cpp b/src/coreclr/vm/pgo.cpp index c3f37a3226b98..d28e143cfe869 100644 --- a/src/coreclr/vm/pgo.cpp +++ b/src/coreclr/vm/pgo.cpp @@ -254,6 +254,34 @@ void PgoManager::WritePgoData() } break; } + case ICorJitInfo::PgoInstrumentationKind::MethodHandle: + { + MethodDesc* md = *(MethodDesc**)(data + entryOffset); + if (md == nullptr) + { + fprintf(pgoDataFile, "MethodHandle: NULL"); + } + else + { + SString garbage1, tMethodName, garbage2; + md->GetMethodInfo(garbage1, tMethodName, garbage2); + StackSString tTypeName; + TypeString::AppendType(tTypeName, TypeHandle(md->GetMethodTable()), TypeString::FormatNamespace | TypeString::FormatFullInst | TypeString::FormatAssembly); + // Format is: + // MethodName|@|fully_qualified_type_name + if (tTypeName.GetCount() + 1 + tMethodName.GetCount() > 8192) + { + fprintf(pgoDataFile, "MethodHandle: unknown"); + } + else + { + StackScratchBuffer methodNameBuffer; + StackScratchBuffer typeBuffer; + fprintf(pgoDataFile, "MethodHandle: %s|@|%s", tMethodName.GetUTF8(methodNameBuffer), tTypeName.GetUTF8(typeBuffer)); + } + } + break; + } default: break; } @@ -469,6 +497,41 @@ void PgoManager::ReadPgoData() ptrVal += 1; // Set low bit to indicate that this isn't actually a TypeHandle, but is instead a pointer } + uint8_t *rawBuffer = methodInstrumentationData.OpenRawBuffer(maxSize); + *(INT_PTR *)(rawBuffer + entryOffset) = ptrVal; + methodInstrumentationData.CloseRawBuffer(); + break; + } + case ICorJitInfo::PgoInstrumentationKind::MethodHandle: + { + char* methodString; + if (strncmp(buffer, "MethodHandle: ", 14) != 0) + { + failed = true; + break; + } + methodString = buffer + 14; + size_t endOfString = strlen(methodString); + if (endOfString == 0 || (methodString[endOfString - 1] != '\n')) + { + failed = true; + break; + } + // Remove \n and replace will null + methodString[endOfString - 1] = '\0'; + + TypeHandle th; + INT_PTR ptrVal = 0; + if (strcmp(methodString, "NULL") != 0) + { + // As early type loading is likely problematic, simply drop the string into the data, and fix it up later + void* tempString = malloc(endOfString); + memcpy(tempString, methodString, endOfString); + + ptrVal = (INT_PTR)tempString; + ptrVal += 1; // Set low bit to indicate that this isn't actually a TypeHandle, but is instead a pointer + } + uint8_t *rawBuffer = methodInstrumentationData.OpenRawBuffer(maxSize); *(INT_PTR *)(rawBuffer + entryOffset) = ptrVal; methodInstrumentationData.CloseRawBuffer(); @@ -747,34 +810,53 @@ HRESULT PgoManager::getPgoInstrumentationResults(MethodDesc* pMD, BYTE** pAlloca for (unsigned iSchema = 0; iSchema < schemaArray.GetCount(); iSchema++) { ICorJitInfo::PgoInstrumentationSchema *schema = &(schemaArray)[iSchema]; - if ((schema->InstrumentationKind & ICorJitInfo::PgoInstrumentationKind::MarshalMask) == ICorJitInfo::PgoInstrumentationKind::TypeHandle) + ICorJitInfo::PgoInstrumentationKind kind = schema->InstrumentationKind & ICorJitInfo::PgoInstrumentationKind::MarshalMask; + if ((kind == ICorJitInfo::PgoInstrumentationKind::TypeHandle) || (kind == ICorJitInfo::PgoInstrumentationKind::MethodHandle)) { for (int iEntry = 0; iEntry < schema->Count; iEntry++) { - INT_PTR* typeHandleValueAddress = (INT_PTR*)(found->GetData() + schema->Offset + iEntry * InstrumentationKindToSize(schema->InstrumentationKind)); - INT_PTR initialTypeHandleValue = VolatileLoad(typeHandleValueAddress); - if (((initialTypeHandleValue & 1) == 1) && !ICorJitInfo::IsUnknownTypeHandle(initialTypeHandleValue)) + INT_PTR* handleValueAddress = (INT_PTR*)(found->GetData() + schema->Offset + iEntry * InstrumentationKindToSize(schema->InstrumentationKind)); + INT_PTR initialHandleValue = VolatileLoad(handleValueAddress); + if (((initialHandleValue & 1) == 1) && !ICorJitInfo::IsUnknownHandle(initialHandleValue)) { INT_PTR newPtr = 0; - TypeHandle th; - char* typeString = ((char *)initialTypeHandleValue) - 1; + char* string = ((char *)initialHandleValue) - 1; - // Don't attempt to load any types until the EE is started + // Don't attempt to load any types or methods until the EE is started if (g_fEEStarted) { - StackSString ss(SString::Utf8, typeString); - th = TypeName::GetTypeManaged(ss.GetUnicode(), NULL, FALSE, FALSE, FALSE, NULL, NULL); + if (kind == ICorJitInfo::PgoInstrumentationKind::TypeHandle) + { + StackSString ts(SString::Utf8, string); + TypeHandle th = TypeName::GetTypeManaged(ts.GetUnicode(), NULL, FALSE, FALSE, FALSE, NULL, NULL); + newPtr = (INT_PTR)th.AsPtr(); + } + else + { + assert(kind == ICorJitInfo::PgoInstrumentationKind::MethodHandle); + // Format is: + // MethodName|@|fully_qualified_type_name + char* sep = strstr(string, "|@|"); + if (sep != nullptr) + { + StackSString typeString(SString::Utf8, sep + 3); + StackSString methodString(SString::Utf8, string, (COUNT_T)(sep - string)); + TypeHandle th = TypeName::GetTypeManaged(typeString.GetUnicode(), NULL, FALSE, FALSE, FALSE, NULL, NULL); + if (!th.IsNull()) + { + MethodDesc* pMD = MemberLoader::FindMethodByName(th.GetMethodTable(), methodString.GetUTF8NoConvert()); + newPtr = (INT_PTR)pMD; + } + } + } } - if (th.IsNull()) + if (newPtr == 0) { - newPtr = HashToPgoUnknownTypeHandle(HashStringA(typeString)); + newPtr = HashToPgoUnknownHandle(HashStringA(string)); } - else - { - newPtr = (INT_PTR)th.AsPtr(); - } - InterlockedCompareExchangeT(typeHandleValueAddress, newPtr, initialTypeHandleValue); + + InterlockedCompareExchangeT(handleValueAddress, newPtr, initialHandleValue); } } } @@ -853,63 +935,77 @@ class R2RInstrumentationDataReader schemaArray[schemaArray.GetCount() - 1].Offset = instrumentationData.GetCount(); } - if ((schema.InstrumentationKind & ICorJitInfo::PgoInstrumentationKind::MarshalMask) == ICorJitInfo::PgoInstrumentationKind::TypeHandle) + ICorJitInfo::PgoInstrumentationKind kind = schema.InstrumentationKind & ICorJitInfo::PgoInstrumentationKind::MarshalMask; + switch (kind) { - intptr_t typeHandleData = 0; - if (dataItem != 0) + case ICorJitInfo::PgoInstrumentationKind::TypeHandle: + case ICorJitInfo::PgoInstrumentationKind::MethodHandle: { - uint32_t importSection = dataItem & 0xF; - int64_t typeIndex = dataItem >> 4; - if (importSection != 0xF) + intptr_t handleData = 0; + if (dataItem != 0) { - COUNT_T countImportSections; - PTR_CORCOMPILE_IMPORT_SECTION pImportSections = m_pReadyToRunInfo->GetImportSections(&countImportSections); - - if (importSection >= countImportSections) + uint32_t importSection = dataItem & 0xF; + int64_t typeIndex = dataItem >> 4; + if (importSection != 0xF) { - _ASSERTE(!"Malformed pgo type handle data"); - return false; - } + COUNT_T countImportSections; + PTR_CORCOMPILE_IMPORT_SECTION pImportSections = m_pReadyToRunInfo->GetImportSections(&countImportSections); - PTR_CORCOMPILE_IMPORT_SECTION pImportSection = &pImportSections[importSection]; - COUNT_T cbData; - TADDR pData = m_pNativeImage->GetDirectoryData(&pImportSection->Section, &cbData); - uint32_t fixupIndex = (uint32_t)typeIndex; - PTR_SIZE_T fixupAddress = dac_cast(pData + fixupIndex * sizeof(TADDR)); - if (!m_pModule->FixupNativeEntry(pImportSections + importSection, fixupIndex, fixupAddress)) + if (importSection >= countImportSections) + { + _ASSERTE(!"Malformed PGO type or method handle data"); + return false; + } + + PTR_CORCOMPILE_IMPORT_SECTION pImportSection = &pImportSections[importSection]; + COUNT_T cbData; + TADDR pData = m_pNativeImage->GetDirectoryData(&pImportSection->Section, &cbData); + uint32_t fixupIndex = (uint32_t)typeIndex; + PTR_SIZE_T fixupAddress = dac_cast(pData + fixupIndex * sizeof(TADDR)); + if (!m_pModule->FixupNativeEntry(pImportSections + importSection, fixupIndex, fixupAddress)) + { + return false; + } + + handleData = *(intptr_t*)fixupAddress; + } + else { - return false; + handleData = HashToPgoUnknownHandle((uint32_t)typeIndex); } - - typeHandleData = *(intptr_t*)fixupAddress; } - else + + BYTE* pHandleData = (BYTE*)&handleData; + for (size_t i = 0; i < sizeof(intptr_t); i++) { - typeHandleData = HashToPgoUnknownTypeHandle((uint32_t)typeIndex); + instrumentationData.Append(pHandleData[i]); } - } - BYTE* pTypeHandleData = (BYTE*)&typeHandleData; - for (size_t i = 0; i < sizeof(intptr_t); i++) - { - instrumentationData.Append(pTypeHandleData[i]); + break; } - } - else if ((schema.InstrumentationKind & ICorJitInfo::PgoInstrumentationKind::MarshalMask) == ICorJitInfo::PgoInstrumentationKind::FourByte) - { - BYTE* pFourByteData = (BYTE*)&dataItem; - for (int i = 0; i < 4; i++) + case ICorJitInfo::PgoInstrumentationKind::FourByte: { - instrumentationData.Append(pFourByteData[i]); + BYTE* pFourByteData = (BYTE*)&dataItem; + for (int i = 0; i < 4; i++) + { + instrumentationData.Append(pFourByteData[i]); + } + + break; } - } - else if ((schema.InstrumentationKind & ICorJitInfo::PgoInstrumentationKind::MarshalMask) == ICorJitInfo::PgoInstrumentationKind::EightByte) - { - BYTE* pEightByteData = (BYTE*)&dataItem; - for (int i = 0; i < 8; i++) + case ICorJitInfo::PgoInstrumentationKind::EightByte: { - instrumentationData.Append(pEightByteData[i]); + BYTE* pEightByteData = (BYTE*)&dataItem; + for (int i = 0; i < 8; i++) + { + instrumentationData.Append(pEightByteData[i]); + } + + break; } + default: + assert(!"Unexpected PGO instrumentation data type"); + break; } return true;