Initial version of class profiling for PGO

eng/pipelines/common/templates/runtimes/run-test-job.yml

@@ -472,6 +472,7 @@ jobs:
           - jitobjectstackallocation
           - jitpgo
           - jitpgo_inline
+          - jitpgo_classes
         ${{ if in(parameters.testGroup, 'ilasm') }}:
           scenarios:
           - ilasmroundtrip

src/coreclr/src/ToolBox/superpmi/superpmi-shared/icorjitinfoimpl.h

@@ -979,6 +979,34 @@ struct BlockCounts  // Also defined here: code:CORBBTPROF_BLOCK_DATA
     UINT32 ILOffset;
     UINT32 ExecutionCount;
 };
+
+// Data structure for a single class probe.
+//
+// ILOffset is the IL offset in the method for the call site being probed.
+// Currently it must be ORed with CLASS_FLAG and (for interface calls)
+// INTERFACE_FLAG.
+//
+// Count is the number of times a call was made at that call site.
+//
+// SIZE is the number of entries in the table.
+//
+// SAMPLE_INTERVAL must be >= SIZE. SAMPLE_INTERVAL / SIZE
+// gives the average number of calls between table updates.
+// 
+struct ClassProfile
+{
+    enum { 
+        SIZE = 8, 
+        SAMPLE_INTERVAL = 32, 
+        CLASS_FLAG     = 0x80000000, 
+        INTERFACE_FLAG = 0x40000000,
+        OFFSET_MASK    = 0x3FFFFFFF
+    };
+
+    UINT32 ILOffset;
+    UINT32 Count;
+    CORINFO_CLASS_HANDLE ClassTable[SIZE];
+};
 */
 
 // allocate a basic block profile buffer where execution counts will be stored
@@ -993,6 +1021,15 @@ HRESULT getMethodBlockCounts(CORINFO_METHOD_HANDLE ftnHnd,
                              BlockCounts**     pBlockCounts,
                              UINT32 *          pNumRuns);
 
+// Get the likely implementing class for a virtual call or interface call made by ftnHnd
+// at the indicated IL offset. baseHnd is the interface class or base class for the method
+// being called. 
+CORINFO_CLASS_HANDLE getLikelyClass(CORINFO_METHOD_HANDLE ftnHnd,
+                                    CORINFO_CLASS_HANDLE  baseHnd,
+                                    UINT32                ilOffset,
+                                    UINT32*               pLikelihood,
+                                    UINT32*               pNumberOfCases);
+
 // Associates a native call site, identified by its offset in the native code stream, with
 // the signature information and method handle the JIT used to lay out the call site. If
 // the call site has no signature information (e.g. a helper call) or has no method handle

src/coreclr/src/ToolBox/superpmi/superpmi-shared/lwmlist.h

@@ -100,6 +100,7 @@ LWM(GetJitFlags, DWORD, DD)
 LWM(GetJitTimeLogFilename, DWORD, DWORD)
 LWM(GetJustMyCodeHandle, DWORDLONG, DLDL)
 LWM(GetLazyStringLiteralHelper, DWORDLONG, DWORD)
+LWM(GetLikelyClass, Agnostic_GetLikelyClass, Agnostic_GetLikelyClassResult)
 LWM(GetLocationOfThisType, DWORDLONG, Agnostic_CORINFO_LOOKUP_KIND)
 LWM(GetMethodAttribs, DWORDLONG, DWORD)
 LWM(GetMethodClass, DWORDLONG, DWORDLONG)

src/coreclr/src/ToolBox/superpmi/superpmi-shared/methodcontext.cpp

@@ -5265,6 +5265,48 @@ HRESULT MethodContext::repGetMethodBlockCounts(CORINFO_METHOD_HANDLE        ftnH
     return result;
 }
 
+void MethodContext::recGetLikelyClass(CORINFO_METHOD_HANDLE ftnHnd, CORINFO_CLASS_HANDLE baseHnd, UINT32 ilOffset, CORINFO_CLASS_HANDLE result, UINT32* pLikelihood, UINT32* pNumberOfClasses)
+{
+    if (GetLikelyClass == nullptr)
+        GetLikelyClass = new LightWeightMap<Agnostic_GetLikelyClass, Agnostic_GetLikelyClassResult>();
+
+    Agnostic_GetLikelyClass key;
+    ZeroMemory(&key, sizeof(Agnostic_GetLikelyClass));
+
+    key.ftnHnd = (DWORDLONG) ftnHnd;
+    key.baseHnd = (DWORDLONG) baseHnd;
+    key.ilOffset = (DWORD) ilOffset;
+
+    Agnostic_GetLikelyClassResult value;
+    ZeroMemory(&value, sizeof(Agnostic_GetLikelyClassResult));
+    value.classHnd = (DWORDLONG) result;
+    value.likelihood = *pLikelihood;
+    value.numberOfClasses = *pNumberOfClasses;
+
+    GetLikelyClass->Add(key, value);
+    DEBUG_REC(dmpGetLikelyClass(key, value));
+}
+void MethodContext::dmpGetLikelyClass(const Agnostic_GetLikelyClass& key, const Agnostic_GetLikelyClassResult& value)
+{
+    printf("GetLikelyClass key ftn-%016llX base-%016llX il-%u, class-%016llX likelihood-%u numberOfClasses-%u", 
+        key.ftnHnd, key.baseHnd, key.ilOffset, value.classHnd, value.likelihood, value.numberOfClasses);
+}
+CORINFO_CLASS_HANDLE MethodContext::repGetLikelyClass(CORINFO_METHOD_HANDLE ftnHnd, CORINFO_CLASS_HANDLE baseHnd, UINT32 ilOffset, UINT32* pLikelihood, UINT32* pNumberOfClasses)
+{
+    Agnostic_GetLikelyClass key;
+    ZeroMemory(&key, sizeof(Agnostic_GetLikelyClass));
+    key.ftnHnd = (DWORDLONG) ftnHnd;
+    key.baseHnd = (DWORDLONG) baseHnd;
+    key.ilOffset = (DWORD) ilOffset;
+
+    Agnostic_GetLikelyClassResult value = GetLikelyClass->Get(key);
+    DEBUG_REP(dmpGetLikelyClass(key, value));
+
+    *pLikelihood = value.likelihood;
+    *pNumberOfClasses = value.numberOfClasses;
+    return (CORINFO_CLASS_HANDLE) value.classHnd;
+}
+
 void MethodContext::recMergeClasses(CORINFO_CLASS_HANDLE cls1, CORINFO_CLASS_HANDLE cls2, CORINFO_CLASS_HANDLE result)
 {
     if (MergeClasses == nullptr)

src/coreclr/src/ToolBox/superpmi/superpmi-shared/methodcontext.h

@@ -435,6 +435,21 @@ class MethodContext
         DWORD numRuns;
         DWORD result;
     };
+
+    struct Agnostic_GetLikelyClass
+    {
+        DWORDLONG ftnHnd;
+        DWORDLONG baseHnd;
+        DWORD     ilOffset;
+    };
+
+    struct Agnostic_GetLikelyClassResult
+    {
+        DWORDLONG classHnd;
+        DWORD     likelihood;
+        DWORD     numberOfClasses;
+    };
+
     struct Agnostic_GetProfilingHandle
     {
         DWORD     bHookFunction;
@@ -1193,6 +1208,10 @@ class MethodContext
                                     ICorJitInfo::BlockCounts**   pBlockCounts,
                                     UINT32 *                     pNumRuns);
 
+    void recGetLikelyClass(CORINFO_METHOD_HANDLE ftnHnd, CORINFO_CLASS_HANDLE  baseHnd, UINT32 ilOffset, CORINFO_CLASS_HANDLE classHnd, UINT32* pLikelihood, UINT32* pNumberOfClasses);
+    void dmpGetLikelyClass(const Agnostic_GetLikelyClass& key, const Agnostic_GetLikelyClassResult& value);
+    CORINFO_CLASS_HANDLE repGetLikelyClass(CORINFO_METHOD_HANDLE ftnHnd, CORINFO_CLASS_HANDLE  baseHnd, UINT32 ilOffset, UINT32* pLikelihood, UINT32* pNumberOfClasses);
+
     void recMergeClasses(CORINFO_CLASS_HANDLE cls1, CORINFO_CLASS_HANDLE cls2, CORINFO_CLASS_HANDLE result);
     void dmpMergeClasses(DLDL key, DWORDLONG value);
     CORINFO_CLASS_HANDLE repMergeClasses(CORINFO_CLASS_HANDLE cls1, CORINFO_CLASS_HANDLE cls2);
@@ -1359,7 +1378,7 @@ class MethodContext
 };
 
 // ********************* Please keep this up-to-date to ease adding more ***************
-// Highest packet number: 181
+// Highest packet number: 182
 // *************************************************************************************
 enum mcPackets
 {
@@ -1458,6 +1477,7 @@ enum mcPackets
     Packet_GetJitFlags                                   = 154, // Added 2/3/2016
     Packet_GetJitTimeLogFilename                         = 67,
     Packet_GetJustMyCodeHandle                           = 68,
+    Packet_GetLikelyClass                                = 182, // Added 9/27/2020
     Packet_GetLocationOfThisType                         = 69,
     Packet_GetMethodAttribs                              = 70,
     Packet_GetMethodClass                                = 71,

src/coreclr/src/ToolBox/superpmi/superpmi-shim-collector/icorjitinfo.cpp

@@ -2049,6 +2049,21 @@ HRESULT interceptor_ICJI::getMethodBlockCounts(CORINFO_METHOD_HANDLE ftnHnd,
     return temp;
 }
 
+// Get the likely implementing class for a virtual call or interface call made by ftnHnd
+// at the indicated IL offset. baseHnd is the interface class or base class for the method
+// being called. 
+CORINFO_CLASS_HANDLE interceptor_ICJI::getLikelyClass(CORINFO_METHOD_HANDLE ftnHnd,
+                                                      CORINFO_CLASS_HANDLE  baseHnd,
+                                                      UINT32                ilOffset,
+                                                      UINT32*               pLikelihood,
+                                                      UINT32*               pNumberOfClasses)
+{
+    mc->cr->AddCall("getLikelyClass");
+    CORINFO_CLASS_HANDLE result = original_ICorJitInfo->getLikelyClass(ftnHnd, baseHnd, ilOffset, pLikelihood, pNumberOfClasses);
+    mc->recGetLikelyClass(ftnHnd, baseHnd, ilOffset, result, pLikelihood, pNumberOfClasses);
+    return result;
+}
+
 // Associates a native call site, identified by its offset in the native code stream, with
 // the signature information and method handle the JIT used to lay out the call site. If
 // the call site has no signature information (e.g. a helper call) or has no method handle

src/coreclr/src/ToolBox/superpmi/superpmi-shim-counter/icorjitinfo.cpp

@@ -1623,6 +1623,19 @@ HRESULT interceptor_ICJI::getMethodBlockCounts(CORINFO_METHOD_HANDLE ftnHnd,
     return original_ICorJitInfo->getMethodBlockCounts(ftnHnd, pCount, pBlockCounts, pNumRuns);
 }
 
+// Get the likely implementing class for a virtual call or interface call made by ftnHnd
+// at the indicated IL offset. baseHnd is the interface class or base class for the method
+// being called. 
+CORINFO_CLASS_HANDLE interceptor_ICJI::getLikelyClass(CORINFO_METHOD_HANDLE ftnHnd,
+                                                      CORINFO_CLASS_HANDLE  baseHnd,
+                                                      UINT32                ilOffset,
+                                                      UINT32*               pLikelihood,
+                                                      UINT32*               pNumberOfClasses)
+{
+    mcs->AddCall("getLikelyClass");
+    return original_ICorJitInfo->getLikelyClass(ftnHnd, baseHnd, ilOffset, pLikelihood, pNumberOfClasses);
+}
+
 // Associates a native call site, identified by its offset in the native code stream, with
 // the signature information and method handle the JIT used to lay out the call site. If
 // the call site has no signature information (e.g. a helper call) or has no method handle

src/coreclr/src/ToolBox/superpmi/superpmi-shim-simple/icorjitinfo.cpp

@@ -1441,6 +1441,18 @@ HRESULT interceptor_ICJI::getMethodBlockCounts(CORINFO_METHOD_HANDLE ftnHnd,
     return original_ICorJitInfo->getMethodBlockCounts(ftnHnd, pCount, pBlockCounts, pNumRuns);
 }
 
+// Get the likely implementing class for a virtual call or interface call made by ftnHnd
+// at the indicated IL offset. baseHnd is the interface class or base class for the method
+// being called. 
+CORINFO_CLASS_HANDLE interceptor_ICJI::getLikelyClass(CORINFO_METHOD_HANDLE ftnHnd,
+                                                      CORINFO_CLASS_HANDLE  baseHnd,
+                                                      UINT32                ilOffset,
+                                                      UINT32*               pLikelihood,
+                                                      UINT32*               pNumberOfClasses)
+{
+    return original_ICorJitInfo->getLikelyClass(ftnHnd, baseHnd, ilOffset, pLikelihood, pNumberOfClasses);
+}
+
 // Associates a native call site, identified by its offset in the native code stream, with
 // the signature information and method handle the JIT used to lay out the call site. If
 // the call site has no signature information (e.g. a helper call) or has no method handle

src/coreclr/src/ToolBox/superpmi/superpmi/icorjitinfo.cpp

@@ -1805,6 +1805,19 @@ HRESULT MyICJI::getMethodBlockCounts(CORINFO_METHOD_HANDLE ftnHnd,
     return jitInstance->mc->repGetMethodBlockCounts(ftnHnd, pCount, pBlockCounts, pNumRuns);
 }
 
+// Get the likely implementing class for a virtual call or interface call made by ftnHnd
+// at the indicated IL offset. baseHnd is the interface class or base class for the method
+// being called. 
+CORINFO_CLASS_HANDLE MyICJI::getLikelyClass(CORINFO_METHOD_HANDLE ftnHnd,
+                                            CORINFO_CLASS_HANDLE  baseHnd,
+                                            UINT32                ilOffset,
+                                            UINT32*               pLikelihood,
+                                            UINT32*               pNumberOfClasses)
+{
+    jitInstance->mc->cr->AddCall("getLikelyClass");
+    return jitInstance->mc->repGetLikelyClass(ftnHnd, baseHnd, ilOffset, pLikelihood, pNumberOfClasses);
+}
+
 // Associates a native call site, identified by its offset in the native code stream, with
 // the signature information and method handle the JIT used to lay out the call site. If
 // the call site has no signature information (e.g. a helper call) or has no method handle

src/coreclr/src/inc/corinfo.h

@@ -208,11 +208,11 @@ TODO: Talk about initializing strutures before use
 //
 //////////////////////////////////////////////////////////////////////////////////////////////////////////
 
-constexpr GUID JITEEVersionIdentifier = { /* 8031aa05-4568-40fc-a0d2-d971d8edba16 */
-    0x8031aa05,
-    0x4568,
-    0x40fc,
-    {0xa0, 0xd2, 0xd9, 0x71, 0xd8, 0xed, 0xba, 0x16}
+constexpr GUID JITEEVersionIdentifier = { /* 0d235fe4-65a1-487a-8553-c845496da901 */
+    0x0d235fe4,
+    0x65a1,
+    0x487a,
+    {0x85, 0x53, 0xc8, 0x45, 0x49, 0x6d, 0xa9, 0x01}
 };
 
 //////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -623,6 +623,7 @@ enum CorInfoHelpFunc
     CORINFO_HELP_STACK_PROBE,               // Probes each page of the allocated stack frame
 
     CORINFO_HELP_PATCHPOINT,                // Notify runtime that code has reached a patchpoint
+    CORINFO_HELP_CLASSPROFILE,              // Update class profile for a call site
 
     CORINFO_HELP_COUNT,
 };

src/coreclr/src/inc/corjit.h

@@ -251,6 +251,34 @@ class ICorJitInfo : public ICorDynamicInfo
         UINT32 ExecutionCount;
     };
 
+    // Data structure for a single class probe.
+    //
+    // ILOffset is the IL offset in the method for the call site being probed.
+    // Currently it must be ORed with CLASS_FLAG and (for interface calls)
+    // INTERFACE_FLAG.
+    //
+    // Count is the number of times a call was made at that call site.
+    //
+    // SIZE is the number of entries in the table.
+    //
+    // SAMPLE_INTERVAL must be >= SIZE. SAMPLE_INTERVAL / SIZE
+    // gives the average number of calls between table updates.
+    // 
+    struct ClassProfile
+    {
+        enum { 
+            SIZE = 8, 
+            SAMPLE_INTERVAL = 32, 
+            CLASS_FLAG     = 0x80000000, 
+            INTERFACE_FLAG = 0x40000000,
+            OFFSET_MASK    = 0x3FFFFFFF
+        };
+
+        UINT32 ILOffset;
+        UINT32 Count;
+        CORINFO_CLASS_HANDLE ClassTable[SIZE];
+    };
+
     // allocate a basic block profile buffer where execution counts will be stored
     // for jitted basic blocks.
     virtual HRESULT allocMethodBlockCounts (
@@ -267,6 +295,24 @@ class ICorJitInfo : public ICorDynamicInfo
             UINT32 *              pNumRuns         // pointer to the total number of profile scenarios run
             ) = 0;
 
+    // Get the likely implementing class for a virtual call or interface call made by ftnHnd
+    // at the indicated IL offset. baseHnd is the interface class or base class for the method
+    // being called. May returns NULL.
+    // 
+    // pLikelihood is the estimated percent chance that the class at runtime is the class
+    // returned by this method. A well-estimated monomorphic call site will return a likelihood
+    // of 100.
+    // 
+    // pNumberOfClasses is the estimated number of different classes seen at the site.
+    // A well-estimated monomorphic call site will return 1.
+    virtual CORINFO_CLASS_HANDLE getLikelyClass(
+            CORINFO_METHOD_HANDLE ftnHnd,
+            CORINFO_CLASS_HANDLE  baseHnd,
+            UINT32                ilOffset,
+            UINT32 *              pLikelihood,      // OUT, estimated likelihood of the class (0...100)
+            UINT32 *              pNumberOfClasses  // OUT, estimated number of possible classes
+            ) = 0;
+
     // Associates a native call site, identified by its offset in the native code stream, with
     // the signature information and method handle the JIT used to lay out the call site. If
     // the call site has no signature information (e.g. a helper call) or has no method handle

src/coreclr/src/inc/jithelpers.h

@@ -355,6 +355,7 @@
 #endif
 
     JITHELPER(CORINFO_HELP_PATCHPOINT, JIT_Patchpoint, CORINFO_HELP_SIG_REG_ONLY)
+    JITHELPER(CORINFO_HELP_CLASSPROFILE, JIT_ClassProfile, CORINFO_HELP_SIG_REG_ONLY)
 
 #undef JITHELPER
 #undef DYNAMICJITHELPER

src/coreclr/src/jit/block.h

@@ -448,6 +448,7 @@ struct BasicBlock : private LIR::Range
 
 #define BBF_BACKWARD_JUMP_TARGET           MAKE_BBFLAG(36) // Block is a target of a backward jump
 #define BBF_PATCHPOINT                     MAKE_BBFLAG(37) // Block is a patchpoint
+#define BBF_HAS_CLASS_PROFILE              MAKE_BBFLAG(38) // BB contains a call needing a class profile
 
 // clang-format on
 
@@ -492,7 +493,7 @@ struct BasicBlock : private LIR::Range
 #define BBF_SPLIT_GAINED                                                                                               \
     (BBF_DONT_REMOVE | BBF_HAS_LABEL | BBF_HAS_JMP | BBF_BACKWARD_JUMP | BBF_HAS_IDX_LEN | BBF_HAS_NEWARRAY |          \
      BBF_PROF_WEIGHT | BBF_HAS_NEWOBJ | BBF_KEEP_BBJ_ALWAYS | BBF_CLONED_FINALLY_END | BBF_HAS_NULLCHECK |             \
-     BBF_HAS_VTABREF)
+     BBF_HAS_VTABREF | BBF_HAS_CLASS_PROFILE)
 
 #ifndef __GNUC__ // GCC doesn't like C_ASSERT at global scope
     static_assert_no_msg((BBF_SPLIT_NONEXIST & BBF_SPLIT_LOST) == 0);

src/coreclr/src/jit/compiler.cpp

@@ -5982,6 +5982,7 @@ int Compiler::compCompileHelper(CORINFO_MODULE_HANDLE classPtr,
     info.compNativeCodeSize    = 0;
     info.compTotalHotCodeSize  = 0;
     info.compTotalColdCodeSize = 0;
+    info.compClassProbeCount   = 0;
 
     compHasBackwardJump = false;
 

src/coreclr/src/jit/compiler.h

@@ -3779,7 +3779,8 @@ class Compiler
                              CORINFO_CONTEXT_HANDLE* contextHandle,
                              CORINFO_CONTEXT_HANDLE* exactContextHandle,
                              bool                    isLateDevirtualization,
-                             bool                    isExplicitTailCall);
+                             bool                    isExplicitTailCall,
+                             IL_OFFSETX              ilOffset = BAD_IL_OFFSET);
 
     //=========================================================================
     //                          PROTECTED
@@ -6734,7 +6735,8 @@ class Compiler
                                              CORINFO_METHOD_HANDLE methodHandle,
                                              CORINFO_CLASS_HANDLE  classHandle,
                                              unsigned              methodAttr,
-                                             unsigned              classAttr);
+                                             unsigned              classAttr,
+                                             unsigned              likelihood);
 
     bool doesMethodHaveExpRuntimeLookup()
     {
@@ -9299,6 +9301,10 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
 #define CPU_ARM64 0x0400 // The generic ARM64 CPU
 
         unsigned genCPU; // What CPU are we running on
+
+        // Number of class profile probes in this method
+        unsigned compClassProbeCount;
+
     } info;
 
     // Returns true if the method being compiled returns a non-void and non-struct value.