JIT: revise criteria for inline candidates with intrinsics or folding…

… opportunities

Look for IL patterns in an inlinee that are indicative of type folding. Also track
when an inlinee has calls that are intrinsics.

Use this to boost the inlinee profitability estimate.

Also, allow an aggressive inline inline candidate method to go over the budget when
it contains type folding or intrinsics (and so the method may be simpler than it appears).
Remove the old criteria (top-level inline) which was harder to reason about.

Closes dotnet#43761.
Closes dotnet#41692.
Closes dotnet#51587.
Closes dotnet#47434.

src/coreclr/jit/fgbasic.cpp

@@ -853,7 +853,6 @@ void Compiler::fgFindJumpTargets(const BYTE* codeAddr, IL_OFFSET codeSize, Fixed
  const bool isPreJit = opts.jitFlags->IsSet(JitFlags::JIT_FLAG_PREJIT);
  const bool resolveTokens = makeInlineObservations && (isTier1 || isPreJit);
  unsigned retBlocks = 0;
- unsigned intrinsicCalls = 0;
  int prefixFlags = 0;
  int value = 0;
 
@@ -967,7 +966,8 @@ void Compiler::fgFindJumpTargets(const BYTE* codeAddr, IL_OFFSET codeSize, Fixed
 
  if (isJitIntrinsic)
  {
- intrinsicCalls++;
+ compInlineResult->Note(InlineObservation::CALLEE_INTRINSIC_CALL);
+
  ni = lookupNamedIntrinsic(methodHnd);
 
  switch (ni)
@@ -1487,6 +1487,56 @@ void Compiler::fgFindJumpTargets(const BYTE* codeAddr, IL_OFFSET codeSize, Fixed
  retBlocks++;
  break;
 
+ case CEE_BOX:
+ {
+ // Look for evidence of type specialization patterns.
+ //
+ // box + br
+ // box + isinst + br
+ //
+ // Todo: share pattern recognition with impBoxPatternMatch somehow
+ //
+ if (makeInlineObservations && (codeAddr < codeEndp))
+ {
+ switch (codeAddr[0])
+ {
+ case CEE_BRTRUE:
+ case CEE_BRTRUE_S:
+ case CEE_BRFALSE:
+ case CEE_BRFALSE_S:
+ // box + br_true/false
+ if ((codeAddr + ((codeAddr[0] >= CEE_BRFALSE) ? 5 : 2)) <= codeEndp)
+ {
+ compInlineResult->Note(InlineObservation::CALLEE_POSSIBLE_TYPE_FOLD);
+ }
+
+ case CEE_ISINST:
+ if (codeAddr + 1 + sizeof(mdToken) + 1 <= codeEndp)
+ {
+ const BYTE* nextCodeAddr = codeAddr + 1 + sizeof(mdToken);
+ switch (nextCodeAddr[0])
+ {
+ // box + isinst + br_true/false
+ case CEE_BRTRUE:
+ case CEE_BRTRUE_S:
+ case CEE_BRFALSE:
+ case CEE_BRFALSE_S:
+ if ((nextCodeAddr + ((nextCodeAddr[0] >= CEE_BRFALSE) ? 5 : 2)) <= codeEndp)
+ {
+ compInlineResult->Note(InlineObservation::CALLEE_POSSIBLE_TYPE_FOLD);
+ }
+ break;
+ default:
+ break;
+ }
+ }
+
+ default:
+ break;
+ }
+ }
+ }
+
  default:
  break;
  }

src/coreclr/jit/inline.def

@@ -80,6 +80,7 @@ INLINE_OBSERVATION(HAS_PINNED_LOCALS, bool, "has pinned locals",
 INLINE_OBSERVATION(HAS_SIMD, bool, "has SIMD arg, local, or ret", INFORMATION, CALLEE)
 INLINE_OBSERVATION(HAS_SWITCH, bool, "has switch", INFORMATION, CALLEE)
 INLINE_OBSERVATION(IL_CODE_SIZE, int, "number of bytes of IL", INFORMATION, CALLEE)
+INLINE_OBSERVATION(INTRINSIC_CALL, bool, "intrinsic call", INFORMATION, CALLEE)
 INLINE_OBSERVATION(IS_CLASS_CTOR, bool, "class constructor", INFORMATION, CALLEE)
 INLINE_OBSERVATION(IS_DISCRETIONARY_INLINE, bool, "can inline, check heuristics", INFORMATION, CALLEE)
 INLINE_OBSERVATION(IS_FORCE_INLINE, bool, "aggressive inline attribute", INFORMATION, CALLEE)
@@ -94,6 +95,7 @@ INLINE_OBSERVATION(OPCODE_NORMED, int, "next opcode in IL stream"
 INLINE_OBSERVATION(NUMBER_OF_ARGUMENTS, int, "number of arguments", INFORMATION, CALLEE)
 INLINE_OBSERVATION(NUMBER_OF_BASIC_BLOCKS, int, "number of basic blocks", INFORMATION, CALLEE)
 INLINE_OBSERVATION(NUMBER_OF_LOCALS, int, "number of locals", INFORMATION, CALLEE)
+INLINE_OBSERVATION(POSSIBLE_TYPE_FOLD, bool, "possible type fold branch", INFORMATION, CALLEE)
 INLINE_OBSERVATION(RANDOM_ACCEPT, bool, "random accept", INFORMATION, CALLEE)
 INLINE_OBSERVATION(UNSUPPORTED_OPCODE, bool, "unsupported opcode", INFORMATION, CALLEE)
 

src/coreclr/jit/inlinepolicy.cpp

@@ -312,6 +312,14 @@ void DefaultPolicy::NoteBool(InlineObservation obs, bool value)
  m_ArgFeedsRangeCheck++;
  break;
 
+ case InlineObservation::CALLEE_INTRINSIC_CALL:
+ m_IntrinsicCalls++;
+ break;
+
+ case InlineObservation::CALLEE_POSSIBLE_TYPE_FOLD:
+ m_PossibleTypeFolds++;
+ break;
+
  case InlineObservation::CALLEE_HAS_SWITCH:
  case InlineObservation::CALLEE_UNSUPPORTED_OPCODE:
  propagate = true;
@@ -472,20 +480,16 @@ bool DefaultPolicy::BudgetCheck() const
 
  if (overBudget)
  {
- // If the candidate is a forceinline and the callsite is
- // not too deep, allow the inline even if it goes over budget.
- //
- // For now, "not too deep" means a top-level inline. Note
- // depth 0 is used for the root method, so inline candidate depth
- // will be 1 or more.
+ // If the candidate is a forceinline and we've seen evidence
+ // that the callee might not be as expensive as it appears,
+ // allow the inline, even if it goes over budget.
  //
  assert(m_IsForceInlineKnown);
- assert(m_CallsiteDepth > 0);
- const bool allowOverBudget = m_IsForceInline && (m_CallsiteDepth == 1);
+ const bool hasEvidence = (m_IntrinsicCalls > 0) || (m_PossibleTypeFolds > 0);
 
- if (allowOverBudget)
+ if (hasEvidence && m_IsForceInline)
  {
- JITDUMP("Allowing over-budget top-level forceinline\n");
+ JITDUMP("Allowing over-budget forceinline\n");
  }
  else
  {
@@ -733,6 +737,18 @@ double DefaultPolicy::DetermineMultiplier()
  JITDUMP("\nPrejit root candidate has arg that feeds a conditional. Multiplier increased to %g.", multiplier);
  }
 
+ if (m_IntrinsicCalls > 0)
+ {
+ multiplier += 3.0;
+ JITDUMP("\nPossible intrinsic simplifications. Multiplier increased to %g.", multiplier);
+ }
+
+ if (m_PossibleTypeFolds > 0)
+ {
+ multiplier += 3.0;
+ JITDUMP("\nPossible type folding opportunities. Multiplier increased to %g.", multiplier);
+ }
+
  switch (m_CallsiteFrequency)
  {
  case InlineCallsiteFrequency::RARE:

src/coreclr/jit/inlinepolicy.h

@@ -96,6 +96,8 @@ class DefaultPolicy : public LegalPolicy
  , m_ArgFeedsConstantTest(0)
  , m_ArgFeedsRangeCheck(0)
  , m_ConstantArgFeedsConstantTest(0)
+ , m_IntrinsicCalls(0)
+ , m_PossibleTypeFolds(0)
  , m_CalleeNativeSizeEstimate(0)
  , m_CallsiteNativeSizeEstimate(0)
  , m_IsForceInline(false)
@@ -164,6 +166,8 @@ class DefaultPolicy : public LegalPolicy
  unsigned m_ArgFeedsConstantTest;
  unsigned m_ArgFeedsRangeCheck;
  unsigned m_ConstantArgFeedsConstantTest;
+ unsigned m_IntrinsicCalls;
+ unsigned m_PossibleTypeFolds;
  int m_CalleeNativeSizeEstimate;
  int m_CallsiteNativeSizeEstimate;
  bool m_IsForceInline : 1;