JIT: Remove unneeded unconditional jumps

src/coreclr/jit/codegen.h

@@ -623,7 +623,7 @@ class CodeGen final : public CodeGenInterface
     }
 
 #endif // !FEATURE_EH_FUNCLETS
-
+    void genUpdateLiveRangesForTruncatedIGs();
     void genGeneratePrologsAndEpilogs();
 
 #if defined(DEBUG) && defined(TARGET_ARM64)

src/coreclr/jit/codegencommon.cpp

@@ -1969,6 +1969,8 @@ void CodeGen::genGenerateMachineCode()
 
     GetEmitter()->emitJumpDistBind();
 
+    genUpdateLiveRangesForTruncatedIGs();
+
 #if FEATURE_LOOP_ALIGN
     /* Perform alignment adjustments */
 
@@ -1978,6 +1980,113 @@ void CodeGen::genGenerateMachineCode()
     /* The code is now complete and final; it should not change after this. */
 }
 
+void CodeGen::genUpdateLiveRangesForTruncatedIGs()
+{
+#ifdef DEBUG
+    if (verbose)
+    {
+        printf("*************** In genUpdateLiveRangesForTruncatedIGs()\n");
+    }
+#endif
+    bool updatedGroup = false;
+
+    if (compiler->lvaCount > 0)
+    {
+        insGroup* ig = GetEmitter()->emitIGlist;
+        while (ig != nullptr)
+        {
+            if (ig->igFlags & IGF_UPD_ICOUNT)
+            {
+
+                unsigned int liveVarCount = varLiveKeeper->getVarCount();
+                if (liveVarCount > 0)
+                {
+#ifdef DEBUG
+                    if (compiler->verbose)
+                    {
+                        JITDUMP("IG%02u is marked with IGF_UPD_ICOUNT and has %d live variables\n", ig->igNum,
+                                liveVarCount);
+                    }
+#endif
+                    for (unsigned int varNum = 0; varNum < liveVarCount; varNum++)
+                    {
+                        if (compiler->compMap2ILvarNum(varNum) == (unsigned int)ICorDebugInfo::UNKNOWN_ILNUM)
+                        {
+                            continue;
+                        }
+
+                        for (int rangeIndex = 0; rangeIndex < 2; rangeIndex++)
+                        {
+                            VariableLiveKeeper::LiveRangeList* liveRanges;
+                            if (rangeIndex == 0)
+                            {
+                                liveRanges = varLiveKeeper->getLiveRangesForVarForProlog(varNum);
+                            }
+                            else
+                            {
+                                liveRanges = varLiveKeeper->getLiveRangesForVarForBody(varNum);
+                            }
+
+                            for (VariableLiveKeeper::VariableLiveRange& liveRange : *liveRanges)
+                            {
+                                if (liveRange.m_StartEmitLocation.GetIG()->igNum == ig->igNum &&
+                                    liveRange.m_StartEmitLocation.GetInsNum() > ig->igInsCnt)
+                                {
+#ifdef DEBUG
+                                    if (compiler->verbose)
+                                    {
+                                        JITDUMP("IG%02u varNum %d StartEmitLocation InsCnt changed from %d to %d\n", ig,
+                                                liveVarCount, liveRange.m_StartEmitLocation.GetInsNum(), ig->igInsCnt);
+                                    }
+#endif
+                                    liveRange.m_StartEmitLocation.SetInsNum(ig->igInsCnt);
+                                    assert(liveRange.m_StartEmitLocation.GetInsNum() == ig->igInsCnt);
+                                    updatedGroup = true;
+                                }
+
+                                if (liveRange.m_EndEmitLocation.GetIG()->igNum == ig->igNum &&
+                                    liveRange.m_EndEmitLocation.GetInsNum() > ig->igInsCnt)
+                                {
+                                    liveRange.m_EndEmitLocation.SetInsNum(ig->igInsCnt);
+#ifdef DEBUG
+                                    if (compiler->verbose)
+                                    {
+                                        JITDUMP("IG%02u varNum %d EndEmitLocation InsCnt changed from %d to %d\n", ig,
+                                                liveVarCount, liveRange.m_StartEmitLocation.GetInsNum(), ig->igInsCnt);
+                                    }
+#endif
+                                    assert(liveRange.m_EndEmitLocation.GetInsNum() == ig->igInsCnt);
+                                    updatedGroup = true;
+                                }
+                            }
+                        }
+                    }
+                }
+                else
+                {
+#ifdef DEBUG
+                    if (compiler->verbose)
+                    {
+                        JITDUMP("IG%02u is marked with IGF_UPD_ICOUNT but has no live variables\n", ig);
+                    }
+#endif
+                }
+
+                ig->igFlags ^= IGF_UPD_ICOUNT;
+            }
+
+            ig = ig->igNext;
+        }
+    }
+#ifdef DEBUG
+    if (updatedGroup && verbose)
+    {
+        printf("\nlvaTable after genUpdateLiveRangesForTruncatedIGs\n");
+        compiler->lvaTableDump();
+    }
+#endif
+}
+
 //----------------------------------------------------------------------
 // genEmitMachineCode -- emit the actual machine instruction code
 //
@@ -7174,7 +7283,6 @@ void CodeGen::genSetScopeInfoUsingVariableRanges()
             reportRange(curLoc, curStart, curEnd);
         }
     }
-
     compiler->eeVarsCount = liveRangeIndex;
 }
 #endif // USING_VARIABLE_LIVE_RANGE
@@ -9279,7 +9387,7 @@ CodeGenInterface::VariableLiveKeeper::LiveRangeList* CodeGenInterface::VariableL
 }
 
 //------------------------------------------------------------------------
-// getLiveRangesCount: Returns the count of variable locations reported for the tracked
+// getLiveRangesCount: Returns the size of variable locations reported for the tracked
 //  variables, which are arguments, special arguments, and local IL variables.
 //
 // Return Value:
@@ -9311,6 +9419,30 @@ size_t CodeGenInterface::VariableLiveKeeper::getLiveRangesCount() const
     return liveRangesCount;
 }
 
+//------------------------------------------------------------------------
+// getVarCount: Returns the count of variable locations reported for the tracked
+//  variables, which are arguments, special arguments, and local IL variables.
+//
+// Return Value:
+//    unsigned int - the count of variables
+//
+unsigned int CodeGenInterface::VariableLiveKeeper::getVarCount() const
+{
+    unsigned int liveRangesCount = 0;
+
+    if (m_Compiler->opts.compDbgInfo)
+    {
+        for (unsigned int varNum = 0; varNum < m_LiveDscCount; varNum++)
+        {
+            if (m_Compiler->compMap2ILvarNum(varNum) != (unsigned int)ICorDebugInfo::UNKNOWN_ILNUM)
+            {
+                liveRangesCount += 1;
+            }
+        }
+    }
+    return liveRangesCount;
+};
+
 //------------------------------------------------------------------------
 // psiStartVariableLiveRange: Reports the given variable as being born.
 //

src/coreclr/jit/codegeninterface.h

@@ -743,7 +743,8 @@ class CodeGenInterface
 
         LiveRangeList* getLiveRangesForVarForBody(unsigned int varNum) const;
         LiveRangeList* getLiveRangesForVarForProlog(unsigned int varNum) const;
-        size_t getLiveRangesCount() const;
+        size_t       getLiveRangesCount() const;
+        unsigned int getVarCount() const;
 
         // For parameters locations on prolog
         void psiStartVariableLiveRange(CodeGenInterface::siVarLoc varLocation, unsigned int varNum);

src/coreclr/jit/emit.cpp

@@ -29,7 +29,6 @@ void emitLocation::CaptureLocation(emitter* emit)
 {
     ig      = emit->emitCurIG;
     codePos = emit->emitCurOffset();
-
     assert(Valid());
 }
 
@@ -50,6 +49,13 @@ int emitLocation::GetInsNum() const
     return emitGetInsNumFromCodePos(codePos);
 }
 
+void emitLocation::SetInsNum(int insNum)
+{
+    assert((unsigned int)insNum == emitGetInsNumFromCodePos(insNum));
+    codePos = emitGetInsNumFromCodePos(insNum) + (emitGetInsOfsFromCodePos(codePos) << 16);
+    assert(Valid());
+}
+
 // Get the instruction offset in the current instruction group, which must be a funclet prolog group.
 // This is used to find an instruction offset used in unwind data.
 // TODO-AMD64-Bug?: We only support a single main function prolog group, but allow for multiple funclet prolog
@@ -3613,6 +3619,10 @@ void emitter::emitDispIGflags(unsigned flags)
     {
         printf(", isz");
     }
+    if (flags & IGF_UPD_ICOUNT)
+    {
+        printf(", ict");
+    }
     if (flags & IGF_EXTEND)
     {
         printf(", extend");
@@ -4139,6 +4149,10 @@ void emitter::emitJumpDistBind()
 #endif // DEBUG
 
     int jmp_iteration = 1;
+#ifdef TARGET_XARCH
+    int  jmp_removal = 1;
+    bool jmpRemoved  = false;
+#endif
 
 /*****************************************************************************/
 /* If we iterate to look for more jumps to shorten, we start again here.     */
@@ -4165,6 +4179,110 @@ void emitter::emitJumpDistBind()
     adjIG         = 0;
     minShortExtra = (UNATIVE_OFFSET)-1;
 
+#ifdef TARGET_XARCH
+    // try to remove unconditional jumps at the end of a group to the next group
+
+    {
+        jmp                       = emitJumpList;
+        instrDescJmp* previousJmp = nullptr;
+        while (jmp)
+        {
+            // if the jump is unconditional then it is a candidate
+            if (jmp->idInsFmt() == IF_LABEL && emitIsUncondJump(jmp) && !jmp->idjKeepLong)
+            {
+                // target group is not bound yet so use the cookie to fetch it
+                insGroup* targetGroup = (insGroup*)emitCodeGetCookie(jmp->idAddr()->iiaBBlabel);
+
+                // if the target group is the next instruction group
+                if (targetGroup != nullptr && jmp->idjIG->igNext == targetGroup)
+                {
+                    insGroup* group            = jmp->idjIG;
+                    unsigned  instructionCount = group->igInsCnt;
+
+                    if (instructionCount > 0)
+                    {
+                        BYTE*      dataPtr               = group->igData;
+                        instrDesc* instructionDescriptor = nullptr;
+
+                        do
+                        {
+                            instructionDescriptor = (instrDesc*)dataPtr;
+                            dataPtr += emitSizeOfInsDsc(instructionDescriptor);
+                        } while (--instructionCount);
+
+                        assert(instructionDescriptor != nullptr);
+                        // instructionDescriptor is now the last instruction in the group
+
+                        if (instructionDescriptor->idIns() == jmp->idIns() && jmp == instructionDescriptor)
+                        {
+// the last instruction in the group is the jmp we're looking for
+// and it jumps to the next instruction group so we don't really need it
+#ifdef DEBUG
+                            if (EMITVERBOSE)
+                            {
+                                printf("Removing unconditional jump [%08X/%03u] from the last instruction in IG%02u to "
+                                       "the next IG IG%02u label %s\n",
+                                       dspPtr(jmp), jmp->idDebugOnlyInfo()->idNum, group->igNum, targetGroup->igNum,
+                                       emitLabelString(targetGroup));
+                            }
+#endif
+                            UNATIVE_OFFSET sizeRemoved = instructionDescriptor->idCodeSize();
+
+                            // set state needed at the end of the loop in ADJUST_GROUP
+                            adjIG      = sizeRemoved;
+                            lstIG      = group;
+                            jmpRemoved = true;
+
+                            // unlink the jump
+                            if (previousJmp != nullptr)
+                            {
+                                previousJmp->idjNext = jmp->idjNext;
+                            }
+                            else
+                            {
+                                assert(jmp == emitJumpList);
+                                emitJumpList = jmp->idjNext;
+                            }
+                            jmp->idjNext = nullptr;
+
+#if DEBUG
+                            // clear the instruction data
+                            memset((BYTE*)instructionDescriptor, 0, instructionDescriptor->idCodeSize());
+#endif
+
+                            // remove the instruction from the group
+                            group->igInsCnt -= 1;
+                            group->igSize -= sizeRemoved;
+                            emitTotalCodeSize -= sizeRemoved;
+
+                            // flag the group as resized and recounted
+                            group->igFlags |= (IGF_UPD_ISZ | IGF_UPD_ICOUNT);
+
+                            // cleanup
+                            instructionDescriptor = nullptr;
+                            jmp                   = nullptr;
+
+                            // jump to adjusting the size
+                            goto ADJUST_GROUP;
+                        }
+                    }
+                }
+            }
+
+            previousJmp = jmp;
+            if (jmp != nullptr)
+            {
+                jmp = jmp->idjNext;
+            }
+        }
+
+        jmp   = nullptr;
+        lstIG = nullptr;
+        adjLJ = 0;
+        adjIG = 0;
+    }
+#endif
+
 #if defined(TARGET_ARM)
     minMediumExtra = (UNATIVE_OFFSET)-1;
 #endif // TARGET_ARM
@@ -4784,8 +4902,11 @@ void emitter::emitJumpDistBind()
 
     } // end for each jump
 
-    /* Did we shorten any jumps? */
+#ifdef TARGET_XARCH
+ADJUST_GROUP:
+#endif
 
+    /* Did we shorten any jumps? */
     if (adjIG)
     {
         /* Adjust offsets of any remaining blocks */
@@ -4829,6 +4950,22 @@ void emitter::emitJumpDistBind()
 #endif
 #endif
 
+#ifdef TARGET_XARCH
+        if (jmpRemoved)
+        {
+
+            jmp_removal++;
+#ifdef DEBUG
+            if (EMITVERBOSE)
+            {
+                printf("Iterating branch removal. Iteration = %d\n", jmp_removal);
+            }
+#endif
+            jmpRemoved = false;
+            goto AGAIN;
+        }
+#endif
+
         if ((minShortExtra <= adjIG)
 #if defined(TARGET_ARM)
             || (minMediumExtra <= adjIG)