diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index d05bd97d3276ee..3d8ab00af272b4 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -1495,6 +1495,14 @@ bool Lowering::TryLowerSwitchToBitTest(FlowEdge* jumpTable[], return true; } +//------------------------------------------------------------------------ +// ReplaceArgWithPutArgOrBitcast: Insert a PUTARG_* node in the right location +// and replace the call operand with that node. +// +// Arguments: +// argSlot - slot in call of argument +// putArgOrBitcast - the node that is being inserted +// void Lowering::ReplaceArgWithPutArgOrBitcast(GenTree** argSlot, GenTree* putArgOrBitcast) { assert(argSlot != nullptr); @@ -1981,6 +1989,113 @@ void Lowering::LowerArgsForCall(GenTreeCall* call) { LowerArg(call, &arg, true); } + + LegalizeArgPlacement(call); +} + +//------------------------------------------------------------------------ +// LegalizeArgPlacement: Move arg placement nodes (PUTARG_*) into a legal +// ordering after they have been created. +// +// Arguments: +// call - GenTreeCall node that has had PUTARG_* nodes created for arguments. +// +// Remarks: +// PUTARG_* nodes are created and inserted right after the definitions of the +// argument values. However, there are constraints on how the PUTARG nodes +// can appear: +// +// - No other GT_CALL nodes are allowed between a PUTARG_REG/PUTARG_SPLIT +// node and the call. For FEATURE_FIXED_OUT_ARGS this condition is also true +// for PUTARG_STK. +// - For !FEATURE_FIXED_OUT_ARGS, the PUTARG_STK nodes must come in push +// order. +// +// Morph has mostly already solved this problem, but transformations on LIR +// can make the ordering we end up with here illegal. This function legalizes +// the placement while trying to minimize the distance between an argument +// definition and its corresponding placement node. +// +void Lowering::LegalizeArgPlacement(GenTreeCall* call) +{ + size_t numMarked = MarkCallPutArgAndFieldListNodes(call); + + // We currently do not try to resort the PUTARG_STK nodes, but rather just + // assert here that they are ordered. +#if defined(DEBUG) && !FEATURE_FIXED_OUT_ARGS + unsigned nextPushOffset = UINT_MAX; +#endif + + GenTree* cur = call->gtPrev; + while (numMarked > 0) + { + assert(cur != nullptr); + + if ((cur->gtLIRFlags & LIR::Flags::Mark) != 0) + { + numMarked--; + cur->gtLIRFlags &= ~LIR::Flags::Mark; + +#if defined(DEBUG) && !FEATURE_FIXED_OUT_ARGS + if (cur->OperIs(GT_PUTARG_STK)) + { + // For !FEATURE_FIXED_OUT_ARGS (only x86) byte offsets are + // subtracted from the top of the stack frame; so last pushed + // arg has highest offset. + assert(nextPushOffset > cur->AsPutArgStk()->getArgOffset()); + nextPushOffset = cur->AsPutArgStk()->getArgOffset(); + } +#endif + } + + if (cur->IsCall()) + { + break; + } + + cur = cur->gtPrev; + } + + if (numMarked == 0) + { + // Already legal; common case + return; + } + + JITDUMP("Call [%06u] has %zu PUTARG nodes that interfere with [%06u]; will move them after it\n", + Compiler::dspTreeID(call), numMarked, Compiler::dspTreeID(cur)); + + // We found interference; remaining PUTARG nodes need to be moved after + // this point. + GenTree* insertionPoint = cur; + + while (numMarked > 0) + { + assert(cur != nullptr); + + GenTree* prev = cur->gtPrev; + if ((cur->gtLIRFlags & LIR::Flags::Mark) != 0) + { + numMarked--; + cur->gtLIRFlags &= ~LIR::Flags::Mark; + + // For FEATURE_FIXED_OUT_ARGS: all PUTARG nodes must be moved after the interfering call + // For !FEATURE_FIXED_OUT_ARGS: only PUTARG_REG nodes must be moved after the interfering call + if (FEATURE_FIXED_OUT_ARGS || cur->OperIs(GT_FIELD_LIST, GT_PUTARG_REG)) + { + JITDUMP("Relocating [%06u] after [%06u]\n", Compiler::dspTreeID(cur), + Compiler::dspTreeID(insertionPoint)); + + BlockRange().Remove(cur); + BlockRange().InsertAfter(insertionPoint, cur); + } + } + + cur = prev; + } + + JITDUMP("Final result after legalization:\n"); + DISPTREERANGE(BlockRange(), call); } // helper that create a node representing a relocatable physical address computation @@ -1997,6 +2112,7 @@ GenTree* Lowering::AddrGen(void* addr) return AddrGen((ssize_t)addr); } +//------------------------------------------------------------------------ // LowerCallMemset: Replaces the following memset-like special intrinsics: // // SpanHelpers.Fill(ref dstRef, CNS_SIZE, CNS_VALUE) @@ -2780,19 +2896,7 @@ void Lowering::InsertProfTailCallHook(GenTreeCall* call, GenTree* insertionPoint // GenTree* Lowering::FindEarliestPutArg(GenTreeCall* call) { - size_t numMarkedNodes = 0; - for (CallArg& arg : call->gtArgs.Args()) - { - if (arg.GetEarlyNode() != nullptr) - { - numMarkedNodes += MarkPutArgNodes(arg.GetEarlyNode()); - } - - if (arg.GetLateNode() != nullptr) - { - numMarkedNodes += MarkPutArgNodes(arg.GetLateNode()); - } - } + size_t numMarkedNodes = MarkCallPutArgAndFieldListNodes(call); if (numMarkedNodes <= 0) { @@ -2818,33 +2922,68 @@ GenTree* Lowering::FindEarliestPutArg(GenTreeCall* call) } //------------------------------------------------------------------------ -// MarkPutArgNodes: Mark all direct operand PUTARG nodes with a LIR mark. +// MarkCallPutArgNodes: Mark all operand FIELD_LIST and PUTARG nodes +// corresponding to a call. // // Arguments: -// node - the node (either a field list or PUTARG node) +// call - the call // // Returns: -// The number of marks added. +// The number of nodes marked. +// +// Remarks: +// FIELD_LIST operands are marked too, and their PUTARG operands are in turn +// marked as well. // -size_t Lowering::MarkPutArgNodes(GenTree* node) +size_t Lowering::MarkCallPutArgAndFieldListNodes(GenTreeCall* call) +{ + size_t numMarkedNodes = 0; + for (CallArg& arg : call->gtArgs.Args()) + { + if (arg.GetEarlyNode() != nullptr) + { + numMarkedNodes += MarkPutArgAndFieldListNodes(arg.GetEarlyNode()); + } + + if (arg.GetLateNode() != nullptr) + { + numMarkedNodes += MarkPutArgAndFieldListNodes(arg.GetLateNode()); + } + } + + return numMarkedNodes; +} + +//------------------------------------------------------------------------ +// MarkPutArgAndFieldListNodes: Mark all operand FIELD_LIST and PUTARG nodes +// with a LIR mark. +// +// Arguments: +// node - the node (either a FIELD_LIST or PUTARG operand) +// +// Returns: +// The number of marks added. +// +// Remarks: +// FIELD_LIST operands are marked too, and their PUTARG operands are in turn +// marked as well. +// +size_t Lowering::MarkPutArgAndFieldListNodes(GenTree* node) { assert(node->OperIsPutArg() || node->OperIsFieldList()); - size_t result = 0; + assert((node->gtLIRFlags & LIR::Flags::Mark) == 0); + node->gtLIRFlags |= LIR::Flags::Mark; + + size_t result = 1; if (node->OperIsFieldList()) { for (GenTreeFieldList::Use& operand : node->AsFieldList()->Uses()) { assert(operand.GetNode()->OperIsPutArg()); - result += MarkPutArgNodes(operand.GetNode()); + result += MarkPutArgAndFieldListNodes(operand.GetNode()); } } - else - { - assert((node->gtLIRFlags & LIR::Flags::Mark) == 0); - node->gtLIRFlags |= LIR::Flags::Mark; - result++; - } return result; } diff --git a/src/coreclr/jit/lower.h b/src/coreclr/jit/lower.h index 898bc8fef53897..480b343d98b537 100644 --- a/src/coreclr/jit/lower.h +++ b/src/coreclr/jit/lower.h @@ -175,7 +175,8 @@ class Lowering final : public Phase GenTreeCall* callNode); void InsertProfTailCallHook(GenTreeCall* callNode, GenTree* insertionPoint); GenTree* FindEarliestPutArg(GenTreeCall* call); - size_t MarkPutArgNodes(GenTree* node); + size_t MarkCallPutArgAndFieldListNodes(GenTreeCall* call); + size_t MarkPutArgAndFieldListNodes(GenTree* node); GenTree* LowerVirtualVtableCall(GenTreeCall* call); GenTree* LowerVirtualStubCall(GenTreeCall* call); void LowerArgsForCall(GenTreeCall* call); @@ -186,6 +187,7 @@ class Lowering final : public Phase GenTree* LowerFloatArg(GenTree** pArg, CallArg* callArg); GenTree* LowerFloatArgReg(GenTree* arg, regNumber regNum); #endif + void LegalizeArgPlacement(GenTreeCall* call); void InsertPInvokeCallProlog(GenTreeCall* call); void InsertPInvokeCallEpilog(GenTreeCall* call);