Skip to content

Commit

Permalink
JIT: Store one segment inline in ABIPassingInformation (#103690)
Browse files Browse the repository at this point in the history 
By far the most common case on all platforms is that an argument takes
up one segment to describe its passing behavior, so we can avoid
additional allocations by storing one segment inline.
  • Loading branch information
@jakobbotsch
jakobbotsch authored Jun 19, 2024
1 parent 74f7f91 commit 0498939
Show file tree
Hide file tree
Showing 13 changed files with 169 additions and 80 deletions.
110 changes: 96 additions & 14 deletions src/coreclr/jit/abi.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,65 @@ ABIPassingSegment ABIPassingSegment::OnStack(unsigned stackOffset, unsigned offs
return segment;
}

//-----------------------------------------------------------------------------
// ABIPassingInformation:
// Construct an instance with the specified number of segments allocated in
// the backing storage.
//
// Parameters:
// comp - Compiler instance
// numSegments - Number of segments
//
// Remarks:
// The segments are expected to be filled out by the caller after the
// allocation; they are not zeroed out by the allocation.
//
ABIPassingInformation::ABIPassingInformation(Compiler* comp, unsigned numSegments)
{
NumSegments = numSegments;

if (numSegments > 1)
{
Segments = new (comp, CMK_ABI) ABIPassingSegment[numSegments];
}
}

//-----------------------------------------------------------------------------
// Segment:
// Access a segment by the specified index.
//
// Parameters:
// index - The index of the segment
//
// Returns:
// Reference to segment.
//
const ABIPassingSegment& ABIPassingInformation::Segment(unsigned index) const
{
assert(index < NumSegments);
if (NumSegments == 1)
{
return SingleSegment;
}

return Segments[index];
}

//-----------------------------------------------------------------------------
// Segment:
// Access a segment by the specified index.
//
// Parameters:
// index - The index of the segment
//
// Returns:
// Reference to segment.
//
ABIPassingSegment& ABIPassingInformation::Segment(unsigned index)
{
return const_cast<ABIPassingSegment&>(static_cast<const ABIPassingInformation&>(*this).Segment(index));
}

//-----------------------------------------------------------------------------
// HasAnyRegisterSegment:
// Check if any part of this value is passed in a register.
Expand All @@ -198,7 +257,7 @@ bool ABIPassingInformation::HasAnyRegisterSegment() const
{
for (unsigned i = 0; i < NumSegments; i++)
{
if (Segments[i].IsPassedInRegister())
if (Segment(i).IsPassedInRegister())
{
return true;
}
Expand All @@ -217,7 +276,7 @@ bool ABIPassingInformation::HasAnyStackSegment() const
{
for (unsigned i = 0; i < NumSegments; i++)
{
if (Segments[i].IsPassedOnStack())
if (Segment(i).IsPassedOnStack())
{
return true;
}
Expand All @@ -234,7 +293,7 @@ bool ABIPassingInformation::HasAnyStackSegment() const
//
bool ABIPassingInformation::HasExactlyOneRegisterSegment() const
{
return (NumSegments == 1) && Segments[0].IsPassedInRegister();
return (NumSegments == 1) && Segment(0).IsPassedInRegister();
}

//-----------------------------------------------------------------------------
Expand All @@ -246,7 +305,7 @@ bool ABIPassingInformation::HasExactlyOneRegisterSegment() const
//
bool ABIPassingInformation::HasExactlyOneStackSegment() const
{
return (NumSegments == 1) && Segments[0].IsPassedOnStack();
return (NumSegments == 1) && Segment(0).IsPassedOnStack();
}

//-----------------------------------------------------------------------------
Expand All @@ -264,10 +323,10 @@ bool ABIPassingInformation::IsSplitAcrossRegistersAndStack() const
return false;
}

bool isFirstInReg = Segments[0].IsPassedInRegister();
bool isFirstInReg = Segment(0).IsPassedInRegister();
for (unsigned i = 1; i < NumSegments; i++)
{
if (isFirstInReg != Segments[i].IsPassedInRegister())
if (isFirstInReg != Segment(i).IsPassedInRegister())
{
return true;
}
Expand All @@ -288,7 +347,32 @@ bool ABIPassingInformation::IsSplitAcrossRegistersAndStack() const
//
ABIPassingInformation ABIPassingInformation::FromSegment(Compiler* comp, const ABIPassingSegment& segment)
{
return {1, new (comp, CMK_ABI) ABIPassingSegment(segment)};
ABIPassingInformation info;
info.NumSegments = 1;
info.SingleSegment = segment;
return info;
}

//-----------------------------------------------------------------------------
// FromSegments:
// Create ABIPassingInformation from two segments.
//
// Parameters:
// comp - Compiler instance
// firstSegment - The first segment that represents the passing information
// secondSegment - The second segment that represents the passing information
//
// Return Value:
// An instance of ABIPassingInformation.
//
ABIPassingInformation ABIPassingInformation::FromSegments(Compiler* comp,
const ABIPassingSegment& firstSegment,
const ABIPassingSegment& secondSegment)
{
ABIPassingInformation info;
info.NumSegments = 2;
info.Segments = new (comp, CMK_ABI) ABIPassingSegment[2]{firstSegment, secondSegment};
return info;
}

#ifdef DEBUG
Expand All @@ -310,9 +394,9 @@ void ABIPassingInformation::Dump() const
printf(" [%u] ", i);
}

const ABIPassingSegment& seg = Segments[i];
const ABIPassingSegment& seg = Segment(i);

if (Segments[i].IsPassedInRegister())
if (seg.IsPassedInRegister())
{
printf("[%02u..%02u) reg %s\n", seg.Offset, seg.Offset + seg.Size, getRegName(seg.GetRegister()));
}
Expand Down Expand Up @@ -418,7 +502,7 @@ ABIPassingInformation SwiftABIClassifier::Classify(Compiler* comp,

for (unsigned j = 0; j < elemInfo.NumSegments; j++)
{
ABIPassingSegment newSegment = elemInfo.Segments[j];
ABIPassingSegment newSegment = elemInfo.Segment(j);
newSegment.Offset += lowering->offsets[i];
// Adjust the tail size if necessary; the lowered sequence can
// pass the tail as a larger type than the tail size.
Expand All @@ -427,12 +511,10 @@ ABIPassingInformation SwiftABIClassifier::Classify(Compiler* comp,
}
}

ABIPassingInformation result;
result.NumSegments = static_cast<unsigned>(segments.Height());
result.Segments = new (comp, CMK_ABI) ABIPassingSegment[result.NumSegments];
ABIPassingInformation result(comp, static_cast<unsigned>(segments.Height()));
for (int i = 0; i < segments.Height(); i++)
{
result.Segments[i] = segments.Bottom(i);
result.Segment(i) = segments.Bottom(i);
}

return result;
Expand Down
24 changes: 19 additions & 5 deletions src/coreclr/jit/abi.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,14 @@ class ABIPassingSegment

struct ABIPassingInformation
{
private:
union
{
ABIPassingSegment* Segments;
ABIPassingSegment SingleSegment;
};

public:
// The number of segments used to pass the value. Examples:
// - On SysV x64, structs can be passed in two registers, resulting in two
// register segments
Expand All @@ -51,22 +59,28 @@ struct ABIPassingInformation
// - On loongarch64/riscv64, structs can be passed in two registers or
// can be split out over register and stack, giving
// multiple register segments and a struct segment.
unsigned NumSegments;
ABIPassingSegment* Segments;
unsigned NumSegments;

ABIPassingInformation(unsigned numSegments = 0, ABIPassingSegment* segments = nullptr)
: NumSegments(numSegments)
, Segments(segments)
ABIPassingInformation()
: NumSegments(0)
{
}

ABIPassingInformation(Compiler* comp, unsigned numSegments);

const ABIPassingSegment& Segment(unsigned index) const;
ABIPassingSegment& Segment(unsigned index);

bool HasAnyRegisterSegment() const;
bool HasAnyStackSegment() const;
bool HasExactlyOneRegisterSegment() const;
bool HasExactlyOneStackSegment() const;
bool IsSplitAcrossRegistersAndStack() const;

static ABIPassingInformation FromSegment(Compiler* comp, const ABIPassingSegment& segment);
static ABIPassingInformation FromSegments(Compiler* comp,
const ABIPassingSegment& firstSegment,
const ABIPassingSegment& secondSegment);

#ifdef WINDOWS_AMD64_ABI
static bool GetShadowSpaceCallerOffsetForReg(regNumber reg, int* offset);
Expand Down
2 changes: 1 addition & 1 deletion src/coreclr/jit/codegenarmarch.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3802,7 +3802,7 @@ void CodeGen::genJmpPlaceVarArgs()
const ABIPassingInformation& abiInfo = compiler->lvaGetParameterABIInfo(varNum);
for (unsigned i = 0; i < abiInfo.NumSegments; i++)
{
const ABIPassingSegment& segment = abiInfo.Segments[i];
const ABIPassingSegment& segment = abiInfo.Segment(i);
if (segment.IsPassedInRegister())
{
potentialArgs &= ~segment.GetRegisterMask();
Expand Down
14 changes: 7 additions & 7 deletions src/coreclr/jit/codegencommon.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3202,7 +3202,7 @@ void CodeGen::genSpillOrAddRegisterParam(unsigned lclNum, RegGraph* graph)
const ABIPassingInformation& abiInfo = compiler->lvaGetParameterABIInfo(paramLclNum);
for (unsigned i = 0; i < abiInfo.NumSegments; i++)
{
const ABIPassingSegment& seg = abiInfo.Segments[i];
const ABIPassingSegment& seg = abiInfo.Segment(i);
if (!seg.IsPassedInRegister() || ((paramRegs & genRegMask(seg.GetRegister())) == 0))
{
continue;
Expand Down Expand Up @@ -3325,7 +3325,7 @@ void CodeGen::genHomeRegisterParams(regNumber initReg, bool* initRegStillZeroed)
const ABIPassingInformation& abiInfo = compiler->lvaGetParameterABIInfo(lclNum);
for (unsigned i = 0; i < abiInfo.NumSegments; i++)
{
const ABIPassingSegment& seg = abiInfo.Segments[i];
const ABIPassingSegment& seg = abiInfo.Segment(i);
if (seg.IsPassedInRegister() && ((paramRegs & genRegMask(seg.GetRegister())) != 0))
{
var_types storeType = genParamStackType(lclDsc, seg);
Expand Down Expand Up @@ -4373,7 +4373,7 @@ void CodeGen::genHomeSwiftStructParameters(bool handleStack)

for (unsigned i = 0; i < abiInfo.NumSegments; i++)
{
const ABIPassingSegment& seg = abiInfo.Segments[i];
const ABIPassingSegment& seg = abiInfo.Segment(i);
if (seg.IsPassedOnStack() != handleStack)
{
continue;
Expand Down Expand Up @@ -4434,9 +4434,9 @@ void CodeGen::genHomeStackPartOfSplitParameter(regNumber initReg, bool* initRegS
JITDUMP("Homing stack part of split parameter V%02u\n", lclNum);

assert(abiInfo.NumSegments == 2);
assert(abiInfo.Segments[0].GetRegister() == REG_ARG_LAST);
assert(abiInfo.Segments[1].GetStackOffset() == 0);
const ABIPassingSegment& seg = abiInfo.Segments[1];
assert(abiInfo.Segment(0).GetRegister() == REG_ARG_LAST);
assert(abiInfo.Segment(1).GetStackOffset() == 0);
const ABIPassingSegment& seg = abiInfo.Segment(1);

genHomeStackSegment(lclNum, seg, initReg, initRegStillZeroed);

Expand Down Expand Up @@ -7514,7 +7514,7 @@ void CodeGen::genJmpPlaceArgs(GenTree* jmp)
const ABIPassingInformation& abiInfo = compiler->lvaGetParameterABIInfo(varNum);
for (unsigned i = 0; i < abiInfo.NumSegments; i++)
{
const ABIPassingSegment& segment = abiInfo.Segments[i];
const ABIPassingSegment& segment = abiInfo.Segment(i);
if (segment.IsPassedOnStack())
{
continue;
Expand Down
2 changes: 1 addition & 1 deletion src/coreclr/jit/codegenxarch.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6565,7 +6565,7 @@ void CodeGen::genJmpPlaceVarArgs()
const ABIPassingInformation& abiInfo = compiler->lvaGetParameterABIInfo(varNum);
for (unsigned i = 0; i < abiInfo.NumSegments; i++)
{
const ABIPassingSegment& segment = abiInfo.Segments[i];
const ABIPassingSegment& segment = abiInfo.Segment(i);
if (segment.IsPassedOnStack())
{
continue;
Expand Down
20 changes: 10 additions & 10 deletions src/coreclr/jit/lclvars.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1809,24 +1809,24 @@ void Compiler::lvaClassifyParameterABI()
else
{
assert(abiInfo.NumSegments == 1);
if (abiInfo.Segments[0].IsPassedInRegister())
if (abiInfo.Segment(0).IsPassedInRegister())
{
dsc->lvIsRegArg = true;
dsc->SetArgReg(abiInfo.Segments[0].GetRegister());
dsc->SetArgReg(abiInfo.Segment(0).GetRegister());
dsc->SetOtherArgReg(REG_NA);
}
else
{
dsc->lvIsRegArg = false;
dsc->SetArgReg(REG_STK);
dsc->SetOtherArgReg(REG_NA);
dsc->SetStackOffset(abiInfo.Segments[0].GetStackOffset());
dsc->SetStackOffset(abiInfo.Segment(0).GetStackOffset());
}
}

for (unsigned i = 0; i < abiInfo.NumSegments; i++)
{
const ABIPassingSegment& segment = abiInfo.Segments[i];
const ABIPassingSegment& segment = abiInfo.Segment(i);
if (segment.IsPassedInRegister())
{
argRegs |= segment.GetRegisterMask();
Expand Down Expand Up @@ -1887,7 +1887,7 @@ void Compiler::lvaClassifyParameterABI()

for (unsigned i = 0; i < numSegmentsToCompare; i++)
{
const ABIPassingSegment& expected = abiInfo.Segments[i];
const ABIPassingSegment& expected = abiInfo.Segment(i);
regNumber reg = REG_NA;
if (i == 0)
{
Expand Down Expand Up @@ -1934,19 +1934,19 @@ void Compiler::lvaClassifyParameterABI()

if (lvaIsImplicitByRefLocal(lclNum))
{
assert((abiInfo.NumSegments == 1) && (abiInfo.Segments[0].Size == TARGET_POINTER_SIZE));
assert((abiInfo.NumSegments == 1) && (abiInfo.Segment(0).Size == TARGET_POINTER_SIZE));
}
else
{
for (unsigned i = 0; i < abiInfo.NumSegments; i++)
{
const ABIPassingSegment& segment = abiInfo.Segments[i];
const ABIPassingSegment& segment = abiInfo.Segment(i);
assert(segment.Size > 0);
assert(segment.Offset + segment.Size <= lvaLclExactSize(lclNum));

if (i > 0)
{
assert(segment.Offset > abiInfo.Segments[i - 1].Offset);
assert(segment.Offset > abiInfo.Segment(i - 1).Offset);
}

for (unsigned j = 0; j < abiInfo.NumSegments; j++)
Expand All @@ -1956,7 +1956,7 @@ void Compiler::lvaClassifyParameterABI()
continue;
}

const ABIPassingSegment& otherSegment = abiInfo.Segments[j];
const ABIPassingSegment& otherSegment = abiInfo.Segment(j);
assert((segment.Offset + segment.Size <= otherSegment.Offset) ||
(segment.Offset >= otherSegment.Offset + otherSegment.Size));
}
Expand Down Expand Up @@ -6054,7 +6054,7 @@ bool Compiler::lvaGetRelativeOffsetToCallerAllocatedSpaceForParameter(unsigned l

for (unsigned i = 0; i < abiInfo.NumSegments; i++)
{
const ABIPassingSegment& segment = abiInfo.Segments[i];
const ABIPassingSegment& segment = abiInfo.Segment(i);
if (!segment.IsPassedOnStack())
{
#if defined(WINDOWS_AMD64_ABI)
Expand Down
2 changes: 1 addition & 1 deletion src/coreclr/jit/lsrabuild.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2302,7 +2302,7 @@ void LinearScan::buildIntervals()
const ABIPassingInformation& abiInfo = compiler->lvaGetParameterABIInfo(lclNum);
for (unsigned i = 0; i < abiInfo.NumSegments; i++)
{
const ABIPassingSegment& seg = abiInfo.Segments[i];
const ABIPassingSegment& seg = abiInfo.Segment(i);
if (seg.IsPassedInRegister())
{
RegState* regState = genIsValidFloatReg(seg.GetRegister()) ? floatRegState : intRegState;
Expand Down
4 changes: 2 additions & 2 deletions src/coreclr/jit/morph.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4598,8 +4598,8 @@ GenTree* Compiler::fgMorphExpandStackArgForVarArgs(GenTreeLclVarCommon* lclNode)
assert(abiInfo.HasExactlyOneStackSegment());

GenTree* argsBaseAddr = gtNewLclvNode(lvaVarargsBaseOfStkArgs, TYP_I_IMPL);
ssize_t offset = (ssize_t)abiInfo.Segments[0].GetStackOffset() - lclNode->GetLclOffs();
assert(abiInfo.Segments[0].GetStackOffset() ==
ssize_t offset = (ssize_t)abiInfo.Segment(0).GetStackOffset() - lclNode->GetLclOffs();
assert(abiInfo.Segment(0).GetStackOffset() ==
(varDsc->GetStackOffset() - codeGen->intRegState.rsCalleeRegArgCount * REGSIZE_BYTES));
GenTree* offsetNode = gtNewIconNode(offset, TYP_I_IMPL);
GenTree* argAddr = gtNewOperNode(GT_SUB, TYP_I_IMPL, argsBaseAddr, offsetNode);
Expand Down
Loading

0 comments on commit 0498939

Please sign in to comment.