Skip to content

Commit 5ed292c

Browse files
committed
[MERGE #4727 @sigatrev] Array profile data usage optimzation
Merge pull request #4727 from sigatrev:arrayOpts This commit changes usage of array profile data in an attempt to better handle certain cases (such as inlined common functions) by using the most specialized (int>float>var) array type profiled when merging profile data, unless the instruction has previous bailed out on NotNativeArray, in which case it will use the least specialized type to avoid infinite bailouts. This also fixes an infinite bailout in Ares-6 ML benchmark which was introduced by #3169
2 parents 74462bc + 395a879 commit 5ed292c

File tree

11 files changed

+107
-29
lines changed

11 files changed

+107
-29
lines changed

lib/Backend/BailOut.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1464,6 +1464,11 @@ BailOutRecord::BailOutHelper(Js::JavascriptCallStackLayout * layout, Js::ScriptF
14641464
{
14651465
newInstance->OrFlags(Js::InterpreterStackFrameFlags_ProcessingBailOutOnArrayAccessHelperCall);
14661466
}
1467+
else if (bailOutKind == IR::BailOutOnNotNativeArray)
1468+
{
1469+
newInstance->OrFlags(Js::InterpreterStackFrameFlags_ProcessingBailOutOnArraySpecialization);
1470+
}
1471+
14671472
if (isInlinee)
14681473
{
14691474
newInstance->OrFlags(Js::InterpreterStackFrameFlags_FromBailOutInInlinee);

lib/Backend/GlobOpt.cpp

Lines changed: 28 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -3406,20 +3406,22 @@ GlobOpt::OptSrc(IR::Opnd *opnd, IR::Instr * *pInstr, Value **indirIndexValRef, I
34063406
{
34073407
ValueType valueType(val->GetValueInfo()->Type());
34083408

3409-
// This block uses local profiling data to optimize the case of a native array being passed to a function that fills it with other types. When the function is inlined
3410-
// into different call paths which use different types this can cause a perf hit by performing unnecessary array conversions, so only perform this optimization when
3411-
// the function is not inlined.
3412-
if (valueType.IsLikelyNativeArray() && !valueType.IsObject() && instr->IsProfiledInstr() && !instr->m_func->IsInlined())
3409+
// This block uses per-instruction profile information on array types to optimize using the best available profile
3410+
// information and to prevent infinite bailouts by ensuring array type information is updated on bailouts.
3411+
if (valueType.IsLikelyArray() && !valueType.IsObject() && instr->IsProfiledInstr())
34133412
{
34143413
// See if we have profile data for the array type
34153414
IR::ProfiledInstr *const profiledInstr = instr->AsProfiledInstr();
34163415
ValueType profiledArrayType;
3416+
3417+
bool useAggressiveSpecialization = true;
34173418
switch(instr->m_opcode)
34183419
{
34193420
case Js::OpCode::LdElemI_A:
34203421
if(instr->GetSrc1()->IsIndirOpnd() && opnd == instr->GetSrc1()->AsIndirOpnd()->GetBaseOpnd())
34213422
{
34223423
profiledArrayType = profiledInstr->u.ldElemInfo->GetArrayType();
3424+
useAggressiveSpecialization = !profiledInstr->u.ldElemInfo->IsAggressiveSpecializationDisabled();
34233425
}
34243426
break;
34253427

@@ -3429,23 +3431,36 @@ GlobOpt::OptSrc(IR::Opnd *opnd, IR::Instr * *pInstr, Value **indirIndexValRef, I
34293431
if(instr->GetDst()->IsIndirOpnd() && opnd == instr->GetDst()->AsIndirOpnd()->GetBaseOpnd())
34303432
{
34313433
profiledArrayType = profiledInstr->u.stElemInfo->GetArrayType();
3434+
useAggressiveSpecialization = !profiledInstr->u.stElemInfo->IsAggressiveSpecializationDisabled();
34323435
}
34333436
break;
34343437

34353438
case Js::OpCode::LdLen_A:
34363439
if(instr->GetSrc1()->IsRegOpnd() && opnd == instr->GetSrc1())
34373440
{
34383441
profiledArrayType = profiledInstr->u.LdLenInfo().GetArrayType();
3442+
useAggressiveSpecialization = !profiledInstr->u.LdLenInfo().IsAggressiveSpecializationDisabled();
34393443
}
34403444
break;
34413445
}
3442-
if(profiledArrayType.IsLikelyObject() &&
3443-
profiledArrayType.GetObjectType() == valueType.GetObjectType() &&
3444-
(profiledArrayType.HasVarElements() || (valueType.HasIntElements() && profiledArrayType.HasFloatElements())))
3446+
3447+
if (profiledArrayType.IsLikelyObject() && profiledArrayType.GetObjectType() == valueType.GetObjectType())
34453448
{
3446-
// Merge array type we pulled from profile with type propagated by dataflow.
3447-
valueType = valueType.Merge(profiledArrayType).SetHasNoMissingValues(valueType.HasNoMissingValues());
3448-
ChangeValueType(this->currentBlock, CurrentBlockData()->FindValue(opnd->AsRegOpnd()->m_sym), valueType, false);
3449+
// Ideally we want to use the most specialized type seen by this path, but when that causes bailouts use the least specialized type instead.
3450+
if (useAggressiveSpecialization && !valueType.IsLikelyNativeIntArray() &&
3451+
(profiledArrayType.HasIntElements() || (valueType.HasVarElements() && profiledArrayType.HasFloatElements())))
3452+
{
3453+
// use the more specialized type profiled by the instruction.
3454+
valueType = profiledArrayType.SetHasNoMissingValues(valueType.HasNoMissingValues());
3455+
ChangeValueType(this->currentBlock, CurrentBlockData()->FindValue(opnd->AsRegOpnd()->m_sym), valueType, false);
3456+
}
3457+
else if (!useAggressiveSpecialization && valueType.IsLikelyNativeArray() &&
3458+
(profiledArrayType.HasVarElements() || (valueType.HasIntElements() && profiledArrayType.HasFloatElements())))
3459+
{
3460+
// Merge array type we pulled from profile with type propagated by dataflow.
3461+
valueType = valueType.Merge(profiledArrayType).SetHasNoMissingValues(valueType.HasNoMissingValues());
3462+
ChangeValueType(this->currentBlock, CurrentBlockData()->FindValue(opnd->AsRegOpnd()->m_sym), valueType, false);
3463+
}
34493464
}
34503465
}
34513466

@@ -13391,11 +13406,9 @@ GlobOpt::OptArraySrc(IR::Instr * *const instrRef)
1339113406
if (!baseValueInLoopLandingPad->GetValueInfo()->CanMergeToSpecificObjectType())
1339213407
{
1339313408
ValueType landingPadValueType = baseValueInLoopLandingPad->GetValueInfo()->Type();
13394-
Assert(landingPadValueType.IsSimilar(baseValueType) ||
13395-
(
13396-
landingPadValueType.IsLikelyNativeArray() &&
13397-
landingPadValueType.Merge(baseValueType).IsSimilar(baseValueType)
13398-
)
13409+
Assert(landingPadValueType.IsSimilar(baseValueType)
13410+
|| (landingPadValueType.IsLikelyNativeArray() && landingPadValueType.Merge(baseValueType).IsSimilar(baseValueType))
13411+
|| (baseValueType.IsLikelyNativeArray() && baseValueType.Merge(landingPadValueType).IsSimilar(landingPadValueType))
1339913412
);
1340013413
}
1340113414
#endif

lib/Backend/IRBuilder.cpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4482,9 +4482,10 @@ IRBuilder::BuildProfiledElementCP(Js::OpCode newOpcode, uint32 offset, Js::RegSl
44824482

44834483
ValueType arrayType = ValueType::Uninitialized;
44844484

4485+
const Js::LdLenInfo * ldLenInfo = nullptr;
44854486
if (m_func->HasProfileInfo())
44864487
{
4487-
const Js::LdLenInfo * ldLenInfo = m_func->GetReadOnlyProfileInfo()->GetLdLenInfo(profileId);
4488+
ldLenInfo = m_func->GetReadOnlyProfileInfo()->GetLdLenInfo(profileId);
44884489
arrayType = (ldLenInfo->GetArrayType());
44894490
if (arrayType.IsLikelyNativeArray() &&
44904491
(
@@ -4522,7 +4523,8 @@ IRBuilder::BuildProfiledElementCP(Js::OpCode newOpcode, uint32 offset, Js::RegSl
45224523
IR::ProfiledInstr * profiledInstr = IR::ProfiledInstr::New(newOpcode, dstOpnd, fieldSymOpnd, m_func);
45234524
instr = profiledInstr;
45244525
profiledInstr->u.FldInfo() = *(m_func->GetReadOnlyProfileInfo()->GetFldInfo(inlineCacheIndex));
4525-
profiledInstr->u.LdLenInfo().GetArrayType() = arrayType;
4526+
profiledInstr->u.LdLenInfo() = *ldLenInfo;
4527+
profiledInstr->u.LdLenInfo().arrayType = arrayType;
45264528
wasNotProfiled = !profiledInstr->u.FldInfo().WasLdFldProfiled();
45274529
dstOpnd->SetValueType(instr->AsProfiledInstr()->u.FldInfo().valueType);
45284530
#if ENABLE_DEBUG_CONFIG_OPTIONS

lib/Backend/JITTimeProfileInfo.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ JITTimeProfileInfo::InitializeJITProfileData(
2525
return;
2626
}
2727

28+
CompileAssert(sizeof(LdLenIDL) == sizeof(Js::LdLenInfo));
2829
CompileAssert(sizeof(LdElemIDL) == sizeof(Js::LdElemInfo));
2930
CompileAssert(sizeof(StElemIDL) == sizeof(Js::StElemInfo));
3031

lib/Backend/Lower.cpp

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8696,11 +8696,13 @@ void Lowerer::LowerProfiledLdElemI(IR::JitProfilingInstr *const instr)
86968696
const Var varIndex,
86978697
FunctionBody *const functionBody,
86988698
const ProfileId profileId,
8699-
bool didArrayAccessHelperCall)
8699+
bool didArrayAccessHelperCall,
8700+
bool bailedOutOnArraySpecialization)
87008701
*/
87018702

87028703
Func *const func = instr->m_func;
87038704

8705+
m_lowererMD.LoadHelperArgument(instr, IR::IntConstOpnd::New(false, TyInt8, func));
87048706
m_lowererMD.LoadHelperArgument(instr, IR::IntConstOpnd::New(false, TyInt8, func));
87058707
m_lowererMD.LoadHelperArgument(instr, IR::Opnd::CreateProfileIdOpnd(instr->profileId, func));
87068708
m_lowererMD.LoadHelperArgument(instr, CreateFunctionBodyOpnd(func));
@@ -8731,7 +8733,8 @@ void Lowerer::LowerProfiledStElemI(IR::JitProfilingInstr *const instr, const Js:
87318733
FunctionBody *const functionBody,
87328734
const ProfileId profileId,
87338735
const PropertyOperationFlags flags,
8734-
bool didArrayAccessHelperCall)
8736+
bool didArrayAccessHelperCall,
8737+
bool bailedOutOnArraySpecialization)
87358738
*/
87368739

87378740
Func *const func = instr->m_func;
@@ -8745,6 +8748,7 @@ void Lowerer::LowerProfiledStElemI(IR::JitProfilingInstr *const instr, const Js:
87458748
{
87468749
helper = IR::HelperProfiledStElem;
87478750
m_lowererMD.LoadHelperArgument(instr, IR::IntConstOpnd::New(false, TyInt8, func));
8751+
m_lowererMD.LoadHelperArgument(instr, IR::IntConstOpnd::New(false, TyInt8, func));
87488752
m_lowererMD.LoadHelperArgument(instr, IR::IntConstOpnd::New(flags, TyInt32, func, true));
87498753
}
87508754
m_lowererMD.LoadHelperArgument(instr, IR::Opnd::CreateProfileIdOpnd(instr->profileId, func));

lib/JITIDL/JITTypes.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,8 @@ typedef struct ArrayCallSiteIDL
243243
typedef struct LdLenIDL
244244
{
245245
unsigned short arrayType;
246+
byte bits;
247+
IDL_PAD1(0)
246248
} LdLenIDL;
247249

248250
typedef struct LdElemIDL

lib/Runtime/Language/DynamicProfileInfo.h

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -179,10 +179,23 @@ namespace Js
179179

180180
struct LdLenInfo
181181
{
182-
typedef struct { ValueType::TSize f1; } TSize;
182+
typedef struct { ValueType::TSize f1; byte f2; } TSize;
183183

184184
ValueType arrayType;
185185

186+
union
187+
{
188+
struct
189+
{
190+
bool disableAggressiveSpecialization : 1;
191+
};
192+
byte bits;
193+
};
194+
195+
LdLenInfo() : bits(0)
196+
{
197+
}
198+
186199
void Merge(const LdLenInfo & other)
187200
{
188201
arrayType = arrayType.Merge(other.arrayType);
@@ -192,6 +205,11 @@ namespace Js
192205
{
193206
return arrayType;
194207
}
208+
209+
bool IsAggressiveSpecializationDisabled() const
210+
{
211+
return disableAggressiveSpecialization;
212+
}
195213
};
196214
CompileAssert(sizeof(LdLenInfo::TSize) == sizeof(LdLenInfo));
197215

@@ -206,6 +224,7 @@ namespace Js
206224
{
207225
bool wasProfiled : 1;
208226
bool neededHelperCall : 1;
227+
bool disableAggressiveSpecialization : 1;
209228
};
210229
byte bits;
211230
};
@@ -241,6 +260,11 @@ namespace Js
241260
{
242261
return neededHelperCall;
243262
}
263+
264+
bool IsAggressiveSpecializationDisabled() const
265+
{
266+
return disableAggressiveSpecialization;
267+
}
244268
};
245269

246270
struct StElemInfo
@@ -257,6 +281,7 @@ namespace Js
257281
bool neededHelperCall : 1;
258282
bool storedOutsideHeadSegmentBounds : 1;
259283
bool storedOutsideArrayBounds : 1;
284+
bool disableAggressiveSpecialization : 1;
260285
};
261286
byte bits;
262287
};
@@ -306,6 +331,11 @@ namespace Js
306331
{
307332
return storedOutsideArrayBounds;
308333
}
334+
335+
bool IsAggressiveSpecializationDisabled() const
336+
{
337+
return disableAggressiveSpecialization;
338+
}
309339
};
310340

311341
struct ArrayCallSiteInfo

lib/Runtime/Language/InterpreterStackFrame.cpp

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4977,9 +4977,10 @@ namespace Js
49774977
GetReg(playout->Element),
49784978
m_functionBody,
49794979
playout->profileId,
4980-
this->TestFlags(InterpreterStackFrameFlags_ProcessingBailOutOnArrayAccessHelperCall)));
4980+
this->TestFlags(InterpreterStackFrameFlags_ProcessingBailOutOnArrayAccessHelperCall),
4981+
this->TestFlags(InterpreterStackFrameFlags_ProcessingBailOutOnArraySpecialization)));
49814982

4982-
this->ClearFlags(InterpreterStackFrameFlags_ProcessingBailOutOnArrayAccessHelperCall);
4983+
this->ClearFlags(InterpreterStackFrameFlags_ProcessingBailOutOnArrayAccessHelperCall | InterpreterStackFrameFlags_ProcessingBailOutOnArraySpecialization);
49834984

49844985
threadContext->CheckAndResetImplicitCallAccessorFlag();
49854986
threadContext->AddImplicitCallFlags(savedImplicitCallFlags);
@@ -5077,9 +5078,10 @@ namespace Js
50775078
m_functionBody,
50785079
playout->profileId,
50795080
flags,
5080-
this->TestFlags(InterpreterStackFrameFlags_ProcessingBailOutOnArrayAccessHelperCall));
5081+
this->TestFlags(InterpreterStackFrameFlags_ProcessingBailOutOnArrayAccessHelperCall),
5082+
this->TestFlags(InterpreterStackFrameFlags_ProcessingBailOutOnArraySpecialization));
50815083

5082-
this->ClearFlags(InterpreterStackFrameFlags_ProcessingBailOutOnArrayAccessHelperCall);
5084+
this->ClearFlags(InterpreterStackFrameFlags_ProcessingBailOutOnArrayAccessHelperCall | InterpreterStackFrameFlags_ProcessingBailOutOnArraySpecialization);
50835085

50845086
threadContext->CheckAndResetImplicitCallAccessorFlag();
50855087
threadContext->AddImplicitCallFlags(savedImplicitCallFlags);
@@ -8412,6 +8414,12 @@ const byte * InterpreterStackFrame::OP_ProfiledLoopBodyStart(uint loopId)
84128414
ldLenInfo.arrayType = ValueType::Uninitialized.Merge(instance);
84138415
profileData->RecordLengthLoad(functionBody, playout->profileId, ldLenInfo);
84148416

8417+
if (this->TestFlags(InterpreterStackFrameFlags_ProcessingBailOutOnArraySpecialization))
8418+
{
8419+
ldLenInfo.disableAggressiveSpecialization = true;
8420+
this->ClearFlags(InterpreterStackFrameFlags_ProcessingBailOutOnArraySpecialization);
8421+
}
8422+
84158423
ThreadContext* threadContext = this->GetScriptContext()->GetThreadContext();
84168424
ImplicitCallFlags savedImplicitCallFlags = threadContext->GetImplicitCallFlags();
84178425
threadContext->ClearImplicitCallFlags();

lib/Runtime/Language/InterpreterStackFrame.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ namespace Js
2828
InterpreterStackFrameFlags_ProcessingBailOutOnArrayAccessHelperCall = 0x10,
2929
InterpreterStackFrameFlags_ProcessingBailOutFromEHCode = 0x20,
3030
InterpreterStackFrameFlags_FromBailOutInInlinee = 0x40,
31+
InterpreterStackFrameFlags_ProcessingBailOutOnArraySpecialization = 0x80,
3132
InterpreterStackFrameFlags_All = 0xFFFF,
3233
};
3334

lib/Runtime/Language/ProfilingHelpers.cpp

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@ namespace Js
1212
const Var varIndex,
1313
FunctionBody *const functionBody,
1414
const ProfileId profileId,
15-
bool didArrayAccessHelperCall)
15+
bool didArrayAccessHelperCall,
16+
bool bailedOutOnArraySpecialization)
1617
{
1718
Assert(base);
1819
Assert(varIndex);
@@ -21,6 +22,11 @@ namespace Js
2122

2223
LdElemInfo ldElemInfo;
2324

25+
if (bailedOutOnArraySpecialization)
26+
{
27+
ldElemInfo.disableAggressiveSpecialization = true;
28+
}
29+
2430
// Only enable fast path if the javascript array is not cross site
2531
#if ENABLE_COPYONACCESS_ARRAY
2632
JavascriptLibrary::CheckAndConvertCopyOnAccessNativeIntArray<Var>(base);
@@ -197,7 +203,7 @@ namespace Js
197203
FunctionBody *const functionBody,
198204
const ProfileId profileId)
199205
{
200-
ProfiledStElem(base, varIndex, value, functionBody, profileId, PropertyOperation_None, false);
206+
ProfiledStElem(base, varIndex, value, functionBody, profileId, PropertyOperation_None, false, false);
201207
}
202208

203209
void ProfilingHelpers::ProfiledStElem(
@@ -207,7 +213,8 @@ namespace Js
207213
FunctionBody *const functionBody,
208214
const ProfileId profileId,
209215
const PropertyOperationFlags flags,
210-
bool didArrayAccessHelperCall)
216+
bool didArrayAccessHelperCall,
217+
bool bailedOutOnArraySpecialization)
211218
{
212219
Assert(base);
213220
Assert(varIndex);
@@ -217,6 +224,11 @@ namespace Js
217224

218225
StElemInfo stElemInfo;
219226

227+
if (bailedOutOnArraySpecialization)
228+
{
229+
stElemInfo.disableAggressiveSpecialization = true;
230+
}
231+
220232
// Only enable fast path if the javascript array is not cross site
221233
const bool isJsArray = !TaggedNumber::Is(base) && VirtualTableInfo<JavascriptArray>::HasVirtualTable(base);
222234
ScriptContext *const scriptContext = functionBody->GetScriptContext();

0 commit comments

Comments
 (0)