From 0087864d9b3b548118191b3f352a0eb3dc742e5f Mon Sep 17 00:00:00 2001 From: Mike Danes Date: Sun, 2 Dec 2018 15:47:50 +0200 Subject: [PATCH] Pull struct type info out of GenTreeObj --- src/jit/codegenarm.cpp | 52 ++-- src/jit/codegenarm64.cpp | 67 +++--- src/jit/codegenarmarch.cpp | 71 +++--- src/jit/codegencommon.cpp | 25 +- src/jit/codegenlinear.cpp | 1 - src/jit/codegenxarch.cpp | 124 +++++----- src/jit/compiler.cpp | 445 +++++++++++++++++++++++++++++------ src/jit/compiler.h | 41 ++-- src/jit/compiler.hpp | 6 +- src/jit/compmemkind.h | 1 + src/jit/gcencode.cpp | 30 ++- src/jit/gcinfo.cpp | 14 +- src/jit/gentree.cpp | 111 ++++----- src/jit/gentree.h | 293 ++++++++++++++--------- src/jit/gschecks.cpp | 2 +- src/jit/hwintrinsicArm64.cpp | 4 +- src/jit/hwintrinsicxarch.cpp | 4 +- src/jit/importer.cpp | 56 +---- src/jit/lclvars.cpp | 79 +------ src/jit/lower.cpp | 39 ++- src/jit/lowerarmarch.cpp | 14 +- src/jit/lowerxarch.cpp | 31 +-- src/jit/lsraarmarch.cpp | 2 +- src/jit/lsrabuild.cpp | 4 +- src/jit/lsraxarch.cpp | 9 +- src/jit/morph.cpp | 108 +++++---- src/jit/rationalize.cpp | 40 ++-- src/jit/simd.cpp | 37 +-- 28 files changed, 950 insertions(+), 760 deletions(-) diff --git a/src/jit/codegenarm.cpp b/src/jit/codegenarm.cpp index 06732fb63872..55e932b7b5ce 100644 --- a/src/jit/codegenarm.cpp +++ b/src/jit/codegenarm.cpp @@ -702,7 +702,7 @@ void CodeGen::genCodeForCpObj(GenTreeObj* cpObjNode) // This GenTree node has data about GC pointers, this means we're dealing // with CpObj. - assert(cpObjNode->gtGcPtrCount > 0); + assert(cpObjNode->gtLayout->HasGCPtr()); #endif // DEBUG // Consume the operands and get them into the right registers. @@ -721,29 +721,18 @@ void CodeGen::genCodeForCpObj(GenTreeObj* cpObjNode) instGen_MemoryBarrier(); } - unsigned slots = cpObjNode->gtSlots; + unsigned slots = cpObjNode->gtLayout->GetSlotCount(); emitter* emit = getEmitter(); - BYTE* gcPtrs = cpObjNode->gtGcPtrs; + StructTypeLayout* layout = cpObjNode->gtLayout; // If we can prove it's on the stack we don't need to use the write barrier. - emitAttr attr = EA_PTRSIZE; if (dstOnStack) { for (unsigned i = 0; i < slots; ++i) { - if (gcPtrs[i] == GCT_GCREF) - { - attr = EA_GCREF; - } - else if (gcPtrs[i] == GCT_BYREF) - { - attr = EA_BYREF; - } - else - { - attr = EA_PTRSIZE; - } + var_types type = layout->GetGCPtrType(i); + emitAttr attr = emitTypeSize(type); emit->emitIns_R_R_I(INS_ldr, attr, tmpReg, REG_WRITE_BARRIER_SRC_BYREF, TARGET_POINTER_SIZE, INS_FLAGS_DONT_CARE, INS_OPTS_LDST_POST_INC); @@ -753,30 +742,21 @@ void CodeGen::genCodeForCpObj(GenTreeObj* cpObjNode) } else { - unsigned gcPtrCount = cpObjNode->gtGcPtrCount; - - unsigned i = 0; - while (i < slots) + for (unsigned i = 0; i < slots; ++i) { - switch (gcPtrs[i]) + if (!layout->IsGCPtr(i)) + { + emit->emitIns_R_R_I(INS_ldr, EA_PTRSIZE, tmpReg, REG_WRITE_BARRIER_SRC_BYREF, TARGET_POINTER_SIZE, + INS_FLAGS_DONT_CARE, INS_OPTS_LDST_POST_INC); + emit->emitIns_R_R_I(INS_str, EA_PTRSIZE, tmpReg, REG_WRITE_BARRIER_DST_BYREF, TARGET_POINTER_SIZE, + INS_FLAGS_DONT_CARE, INS_OPTS_LDST_POST_INC); + } + else { - case TYPE_GC_NONE: - emit->emitIns_R_R_I(INS_ldr, attr, tmpReg, REG_WRITE_BARRIER_SRC_BYREF, TARGET_POINTER_SIZE, - INS_FLAGS_DONT_CARE, INS_OPTS_LDST_POST_INC); - emit->emitIns_R_R_I(INS_str, attr, tmpReg, REG_WRITE_BARRIER_DST_BYREF, TARGET_POINTER_SIZE, - INS_FLAGS_DONT_CARE, INS_OPTS_LDST_POST_INC); - break; - - default: - // In the case of a GC-Pointer we'll call the ByRef write barrier helper - genEmitHelperCall(CORINFO_HELP_ASSIGN_BYREF, 0, EA_PTRSIZE); - - gcPtrCount--; - break; + // In the case of a GC-Pointer we'll call the ByRef write barrier helper + genEmitHelperCall(CORINFO_HELP_ASSIGN_BYREF, 0, EA_PTRSIZE); } - ++i; } - assert(gcPtrCount == 0); } if (cpObjNode->gtFlags & GTF_BLK_VOLATILE) diff --git a/src/jit/codegenarm64.cpp b/src/jit/codegenarm64.cpp index 9d1959e85083..c0c39b2dfa0d 100644 --- a/src/jit/codegenarm64.cpp +++ b/src/jit/codegenarm64.cpp @@ -2630,7 +2630,7 @@ void CodeGen::genCodeForCpObj(GenTreeObj* cpObjNode) // This GenTree node has data about GC pointers, this means we're dealing // with CpObj. - assert(cpObjNode->gtGcPtrCount > 0); + assert(cpObjNode->gtLayout->HasGCPtr()); #endif // DEBUG // Consume the operands and get them into the right registers. @@ -2639,7 +2639,7 @@ void CodeGen::genCodeForCpObj(GenTreeObj* cpObjNode) gcInfo.gcMarkRegPtrVal(REG_WRITE_BARRIER_SRC_BYREF, srcAddrType); gcInfo.gcMarkRegPtrVal(REG_WRITE_BARRIER_DST_BYREF, dstAddr->TypeGet()); - unsigned slots = cpObjNode->gtSlots; + unsigned slots = cpObjNode->gtLayout->GetSlotCount(); // Temp register(s) used to perform the sequence of loads and stores. regNumber tmpReg = cpObjNode->ExtractTempReg(); @@ -2666,7 +2666,7 @@ void CodeGen::genCodeForCpObj(GenTreeObj* cpObjNode) emitter* emit = getEmitter(); - BYTE* gcPtrs = cpObjNode->gtGcPtrs; + StructTypeLayout* layout = cpObjNode->gtLayout; // If we can prove it's on the stack we don't need to use the write barrier. if (dstOnStack) @@ -2675,8 +2675,8 @@ void CodeGen::genCodeForCpObj(GenTreeObj* cpObjNode) // Check if two or more remaining slots and use a ldp/stp sequence while (i < slots - 1) { - emitAttr attr0 = emitTypeSize(compiler->getJitGCType(gcPtrs[i + 0])); - emitAttr attr1 = emitTypeSize(compiler->getJitGCType(gcPtrs[i + 1])); + emitAttr attr0 = emitTypeSize(layout->GetGCPtrType(i + 0)); + emitAttr attr1 = emitTypeSize(layout->GetGCPtrType(i + 1)); emit->emitIns_R_R_R_I(INS_ldp, attr0, tmpReg, tmpReg2, REG_WRITE_BARRIER_SRC_BYREF, 2 * TARGET_POINTER_SIZE, INS_OPTS_POST_INDEX, attr1); @@ -2688,7 +2688,7 @@ void CodeGen::genCodeForCpObj(GenTreeObj* cpObjNode) // Use a ldr/str sequence for the last remainder if (i < slots) { - emitAttr attr0 = emitTypeSize(compiler->getJitGCType(gcPtrs[i + 0])); + emitAttr attr0 = emitTypeSize(layout->GetGCPtrType(i + 0)); emit->emitIns_R_R_I(INS_ldr, attr0, tmpReg, REG_WRITE_BARRIER_SRC_BYREF, TARGET_POINTER_SIZE, INS_OPTS_POST_INDEX); @@ -2698,42 +2698,33 @@ void CodeGen::genCodeForCpObj(GenTreeObj* cpObjNode) } else { - unsigned gcPtrCount = cpObjNode->gtGcPtrCount; - - unsigned i = 0; - while (i < slots) + for (unsigned i = 0; i < slots; ++i) { - switch (gcPtrs[i]) + if (!layout->IsGCPtr(i)) { - case TYPE_GC_NONE: - // Check if the next slot's type is also TYP_GC_NONE and use ldp/stp - if ((i + 1 < slots) && (gcPtrs[i + 1] == TYPE_GC_NONE)) - { - emit->emitIns_R_R_R_I(INS_ldp, EA_8BYTE, tmpReg, tmpReg2, REG_WRITE_BARRIER_SRC_BYREF, - 2 * TARGET_POINTER_SIZE, INS_OPTS_POST_INDEX); - emit->emitIns_R_R_R_I(INS_stp, EA_8BYTE, tmpReg, tmpReg2, REG_WRITE_BARRIER_DST_BYREF, - 2 * TARGET_POINTER_SIZE, INS_OPTS_POST_INDEX); - ++i; // extra increment of i, since we are copying two items - } - else - { - emit->emitIns_R_R_I(INS_ldr, EA_8BYTE, tmpReg, REG_WRITE_BARRIER_SRC_BYREF, TARGET_POINTER_SIZE, - INS_OPTS_POST_INDEX); - emit->emitIns_R_R_I(INS_str, EA_8BYTE, tmpReg, REG_WRITE_BARRIER_DST_BYREF, TARGET_POINTER_SIZE, - INS_OPTS_POST_INDEX); - } - break; - - default: - // In the case of a GC-Pointer we'll call the ByRef write barrier helper - genEmitHelperCall(CORINFO_HELP_ASSIGN_BYREF, 0, EA_PTRSIZE); - - gcPtrCount--; - break; + // Check if the next slot's type is also TYP_GC_NONE and use ldp/stp + if ((i + 1 < slots) && !layout->IsGCPtr(i + 1)) + { + emit->emitIns_R_R_R_I(INS_ldp, EA_8BYTE, tmpReg, tmpReg2, REG_WRITE_BARRIER_SRC_BYREF, + 2 * TARGET_POINTER_SIZE, INS_OPTS_POST_INDEX); + emit->emitIns_R_R_R_I(INS_stp, EA_8BYTE, tmpReg, tmpReg2, REG_WRITE_BARRIER_DST_BYREF, + 2 * TARGET_POINTER_SIZE, INS_OPTS_POST_INDEX); + ++i; // extra increment of i, since we are copying two items + } + else + { + emit->emitIns_R_R_I(INS_ldr, EA_8BYTE, tmpReg, REG_WRITE_BARRIER_SRC_BYREF, TARGET_POINTER_SIZE, + INS_OPTS_POST_INDEX); + emit->emitIns_R_R_I(INS_str, EA_8BYTE, tmpReg, REG_WRITE_BARRIER_DST_BYREF, TARGET_POINTER_SIZE, + INS_OPTS_POST_INDEX); + } + } + else + { + // In the case of a GC-Pointer we'll call the ByRef write barrier helper + genEmitHelperCall(CORINFO_HELP_ASSIGN_BYREF, 0, EA_PTRSIZE); } - ++i; } - assert(gcPtrCount == 0); } if (cpObjNode->gtFlags & GTF_BLK_VOLATILE) diff --git a/src/jit/codegenarmarch.cpp b/src/jit/codegenarmarch.cpp index 5e50e927e428..4ed4607122fb 100644 --- a/src/jit/codegenarmarch.cpp +++ b/src/jit/codegenarmarch.cpp @@ -710,22 +710,14 @@ void CodeGen::genPutArgStk(GenTreePutArgStk* treeNode) // the xor ensures that only one of the two is setup, not both assert((varNode != nullptr) ^ (addrNode != nullptr)); - BYTE gcPtrArray[MAX_ARG_REG_COUNT] = {}; // TYPE_GC_NONE = 0 - BYTE* gcPtrs = gcPtrArray; - - unsigned gcPtrCount; // The count of GC pointers in the struct - int structSize; - bool isHfa; + StructTypeLayout* layout; + int structSize; + bool isHfa; // This is the varNum for our load operations, // only used when we have a multireg struct with a LclVar source unsigned varNumInp = BAD_VAR_NUM; -#ifdef _TARGET_ARM_ - // On ARM32, size of reference map can be larger than MAX_ARG_REG_COUNT - gcPtrs = treeNode->gtGcPtrs; - gcPtrCount = treeNode->gtNumberReferenceSlots; -#endif // Setup the structSize, isHFa, and gcPtrCount if (varNode != nullptr) { @@ -738,14 +730,17 @@ void CodeGen::genPutArgStk(GenTreePutArgStk* treeNode) assert(varDsc->lvType == TYP_STRUCT); assert(varDsc->lvOnFrame && !varDsc->lvRegister); - structSize = varDsc->lvSize(); // This yields the roundUp size, but that is fine - // as that is how much stack is allocated for this LclVar + // This yields the roundUp size, but that is fine + // as that is how much stack is allocated for this LclVar + structSize = varDsc->lvSize(); + isHfa = varDsc->lvIsHfa(); -#ifdef _TARGET_ARM64_ - gcPtrCount = varDsc->lvStructGcCount; - for (unsigned i = 0; i < gcPtrCount; ++i) - gcPtrs[i] = varDsc->lvGcLayout[i]; -#endif // _TARGET_ARM_ + +#ifdef _TARGET_ARM_ + layout = treeNode->gtLayout; +#else // _TARGET_ARM64_ + layout = varDsc->lvLayout; +#endif // _TARGET_ARM64_ } else // addrNode is used { @@ -765,20 +760,16 @@ void CodeGen::genPutArgStk(GenTreePutArgStk* treeNode) } #endif // _TARGET_ARM64_ - CORINFO_CLASS_HANDLE objClass = source->gtObj.gtClass; - - structSize = compiler->info.compCompHnd->getClassSize(objClass); - isHfa = compiler->IsHfa(objClass); -#ifdef _TARGET_ARM64_ - gcPtrCount = compiler->info.compCompHnd->getClassGClayout(objClass, &gcPtrs[0]); -#endif + layout = source->AsObj()->gtLayout; + structSize = layout->GetSize(); + isHfa = compiler->IsHfa(layout->GetClass()); } // If we have an HFA we can't have any GC pointers, // if not then the max size for the the struct is 16 bytes if (isHfa) { - noway_assert(gcPtrCount == 0); + noway_assert(!layout->HasGCPtr()); } #ifdef _TARGET_ARM64_ else @@ -800,8 +791,8 @@ void CodeGen::genPutArgStk(GenTreePutArgStk* treeNode) while (remainingSize >= 2 * TARGET_POINTER_SIZE) { - var_types type0 = compiler->getJitGCType(gcPtrs[nextIndex + 0]); - var_types type1 = compiler->getJitGCType(gcPtrs[nextIndex + 1]); + var_types type0 = layout->GetGCPtrType(nextIndex + 0); + var_types type1 = layout->GetGCPtrType(nextIndex + 1); if (varNode != nullptr) { @@ -836,7 +827,7 @@ void CodeGen::genPutArgStk(GenTreePutArgStk* treeNode) // str r2, [sp, #16] while (remainingSize >= TARGET_POINTER_SIZE) { - var_types type = compiler->getJitGCType(gcPtrs[nextIndex]); + var_types type = layout->GetGCPtrType(nextIndex); if (varNode != nullptr) { @@ -873,7 +864,7 @@ void CodeGen::genPutArgStk(GenTreePutArgStk* treeNode) { if (remainingSize >= TARGET_POINTER_SIZE) { - var_types nextType = compiler->getJitGCType(gcPtrs[nextIndex]); + var_types nextType = layout->GetGCPtrType(nextIndex); emitAttr nextAttr = emitTypeSize(nextType); remainingSize -= TARGET_POINTER_SIZE; @@ -906,7 +897,7 @@ void CodeGen::genPutArgStk(GenTreePutArgStk* treeNode) assert(varNode == nullptr); // the left over size is smaller than a pointer and thus can never be a GC type - assert(varTypeIsGC(compiler->getJitGCType(gcPtrs[nextIndex])) == false); + assert(!layout->IsGCPtr(nextIndex)); var_types loadType = TYP_UINT; if (loadSize == 1) @@ -1079,10 +1070,8 @@ void CodeGen::genPutArgSplit(GenTreePutArgSplit* treeNode) // the xor ensures that only one of the two is setup, not both assert((varNode != nullptr) ^ (addrNode != nullptr)); - // Setup the structSize, isHFa, and gcPtrCount - BYTE* gcPtrs = treeNode->gtGcPtrs; - unsigned gcPtrCount = treeNode->gtNumberReferenceSlots; // The count of GC pointers in the struct - int structSize = treeNode->getArgSize(); + StructTypeLayout* layout = treeNode->gtLayout; + int structSize = treeNode->getArgSize(); // This is the varNum for our load operations, // only used when we have a struct with a LclVar source @@ -1118,7 +1107,7 @@ void CodeGen::genPutArgSplit(GenTreePutArgSplit* treeNode) assert(baseReg != addrReg); // We don't split HFA struct - assert(!compiler->IsHfa(source->gtObj.gtClass)); + assert(!compiler->IsHfa(source->AsObj()->gtLayout->GetClass())); } // Put on stack first @@ -1130,7 +1119,7 @@ void CodeGen::genPutArgSplit(GenTreePutArgSplit* treeNode) assert(remainingSize % TARGET_POINTER_SIZE == 0); while (remainingSize > 0) { - var_types type = compiler->getJitGCType(gcPtrs[nextIndex]); + var_types type = layout->GetGCPtrType(nextIndex); if (varNode != nullptr) { @@ -2679,7 +2668,7 @@ void CodeGen::genJmpMethod(GenTree* jmp) { // Must be <= 16 bytes or else it wouldn't be passed in registers noway_assert(EA_SIZE_IN_BYTES(varDsc->lvSize()) <= MAX_PASS_MULTIREG_BYTES); - loadType = compiler->getJitGCType(varDsc->lvGcLayout[0]); + loadType = varDsc->lvLayout->GetGCPtrType(0); } else { @@ -2705,7 +2694,7 @@ void CodeGen::genJmpMethod(GenTree* jmp) // Restore the second register. argRegNext = genRegArgNext(argReg); - loadType = compiler->getJitGCType(varDsc->lvGcLayout[1]); + loadType = varDsc->lvLayout->GetGCPtrType(1); loadSize = emitActualTypeSize(loadType); getEmitter()->emitIns_R_S(ins_Load(loadType), loadSize, argRegNext, varNum, TARGET_POINTER_SIZE); @@ -2796,7 +2785,7 @@ void CodeGen::genJmpMethod(GenTree* jmp) for (unsigned ofs = 0; ofs < maxSize; ofs += REGSIZE_BYTES) { unsigned idx = ofs / REGSIZE_BYTES; - loadType = compiler->getJitGCType(varDsc->lvGcLayout[idx]); + loadType = varDsc->lvLayout->GetGCPtrType(idx); if (varDsc->lvRegNum != argReg) { @@ -3267,7 +3256,7 @@ void CodeGen::genCodeForStoreBlk(GenTreeBlk* blkOp) if (blkOp->OperIs(GT_STORE_OBJ) && blkOp->OperIsCopyBlkOp()) { - assert(blkOp->AsObj()->gtGcPtrCount != 0); + assert(blkOp->AsObj()->gtLayout->HasGCPtr()); genCodeForCpObj(blkOp->AsObj()); return; } diff --git a/src/jit/codegencommon.cpp b/src/jit/codegencommon.cpp index 55028e9e82e2..1cc2d2f573ed 100644 --- a/src/jit/codegencommon.cpp +++ b/src/jit/codegencommon.cpp @@ -1145,8 +1145,8 @@ unsigned CodeGenInterface::InferStructOpSizeAlign(GenTree* op, unsigned* alignme if (op->gtOper == GT_OBJ) { - CORINFO_CLASS_HANDLE clsHnd = op->AsObj()->gtClass; - opSize = compiler->info.compCompHnd->getClassSize(clsHnd); + CORINFO_CLASS_HANDLE clsHnd = op->AsObj()->gtLayout->GetClass(); + opSize = op->AsObj()->gtLayout->GetSize(); alignment = roundUp(compiler->info.compCompHnd->getClassAlignmentRequirement(clsHnd), TARGET_POINTER_SIZE); } else if (op->gtOper == GT_LCL_VAR) @@ -4551,8 +4551,7 @@ void CodeGen::genCheckUseBlockInit() continue; } - if (compiler->info.compInitMem || varTypeIsGC(varDsc->TypeGet()) || (varDsc->lvStructGcCount > 0) || - varDsc->lvMustInit) + if (compiler->info.compInitMem || compiler->lvaTypeIsGC(varNum) || varDsc->lvMustInit) { if (varDsc->lvTracked) { @@ -4603,8 +4602,7 @@ void CodeGen::genCheckUseBlockInit() #else // !FEATURE_SIMD if ((!varDsc->lvTracked || (varDsc->lvType == TYP_STRUCT)) && #endif // !FEATURE_SIMD - varDsc->lvOnFrame && - (!varDsc->lvIsTemp || varTypeIsGC(varDsc->TypeGet()) || (varDsc->lvStructGcCount > 0))) + varDsc->lvOnFrame && (!varDsc->lvIsTemp || compiler->lvaTypeIsGC(varNum))) { varDsc->lvMustInit = true; @@ -4639,7 +4637,14 @@ void CodeGen::genCheckUseBlockInit() if (varDsc->lvMustInit && varDsc->lvOnFrame) { - initStkLclCnt += varDsc->lvStructGcCount; + if (varDsc->TypeGet() == TYP_STRUCT) + { + initStkLclCnt += varDsc->lvLayout->GetGCPtrCount(); + } + else if (varTypeIsGC(varDsc->TypeGet())) + { + initStkLclCnt += 1; + } } if ((compiler->lvaLclSize(varNum) > (3 * TARGET_POINTER_SIZE)) && (largeGcStructs <= 4)) @@ -6360,12 +6365,12 @@ void CodeGen::genZeroInitFrame(int untrLclHi, int untrLclLo, regNumber initReg, (varDsc->lvExactSize >= TARGET_POINTER_SIZE)) { // We only initialize the GC variables in the TYP_STRUCT - const unsigned slots = (unsigned)compiler->lvaLclSize(varNum) / REGSIZE_BYTES; - const BYTE* gcPtrs = compiler->lvaGetGcLayout(varNum); + StructTypeLayout* layout = varDsc->lvLayout; + const unsigned slots = layout->GetSlotCount(); for (unsigned i = 0; i < slots; i++) { - if (gcPtrs[i] != TYPE_GC_NONE) + if (layout->IsGCPtr(i)) { getEmitter()->emitIns_S_R(ins_Store(TYP_I_IMPL), EA_PTRSIZE, genGetZeroReg(initReg, pInitRegZeroed), varNum, i * REGSIZE_BYTES); diff --git a/src/jit/codegenlinear.cpp b/src/jit/codegenlinear.cpp index 149bb3333f59..f52b01e07601 100644 --- a/src/jit/codegenlinear.cpp +++ b/src/jit/codegenlinear.cpp @@ -1056,7 +1056,6 @@ void CodeGen::genUnspillRegIfNeeded(GenTree* tree) unsigned flags = splitArg->GetRegSpillFlagByIdx(i); if ((flags & GTF_SPILLED) != 0) { - BYTE* gcPtrs = splitArg->gtGcPtrs; var_types dstType = splitArg->GetRegType(i); regNumber dstReg = splitArg->GetRegNumByIdx(i); diff --git a/src/jit/codegenxarch.cpp b/src/jit/codegenxarch.cpp index d7b15025a3f9..597e44026873 100644 --- a/src/jit/codegenxarch.cpp +++ b/src/jit/codegenxarch.cpp @@ -2693,7 +2693,7 @@ void CodeGen::genCodeForStoreBlk(GenTreeBlk* storeBlkNode) if (storeBlkNode->OperIs(GT_STORE_OBJ) && storeBlkNode->OperIsCopyBlkOp() && !storeBlkNode->gtBlkOpGcUnsafe) { - assert(storeBlkNode->AsObj()->gtGcPtrCount != 0); + assert(storeBlkNode->AsObj()->gtLayout->HasGCPtr()); genCodeForCpObj(storeBlkNode->AsObj()); return; } @@ -3488,7 +3488,7 @@ void CodeGen::genCodeForCpObj(GenTreeObj* cpObjNode) // If the GenTree node has data about GC pointers, this means we're dealing // with CpObj, so this requires special logic. - assert(cpObjNode->gtGcPtrCount > 0); + assert(cpObjNode->gtLayout->HasGCPtr()); // MovSp (alias for movsq on x64 and movsd on x86) instruction is used for copying non-gcref fields // and it needs src = RSI and dst = RDI. @@ -3526,7 +3526,7 @@ void CodeGen::genCodeForCpObj(GenTreeObj* cpObjNode) gcInfo.gcMarkRegPtrVal(REG_RSI, srcAddrType); gcInfo.gcMarkRegPtrVal(REG_RDI, dstAddr->TypeGet()); - unsigned slots = cpObjNode->gtSlots; + unsigned slots = cpObjNode->gtLayout->GetSlotCount(); // If we can prove it's on the stack we don't need to use the write barrier. if (dstOnStack) @@ -3553,56 +3553,56 @@ void CodeGen::genCodeForCpObj(GenTreeObj* cpObjNode) } else { - BYTE* gcPtrs = cpObjNode->gtGcPtrs; - unsigned gcPtrCount = cpObjNode->gtGcPtrCount; + StructTypeLayout* layout = cpObjNode->gtLayout; + unsigned gcPtrCount = 0; unsigned i = 0; while (i < slots) { - switch (gcPtrs[i]) + if (!layout->IsGCPtr(i)) { - case TYPE_GC_NONE: - // Let's see if we can use rep movsp instead of a sequence of movsp instructions - // to save cycles and code size. - { - unsigned nonGcSlotCount = 0; + // Let's see if we can use rep movsp instead of a sequence of movsp instructions + // to save cycles and code size. + { + unsigned nonGcSlotCount = 0; - do - { - nonGcSlotCount++; - i++; - } while (i < slots && gcPtrs[i] == TYPE_GC_NONE); + do + { + nonGcSlotCount++; + i++; + } while ((i < slots) && !layout->IsGCPtr(i)); - // If we have a very small contiguous non-gc region, it's better just to - // emit a sequence of movsp instructions - if (nonGcSlotCount < CPOBJ_NONGC_SLOTS_LIMIT) + // If we have a very small contiguous non-gc region, it's better just to + // emit a sequence of movsp instructions + if (nonGcSlotCount < CPOBJ_NONGC_SLOTS_LIMIT) + { + while (nonGcSlotCount > 0) { - while (nonGcSlotCount > 0) - { - instGen(INS_movsp); - nonGcSlotCount--; - } + instGen(INS_movsp); + nonGcSlotCount--; } - else - { - // Otherwise, we can save code-size and improve CQ by emitting - // rep movsp (alias for movsd/movsq for x86/x64) - assert((cpObjNode->gtRsvdRegs & RBM_RCX) != 0); + } + else + { + // Otherwise, we can save code-size and improve CQ by emitting + // rep movsp (alias for movsd/movsq for x86/x64) + assert((cpObjNode->gtRsvdRegs & RBM_RCX) != 0); - getEmitter()->emitIns_R_I(INS_mov, EA_4BYTE, REG_RCX, nonGcSlotCount); - instGen(INS_r_movsp); - } + getEmitter()->emitIns_R_I(INS_mov, EA_4BYTE, REG_RCX, nonGcSlotCount); + instGen(INS_r_movsp); } - break; - default: - // We have a GC pointer, call the memory barrier. - genEmitHelperCall(CORINFO_HELP_ASSIGN_BYREF, 0, EA_PTRSIZE); - gcPtrCount--; - i++; + } + } + else + { + // We have a GC pointer, call the memory barrier. + genEmitHelperCall(CORINFO_HELP_ASSIGN_BYREF, 0, EA_PTRSIZE); + gcPtrCount++; + i++; } } - assert(gcPtrCount == 0); + assert(gcPtrCount == layout->GetGCPtrCount()); } // Clear the gcInfo for RSI and RDI. @@ -5282,12 +5282,12 @@ void CodeGen::genCallInstruction(GenTreeCall* call) if (source->TypeGet() == TYP_STRUCT) { GenTreeObj* obj = source->AsObj(); - unsigned argBytes = roundUp(obj->gtBlkSize, TARGET_POINTER_SIZE); + unsigned argBytes = roundUp(obj->Size(), TARGET_POINTER_SIZE); #ifdef _TARGET_X86_ // If we have an OBJ, we must have created a copy if the original arg was not a // local and was not a multiple of TARGET_POINTER_SIZE. // Note that on x64/ux this will be handled by unrolling in genStructPutArgUnroll. - assert((argBytes == obj->gtBlkSize) || obj->Addr()->IsLocalAddrExpr()); + assert((argBytes == obj->Size()) || obj->Addr()->IsLocalAddrExpr()); #endif // _TARGET_X86_ assert((curArgTabEntry->numSlots * TARGET_POINTER_SIZE) == argBytes); } @@ -7481,11 +7481,11 @@ bool CodeGen::genAdjustStackForPutArgStk(GenTreePutArgStk* putArgStk) { case GenTreePutArgStk::Kind::RepInstr: case GenTreePutArgStk::Kind::Unroll: - assert((putArgStk->gtNumberReferenceSlots == 0) && (source->OperGet() != GT_FIELD_LIST) && (argSize >= 16)); + assert(!putArgStk->gtLayout->HasGCPtr() && (source->OperGet() != GT_FIELD_LIST) && (argSize >= 16)); break; case GenTreePutArgStk::Kind::Push: case GenTreePutArgStk::Kind::PushAllSlots: - assert((putArgStk->gtNumberReferenceSlots != 0) || (source->OperGet() == GT_FIELD_LIST) || (argSize < 16)); + assert(putArgStk->gtLayout->HasGCPtr() || (source->OperGet() == GT_FIELD_LIST) || (argSize < 16)); break; case GenTreePutArgStk::Kind::Invalid: default: @@ -8010,7 +8010,7 @@ void CodeGen::genPutStructArgStk(GenTreePutArgStk* putArgStk) assert(targetType == TYP_STRUCT); - if (putArgStk->gtNumberReferenceSlots == 0) + if (!putArgStk->gtLayout->HasGCPtr()) { switch (putArgStk->gtPutArgStkKind) { @@ -8041,9 +8041,9 @@ void CodeGen::genPutStructArgStk(GenTreePutArgStk* putArgStk) // future. assert(m_pushStkArg); - GenTree* srcAddr = source->gtGetOp1(); - BYTE* gcPtrs = putArgStk->gtGcPtrs; - const unsigned numSlots = putArgStk->gtNumSlots; + GenTree* srcAddr = source->gtGetOp1(); + StructTypeLayout* layout = putArgStk->gtLayout; + const unsigned numSlots = putArgStk->gtNumSlots; regNumber srcRegNum = srcAddr->gtRegNum; const bool srcAddrInReg = srcRegNum != REG_NA; @@ -8067,20 +8067,8 @@ void CodeGen::genPutStructArgStk(GenTreePutArgStk* putArgStk) for (int i = numSlots - 1; i >= 0; --i) { - emitAttr slotAttr; - if (gcPtrs[i] == TYPE_GC_NONE) - { - slotAttr = EA_4BYTE; - } - else if (gcPtrs[i] == TYPE_GC_REF) - { - slotAttr = EA_GCREF; - } - else - { - assert(gcPtrs[i] == TYPE_GC_BYREF); - slotAttr = EA_BYREF; - } + var_types slotType = layout->GetGCPtrType(i); + emitAttr slotAttr = emitTypeSize(slotType); const unsigned offset = i * TARGET_POINTER_SIZE; if (srcAddrInReg) @@ -8106,11 +8094,11 @@ void CodeGen::genPutStructArgStk(GenTreePutArgStk* putArgStk) unsigned numGCSlotsCopied = 0; #endif // DEBUG - BYTE* gcPtrs = putArgStk->gtGcPtrs; - const unsigned numSlots = putArgStk->gtNumSlots; + StructTypeLayout* layout = putArgStk->gtLayout; + const unsigned numSlots = putArgStk->gtNumSlots; for (unsigned i = 0; i < numSlots;) { - if (gcPtrs[i] == TYPE_GC_NONE) + if (!layout->IsGCPtr(i)) { // Let's see if we can use rep movsp (alias for movsd or movsq for 32 and 64 bits respectively) // instead of a sequence of movsp instructions to save cycles and code size. @@ -8119,7 +8107,7 @@ void CodeGen::genPutStructArgStk(GenTreePutArgStk* putArgStk) { adjacentNonGCSlotCount++; i++; - } while ((i < numSlots) && (gcPtrs[i] == TYPE_GC_NONE)); + } while ((i < numSlots) && !layout->IsGCPtr(i)); // If we have a very small contiguous non-ref region, it's better just to // emit a sequence of movsp instructions @@ -8138,15 +8126,13 @@ void CodeGen::genPutStructArgStk(GenTreePutArgStk* putArgStk) } else { - assert((gcPtrs[i] == TYPE_GC_REF) || (gcPtrs[i] == TYPE_GC_BYREF)); - // We have a GC (byref or ref) pointer // TODO-Amd64-Unix: Here a better solution (for code size and CQ) would be to use movsp instruction, // but the logic for emitting a GC info record is not available (it is internal for the emitter // only.) See emitGCVarLiveUpd function. If we could call it separately, we could do // instGen(INS_movsp); and emission of gc info. - var_types memType = (gcPtrs[i] == TYPE_GC_REF) ? TYP_REF : TYP_BYREF; + var_types memType = layout->GetGCPtrType(i); getEmitter()->emitIns_R_AR(ins_Load(memType), emitTypeSize(memType), REG_RCX, REG_RSI, 0); genStoreRegToStackArg(memType, REG_RCX, i * TARGET_POINTER_SIZE); #ifdef DEBUG @@ -8168,7 +8154,7 @@ void CodeGen::genPutStructArgStk(GenTreePutArgStk* putArgStk) } } - assert(numGCSlotsCopied == putArgStk->gtNumberReferenceSlots); + assert(numGCSlotsCopied == putArgStk->gtLayout->GetGCPtrCount()); #endif // _TARGET_X86_ } } diff --git a/src/jit/compiler.cpp b/src/jit/compiler.cpp index 489218800e9d..663a897ffcb3 100644 --- a/src/jit/compiler.cpp +++ b/src/jit/compiler.cpp @@ -445,7 +445,7 @@ Histogram loopExitCountTable(loopExitCountBuckets); // the JIT will often pass the address of a single BYTE, instead of a BYTE[] // -var_types Compiler::getJitGCType(BYTE gcType) +/* static */ var_types Compiler::getJitGCType(BYTE gcType) { var_types result = TYP_UNKNOWN; CorInfoGCType corInfoType = (CorInfoGCType)gcType; @@ -469,61 +469,6 @@ var_types Compiler::getJitGCType(BYTE gcType) return result; } -#if FEATURE_MULTIREG_ARGS -//--------------------------------------------------------------------------- -// getStructGcPtrsFromOp: Given a GenTree node of TYP_STRUCT that represents -// a pass by value argument, return the gcPtr layout -// for the pointers sized fields -// Arguments: -// op - the operand of TYP_STRUCT that is passed by value -// gcPtrsOut - an array of BYTES that are written by this method -// they will contain the VM's CorInfoGCType values -// for each pointer sized field -// Return Value: -// Two [or more] values are written into the gcPtrs array -// -// Note that for ARM64 there will always be exactly two pointer sized fields - -void Compiler::getStructGcPtrsFromOp(GenTree* op, BYTE* gcPtrsOut) -{ - assert(op->TypeGet() == TYP_STRUCT); - -#ifdef _TARGET_ARM64_ - if (op->OperGet() == GT_OBJ) - { - CORINFO_CLASS_HANDLE objClass = op->gtObj.gtClass; - - int structSize = info.compCompHnd->getClassSize(objClass); - assert(structSize <= 2 * TARGET_POINTER_SIZE); - - BYTE gcPtrsTmp[2] = {TYPE_GC_NONE, TYPE_GC_NONE}; - - info.compCompHnd->getClassGClayout(objClass, &gcPtrsTmp[0]); - - gcPtrsOut[0] = gcPtrsTmp[0]; - gcPtrsOut[1] = gcPtrsTmp[1]; - } - else if (op->OperGet() == GT_LCL_VAR) - { - GenTreeLclVarCommon* varNode = op->AsLclVarCommon(); - unsigned varNum = varNode->gtLclNum; - assert(varNum < lvaCount); - LclVarDsc* varDsc = &lvaTable[varNum]; - - // At this point any TYP_STRUCT LclVar must be a 16-byte pass by value argument - assert(varDsc->lvSize() == 2 * TARGET_POINTER_SIZE); - - gcPtrsOut[0] = varDsc->lvGcLayout[0]; - gcPtrsOut[1] = varDsc->lvGcLayout[1]; - } - else -#endif - { - noway_assert(!"Unsupported Oper for getStructGcPtrsFromOp"); - } -} -#endif // FEATURE_MULTIREG_ARGS - #ifdef ARM_SOFTFP //--------------------------------------------------------------------------- // IsSingleFloat32Struct: @@ -2054,6 +1999,8 @@ void Compiler::compInit(ArenaAllocator* pAlloc, InlineInfo* inlineInfo) // We start with the flow graph in tree-order fgOrder = FGOrderTree; + m_typeLayoutCache = nullptr; + #ifdef FEATURE_SIMD m_simdHandleCache = nullptr; #endif // FEATURE_SIMD @@ -5107,28 +5054,11 @@ bool Compiler::compQuirkForPPP() // This fixes the PPP backward compat issue varDscExposedStruct->lvExactSize += 32; - // Update the GC info to indicate that the padding area does - // not contain any GC pointers. - // // The struct is now 64 bytes. - // // We're on x64 so this should be 8 pointer slots. assert((varDscExposedStruct->lvExactSize / TARGET_POINTER_SIZE) == 8); - BYTE* oldGCPtrs = varDscExposedStruct->lvGcLayout; - BYTE* newGCPtrs = getAllocator(CMK_LvaTable).allocate(8); - - for (int i = 0; i < 4; i++) - { - newGCPtrs[i] = oldGCPtrs[i]; - } - - for (int i = 4; i < 8; i++) - { - newGCPtrs[i] = TYPE_GC_NONE; - } - - varDscExposedStruct->lvGcLayout = newGCPtrs; + varDscExposedStruct->lvLayout = varDscExposedStruct->lvLayout->GetPPPQuirkLayout(getAllocator(CMK_Generic)); return true; } @@ -11097,3 +11027,370 @@ bool Compiler::killGCRefs(GenTree* tree) return false; } + +class StructTypeLayoutCache +{ + typedef JitHashTable, unsigned> BlkLayoutMap; + typedef JitHashTable, unsigned> ObjLayoutMap; + + unsigned m_layoutCount; + unsigned m_layoutLargeCapacity; + union { + StructTypeLayout* m_layoutArray[3]; + + struct + { + BlkLayoutMap* m_blkLayoutMap; + ObjLayoutMap* m_objLayoutMap; + StructTypeLayout** m_layoutLargeArray; + }; + }; + + unsigned AddLayout(CompAllocator alloc, StructTypeLayout* layout) + { + if (m_layoutCount >= m_layoutLargeCapacity) + { + unsigned newCapacity = m_layoutCount * 2; + StructTypeLayout** newArray = alloc.allocate(newCapacity); + + if (m_layoutCount <= _countof(m_layoutArray)) + { + BlkLayoutMap* blkLayoutMap = new (alloc) BlkLayoutMap(alloc); + ObjLayoutMap* objLayoutMap = new (alloc) ObjLayoutMap(alloc); + + for (unsigned i = 0; i < m_layoutCount; i++) + { + StructTypeLayout* l = m_layoutArray[i]; + + newArray[i] = l; + + if (l->GetClass() == NO_CLASS_HANDLE) + { + blkLayoutMap->Set(l->GetSize(), i); + } + else + { + objLayoutMap->Set(l->GetClass(), i); + } + } + + m_blkLayoutMap = blkLayoutMap; + m_objLayoutMap = objLayoutMap; + } + else + { + memcpy(newArray, m_layoutLargeArray, m_layoutCount * sizeof(newArray[0])); + } + + m_layoutLargeArray = newArray; + m_layoutLargeCapacity = newCapacity; + } + + m_layoutLargeArray[m_layoutCount] = layout; + return m_layoutCount++; + } + + StructTypeLayout* CreateLayout(Compiler* compiler, unsigned blockSize) + { + return new (compiler, CMK_TypeLayout) StructTypeLayout(blockSize); + } + + StructTypeLayout* CreateLayout(Compiler* compiler, CORINFO_CLASS_HANDLE classHandle) + { + bool isValueClass = compiler->info.compCompHnd->isValueClass(classHandle); + unsigned size; + + if (isValueClass) + { + size = compiler->info.compCompHnd->getClassSize(classHandle); + } + else + { + size = compiler->info.compCompHnd->getHeapClassSize(classHandle); + } + + INDEBUG(const char* className = compiler->info.compCompHnd->getClassName(classHandle);) + + return new (compiler, CMK_TypeLayout) StructTypeLayout(classHandle, isValueClass, size DEBUGARG(className)); + } + +public: + StructTypeLayoutCache() : m_layoutCount(0), m_layoutLargeCapacity(0) + { + } + + StructTypeLayout* GetBlkLayout(Compiler* compiler, unsigned blockSize) + { + if (m_layoutCount <= _countof(m_layoutArray)) + { + for (unsigned i = 0; i < m_layoutCount; i++) + { + if ((m_layoutArray[i]->GetClass() == NO_CLASS_HANDLE) && (m_layoutArray[i]->GetSize() == blockSize)) + { + return m_layoutArray[i]; + } + } + } + else + { + unsigned index; + if (m_blkLayoutMap->Lookup(blockSize, &index)) + { + return m_layoutLargeArray[index]; + } + } + + StructTypeLayout* layout = CreateLayout(compiler, blockSize); + + if (m_layoutCount < _countof(m_layoutArray)) + { + m_layoutArray[m_layoutCount++] = layout; + } + else + { + unsigned index = AddLayout(compiler->getAllocator(CMK_TypeLayout), layout); + m_blkLayoutMap->Set(blockSize, index); + } + + return layout; + } + + unsigned GetBlkLayoutNum(Compiler* compiler, unsigned blockSize) + { + if (m_layoutCount <= _countof(m_layoutArray)) + { + for (unsigned i = 0; i < m_layoutCount; i++) + { + if ((m_layoutArray[i]->GetClass() == NO_CLASS_HANDLE) && (m_layoutArray[i]->GetSize() == blockSize)) + { + return i; + } + } + } + else + { + unsigned index; + if (m_blkLayoutMap->Lookup(blockSize, &index)) + { + return index; + } + } + + StructTypeLayout* layout = CreateLayout(compiler, blockSize); + + if (m_layoutCount < _countof(m_layoutArray)) + { + m_layoutArray[m_layoutCount] = layout; + return m_layoutCount++; + } + else + { + unsigned index = AddLayout(compiler->getAllocator(CMK_TypeLayout), layout); + m_blkLayoutMap->Set(blockSize, index); + return index; + } + } + + StructTypeLayout* GetObjLayout(Compiler* compiler, CORINFO_CLASS_HANDLE classHandle) + { + if (m_layoutCount <= _countof(m_layoutArray)) + { + for (unsigned i = 0; i < m_layoutCount; i++) + { + if (m_layoutArray[i]->GetClass() == classHandle) + { + return m_layoutArray[i]; + } + } + } + else + { + unsigned index; + if (m_objLayoutMap->Lookup(classHandle, &index)) + { + return m_layoutLargeArray[index]; + } + } + + StructTypeLayout* layout = CreateLayout(compiler, classHandle); + + if (m_layoutCount < _countof(m_layoutArray)) + { + m_layoutArray[m_layoutCount++] = layout; + } + else + { + unsigned index = AddLayout(compiler->getAllocator(CMK_TypeLayout), layout); + m_objLayoutMap->Set(classHandle, index); + } + + return layout; + } + + unsigned GetObjLayoutNum(Compiler* compiler, CORINFO_CLASS_HANDLE classHandle) + { + if (m_layoutCount <= _countof(m_layoutArray)) + { + for (unsigned i = 0; i < m_layoutCount; i++) + { + if (m_layoutArray[i]->GetClass() == classHandle) + { + return i; + } + } + } + else + { + unsigned index; + if (m_objLayoutMap->Lookup(classHandle, &index)) + { + return index; + } + } + + StructTypeLayout* layout = CreateLayout(compiler, classHandle); + + if (m_layoutCount < _countof(m_layoutArray)) + { + m_layoutArray[m_layoutCount] = layout; + return m_layoutCount++; + } + else + { + unsigned index = AddLayout(compiler->getAllocator(CMK_TypeLayout), layout); + m_objLayoutMap->Set(classHandle, index); + return index; + } + } + + StructTypeLayout* GetLayoutByNum(unsigned num) + { + assert(num < m_layoutCount); + + if (m_layoutCount <= _countof(m_layoutArray)) + { + return m_layoutArray[num]; + } + else + { + return m_layoutLargeArray[num]; + } + } +}; + +void Compiler::typCreateTypeLayoutCache() +{ + assert(m_typeLayoutCache == nullptr); + + if (compIsForInlining()) + { + m_typeLayoutCache = impInlineInfo->InlinerCompiler->m_typeLayoutCache; + + if (m_typeLayoutCache == nullptr) + { + m_typeLayoutCache = new (this, CMK_TypeLayout) StructTypeLayoutCache(); + impInlineInfo->InlinerCompiler->m_typeLayoutCache = m_typeLayoutCache; + } + } + else + { + m_typeLayoutCache = new (this, CMK_TypeLayout) StructTypeLayoutCache(); + } +} + +StructTypeLayout* Compiler::typGetBlkLayout(unsigned blockSize) +{ + if (m_typeLayoutCache == nullptr) + { + typCreateTypeLayoutCache(); + } + + return m_typeLayoutCache->GetBlkLayout(this, blockSize); +} + +StructTypeLayout* Compiler::typGetObjLayout(CORINFO_CLASS_HANDLE classHandle) +{ + if (m_typeLayoutCache == nullptr) + { + typCreateTypeLayoutCache(); + } + + return m_typeLayoutCache->GetObjLayout(this, classHandle); +} + +void StructTypeLayout::EnsureGCInfoInitialized(Compiler* compiler) +{ + if (m_gcPtrsInitialized) + { + return; + } + + assert(m_classHandle != NO_CLASS_HANDLE); + + if (m_size < TARGET_POINTER_SIZE) + { + m_gcPtrsArray[0] = TYPE_GC_NONE; + } + else + { + BYTE* gcPtrs; + + if (GetSlotCount() > sizeof(m_gcPtrsArray)) + { + gcPtrs = m_gcPtrs = new (compiler, CMK_TypeLayout) BYTE[GetSlotCount()]; + } + else + { + gcPtrs = m_gcPtrsArray; + } + + unsigned gcPtrCount = compiler->info.compCompHnd->getClassGClayout(m_classHandle, gcPtrs); + + assert(gcPtrCount < (1 << 30)); + + assert((gcPtrCount == 0) || ((compiler->info.compCompHnd->getClassAttribs(m_classHandle) & + (CORINFO_FLG_CONTAINS_GC_PTR | CORINFO_FLG_CONTAINS_STACK_PTR)) != 0)); + + // We assume that we cannot have a struct with GC pointers that is not a multiple + // of the register size. + // The EE currently does not allow this, but it could change. + // Let's assert it just to be safe. + noway_assert((gcPtrCount == 0) || (roundUp(m_size, REGSIZE_BYTES) == m_size)); + + m_gcPtrCount = gcPtrCount; + } + + m_gcPtrsInitialized = true; +} + +#ifdef _TARGET_AMD64_ +StructTypeLayout* StructTypeLayout::GetPPPQuirkLayout(CompAllocator alloc) +{ + assert(m_gcPtrsInitialized); + assert(m_classHandle != NO_CLASS_HANDLE); + assert(m_isValueClass); + assert(m_size == 32); + + if (m_pppQuirkLayout == nullptr) + { + m_pppQuirkLayout = new (alloc) StructTypeLayout(m_classHandle, m_isValueClass, 64 DEBUGARG(m_className)); + m_pppQuirkLayout->m_gcPtrCount = m_gcPtrCount; + + static_assert_no_msg(_countof(m_gcPtrsArray) == 8); + + for (int i = 0; i < 4; i++) + { + m_pppQuirkLayout->m_gcPtrsArray[i] = m_gcPtrsArray[i]; + } + + for (int i = 4; i < 8; i++) + { + m_pppQuirkLayout->m_gcPtrsArray[i] = TYPE_GC_NONE; + } + + m_pppQuirkLayout->m_gcPtrsInitialized = true; + } + + return m_pppQuirkLayout; +} +#endif // _TARGET_AMD64_ diff --git a/src/jit/compiler.h b/src/jit/compiler.h index 8c2b9458633c..c4cc5f52cb4c 100644 --- a/src/jit/compiler.h +++ b/src/jit/compiler.h @@ -326,12 +326,10 @@ class LclVarDsc unsigned char lvIsRegArg : 1; // is this a register argument? unsigned char lvFramePointerBased : 1; // 0 = off of REG_SPBASE (e.g., ESP), 1 = off of REG_FPBASE (e.g., EBP) - unsigned char lvStructGcCount : 3; // if struct, how many GC pointer (stop counting at 7). The only use of values >1 - // is to help determine whether to use block init in the prolog. - unsigned char lvOnFrame : 1; // (part of) the variable lives on the frame - unsigned char lvRegister : 1; // assigned to live in a register? For RyuJIT backend, this is only set if the - // variable is in the same register for the entire function. - unsigned char lvTracked : 1; // is this a tracked variable? + unsigned char lvOnFrame : 1; // (part of) the variable lives on the frame + unsigned char lvRegister : 1; // assigned to live in a register? For RyuJIT backend, this is only set if the + // variable is in the same register for the entire function. + unsigned char lvTracked : 1; // is this a tracked variable? bool lvTrackedNonStruct() { return lvTracked && lvType != TYP_STRUCT; @@ -807,7 +805,7 @@ class LclVarDsc CORINFO_FIELD_HANDLE lvFieldHnd; // field handle for promoted struct fields - BYTE* lvGcLayout; // GC layout info for structs + StructTypeLayout* lvLayout; // layout info for structs #if ASSERTION_PROP BlockSet lvRefBlks; // Set of blocks that contain refs @@ -1263,6 +1261,10 @@ struct fgArgTabEntry // placeholder it will point at the actual argument in the gtCallLateArgs list. GenTree* parent; // Points at the GT_LIST node in the gtCallArgs for this argument +#ifdef FEATURE_PUT_STRUCT_ARG_STK + StructTypeLayout* layout; +#endif + unsigned argNum; // The original argument number, also specifies the required argument evaluation order from the IL private: @@ -3247,7 +3249,6 @@ class Compiler } #endif // defined(FEATURE_SIMD) - BYTE* lvaGetGcLayout(unsigned varNum); bool lvaTypeIsGC(unsigned varNum); unsigned lvaGSSecurityCookie; // LclVar number bool lvaTempsHaveLargerOffsetThanVars(); @@ -3586,10 +3587,7 @@ class Compiler GenTree* impGetStructAddr(GenTree* structVal, CORINFO_CLASS_HANDLE structHnd, unsigned curLevel, bool willDeref); - var_types impNormStructType(CORINFO_CLASS_HANDLE structHnd, - BYTE* gcLayout = nullptr, - unsigned* numGCVars = nullptr, - var_types* simdBaseType = nullptr); + var_types impNormStructType(CORINFO_CLASS_HANDLE structHnd, var_types* simdBaseType = nullptr); GenTree* impNormStructVal(GenTree* structVal, CORINFO_CLASS_HANDLE structHnd, @@ -4552,7 +4550,7 @@ class Compiler } // Convert a BYTE which represents the VM's CorInfoGCtype to the JIT's var_types - var_types getJitGCType(BYTE gcType); + static var_types getJitGCType(BYTE gcType); enum structPassingKind { @@ -7708,7 +7706,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX // Pops and returns GenTree node from importers type stack. // Normalizes TYP_STRUCT value in case of GT_CALL, GT_RET_EXPR and arg nodes. - GenTree* impSIMDPopStack(var_types type, bool expectAddr = false, CORINFO_CLASS_HANDLE structType = nullptr); + GenTree* impSIMDPopStack(var_types type, bool expectAddr = false, StructTypeLayout* layout = nullptr); // Create a GT_SIMD tree for a Get property of SIMD vector with a fixed index. GenTreeSIMD* impSIMDGetFixed(var_types simdType, var_types baseType, unsigned simdSize, int index); @@ -8696,12 +8694,6 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #endif // FEATURE_MULTIREG_RET } -#if FEATURE_MULTIREG_ARGS - // Given a GenTree node of TYP_STRUCT that represents a pass by value argument - // return the gcPtr layout for the pointers sized fields - void getStructGcPtrsFromOp(GenTree* op, BYTE* gcPtrsOut); -#endif // FEATURE_MULTIREG_ARGS - // Returns true if the method being compiled returns a value bool compMethodHasRetVal() { @@ -8715,6 +8707,15 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #endif // DEBUG +private: + class StructTypeLayoutCache* m_typeLayoutCache; + + void typCreateTypeLayoutCache(); + +public: + StructTypeLayout* typGetBlkLayout(unsigned blockSize); + StructTypeLayout* typGetObjLayout(CORINFO_CLASS_HANDLE classHandle); + //-------------------------- Global Compiler Data ------------------------------------ #ifdef DEBUG diff --git a/src/jit/compiler.hpp b/src/jit/compiler.hpp index a9dc3bbc5fc5..6e9c4ead6a07 100644 --- a/src/jit/compiler.hpp +++ b/src/jit/compiler.hpp @@ -1859,8 +1859,7 @@ inline bool Compiler::lvaTypeIsGC(unsigned varNum) { if (lvaTable[varNum].TypeGet() == TYP_STRUCT) { - assert(lvaTable[varNum].lvGcLayout != nullptr); // bits are intialized - return (lvaTable[varNum].lvStructGcCount != 0); + return lvaTable[varNum].lvLayout->HasGCPtr(); } return (varTypeIsGC(lvaTable[varNum].TypeGet())); } @@ -4123,7 +4122,8 @@ inline void Compiler::CLR_API_Leave(API_ICorJitInfo_Names ename) bool Compiler::fgStructTempNeedsExplicitZeroInit(LclVarDsc* varDsc, BasicBlock* block) { - bool containsGCPtr = (varDsc->lvStructGcCount > 0); + bool containsGCPtr = + varTypeIsGC(varDsc->TypeGet()) || ((varDsc->TypeGet() == TYP_STRUCT) && varDsc->lvLayout->HasGCPtr()); return (!info.compInitMem || ((block->bbFlags & BBF_BACKWARD_JUMP) != 0) || (!containsGCPtr && varDsc->lvIsTemp)); } diff --git a/src/jit/compmemkind.h b/src/jit/compmemkind.h index 582a2a9e3ae0..6a97956b6a1e 100644 --- a/src/jit/compmemkind.h +++ b/src/jit/compmemkind.h @@ -54,6 +54,7 @@ CompMemKindMacro(RangeCheck) CompMemKindMacro(CopyProp) CompMemKindMacro(SideEffects) CompMemKindMacro(ObjectAllocator) +CompMemKindMacro(TypeLayout) //clang-format on #undef CompMemKindMacro diff --git a/src/jit/gcencode.cpp b/src/jit/gcencode.cpp index 4ec6f4cb8b5b..0a1ff6b5766d 100644 --- a/src/jit/gcencode.cpp +++ b/src/jit/gcencode.cpp @@ -2321,17 +2321,22 @@ size_t GCInfo::gcMakeRegPtrTable(BYTE* dest, int mask, const InfoHdr& header, un // A struct will have gcSlots only if it is at least TARGET_POINTER_SIZE. if (varDsc->lvType == TYP_STRUCT && varDsc->lvOnFrame && (varDsc->lvExactSize >= TARGET_POINTER_SIZE)) { - unsigned slots = compiler->lvaLclSize(varNum) / TARGET_POINTER_SIZE; - BYTE* gcPtrs = compiler->lvaGetGcLayout(varNum); + StructTypeLayout* layout = varDsc->lvLayout; + const unsigned slots = varDsc->lvLayout->GetSlotCount(); // walk each member of the array for (unsigned i = 0; i < slots; i++) { - if (gcPtrs[i] == TYPE_GC_NONE) // skip non-gc slots + if (!layout->IsGCPtr(i)) + { + // skip non-gc slots continue; + } if (pass == 0) + { count++; + } else { assert(pass == 1); @@ -2342,16 +2347,22 @@ size_t GCInfo::gcMakeRegPtrTable(BYTE* dest, int mask, const InfoHdr& header, un // arguments are addressed relative to EBP. if (compiler->genDoubleAlign() && varDsc->lvIsParam && !varDsc->lvIsRegArg) + { offset += compiler->codeGen->genTotalFrameSize(); + } #endif - if (gcPtrs[i] == TYPE_GC_BYREF) + if (layout->GetGCPtrType(i) == TYP_BYREF) + { offset |= byref_OFFSET_FLAG; // indicate it is a byref GC pointer + } int encodedoffset = lastoffset - offset; lastoffset = offset; if (mask == 0) + { totalSize += encodeSigned(NULL, encodedoffset); + } else { unsigned sz = encodeSigned(dest, encodedoffset); @@ -4276,14 +4287,15 @@ void GCInfo::gcMakeRegPtrTable( // Note that the enregisterable struct types cannot have GC pointers in them. if ((varDsc->lvType == TYP_STRUCT) && varDsc->lvOnFrame && (varDsc->lvExactSize >= TARGET_POINTER_SIZE)) { - unsigned slots = compiler->lvaLclSize(varNum) / TARGET_POINTER_SIZE; - BYTE* gcPtrs = compiler->lvaGetGcLayout(varNum); + StructTypeLayout* layout = varDsc->lvLayout; + const unsigned slots = layout->GetSlotCount(); // walk each member of the array for (unsigned i = 0; i < slots; i++) { - if (gcPtrs[i] == TYPE_GC_NONE) - { // skip non-gc slots + if (!layout->IsGCPtr(i)) + { + // skip non-gc slots continue; } @@ -4296,7 +4308,7 @@ void GCInfo::gcMakeRegPtrTable( offset += compiler->codeGen->genTotalFrameSize(); #endif GcSlotFlags flags = GC_SLOT_UNTRACKED; - if (gcPtrs[i] == TYPE_GC_BYREF) + if (layout->GetGCPtrType(i) == TYP_BYREF) { flags = (GcSlotFlags)(flags | GC_SLOT_INTERIOR); } diff --git a/src/jit/gcinfo.cpp b/src/jit/gcinfo.cpp index 14ca8a873f5e..7c5e3ad491e9 100644 --- a/src/jit/gcinfo.cpp +++ b/src/jit/gcinfo.cpp @@ -477,19 +477,9 @@ void GCInfo::gcCountForHeader(UNALIGNED unsigned int* untrackedCount, UNALIGNED count++; } - else if (varDsc->lvType == TYP_STRUCT && varDsc->lvOnFrame && (varDsc->lvExactSize >= TARGET_POINTER_SIZE)) + else if ((varDsc->lvType == TYP_STRUCT) && varDsc->lvOnFrame && (varDsc->lvExactSize >= TARGET_POINTER_SIZE)) { - unsigned slots = compiler->lvaLclSize(varNum) / TARGET_POINTER_SIZE; - BYTE* gcPtrs = compiler->lvaGetGcLayout(varNum); - - // walk each member of the array - for (unsigned i = 0; i < slots; i++) - { - if (gcPtrs[i] != TYPE_GC_NONE) - { // count only gc slots - count++; - } - } + count += varDsc->lvLayout->GetGCPtrCount(); } } diff --git a/src/jit/gentree.cpp b/src/jit/gentree.cpp index 623a98c22ca8..40d048a59804 100644 --- a/src/jit/gentree.cpp +++ b/src/jit/gentree.cpp @@ -262,13 +262,11 @@ void GenTree::InitNodeSize() GenTree::s_gtNodeSizes[GT_ARR_INDEX] = TREE_NODE_SZ_LARGE; GenTree::s_gtNodeSizes[GT_ARR_OFFSET] = TREE_NODE_SZ_LARGE; GenTree::s_gtNodeSizes[GT_RET_EXPR] = TREE_NODE_SZ_LARGE; - GenTree::s_gtNodeSizes[GT_OBJ] = TREE_NODE_SZ_LARGE; GenTree::s_gtNodeSizes[GT_FIELD] = TREE_NODE_SZ_LARGE; GenTree::s_gtNodeSizes[GT_STMT] = TREE_NODE_SZ_LARGE; GenTree::s_gtNodeSizes[GT_CMPXCHG] = TREE_NODE_SZ_LARGE; GenTree::s_gtNodeSizes[GT_QMARK] = TREE_NODE_SZ_LARGE; GenTree::s_gtNodeSizes[GT_LEA] = TREE_NODE_SZ_LARGE; - GenTree::s_gtNodeSizes[GT_STORE_OBJ] = TREE_NODE_SZ_LARGE; GenTree::s_gtNodeSizes[GT_DYN_BLK] = TREE_NODE_SZ_LARGE; GenTree::s_gtNodeSizes[GT_STORE_DYN_BLK] = TREE_NODE_SZ_LARGE; GenTree::s_gtNodeSizes[GT_INTRINSIC] = TREE_NODE_SZ_LARGE; @@ -329,8 +327,9 @@ void GenTree::InitNodeSize() static_assert_no_msg(sizeof(GenTreeIndir) <= TREE_NODE_SZ_SMALL); static_assert_no_msg(sizeof(GenTreeStoreInd) <= TREE_NODE_SZ_SMALL); static_assert_no_msg(sizeof(GenTreeAddrMode) <= TREE_NODE_SZ_SMALL); - static_assert_no_msg(sizeof(GenTreeObj) <= TREE_NODE_SZ_LARGE); // *** large node + static_assert_no_msg(sizeof(GenTreeObj) <= TREE_NODE_SZ_SMALL); static_assert_no_msg(sizeof(GenTreeBlk) <= TREE_NODE_SZ_SMALL); + static_assert_no_msg(sizeof(GenTreeDynBlk) <= TREE_NODE_SZ_LARGE); // *** large node static_assert_no_msg(sizeof(GenTreeRetExpr) <= TREE_NODE_SZ_LARGE); // *** large node static_assert_no_msg(sizeof(GenTreeStmt) <= TREE_NODE_SZ_LARGE); // *** large node static_assert_no_msg(sizeof(GenTreeClsVar) <= TREE_NODE_SZ_SMALL); @@ -1264,7 +1263,7 @@ bool GenTree::Compare(GenTree* op1, GenTree* op2, bool swapOK) } break; case GT_OBJ: - if (op1->AsObj()->gtClass != op2->AsObj()->gtClass) + if (op1->AsObj()->gtLayout->GetClass() != op2->AsObj()->gtLayout->GetClass()) { return false; } @@ -1956,8 +1955,8 @@ unsigned Compiler::gtHashValue(GenTree* tree) break; case GT_OBJ: - hash = - genTreeHashAdd(hash, static_cast(reinterpret_cast(tree->gtObj.gtClass))); + hash = genTreeHashAdd(hash, static_cast( + reinterpret_cast(tree->AsObj()->gtLayout->GetClass()))); break; // For the ones below no extra argument matters for comparison. case GT_BOX: @@ -1994,12 +1993,12 @@ unsigned Compiler::gtHashValue(GenTree* tree) case GT_BLK: case GT_STORE_BLK: - hash += tree->gtBlk.gtBlkSize; + hash += tree->gtBlk.Size(); break; case GT_OBJ: case GT_STORE_OBJ: - hash ^= PtrToUlong(tree->AsObj()->gtClass); + hash ^= PtrToUlong(tree->AsObj()->gtLayout->GetClass()); break; case GT_DYN_BLK: @@ -6189,7 +6188,6 @@ GenTree* Compiler::gtNewObjNode(CORINFO_CLASS_HANDLE structHnd, GenTree* addr) { var_types nodeType = impNormStructType(structHnd); assert(varTypeIsStruct(nodeType)); - unsigned size = info.compCompHnd->getClassSize(structHnd); // It would be convenient to set the GC info at this time, but we don't actually require // it unless this is going to be a destination. @@ -6204,7 +6202,7 @@ GenTree* Compiler::gtNewObjNode(CORINFO_CLASS_HANDLE structHnd, GenTree* addr) return gtNewOperNode(GT_IND, nodeType, addr); } } - GenTreeBlk* newBlkOrObjNode = new (this, GT_OBJ) GenTreeObj(nodeType, addr, structHnd, size); + GenTreeBlk* newBlkOrObjNode = new (this, GT_OBJ) GenTreeObj(nodeType, addr, typGetObjLayout(structHnd)); // An Obj is not a global reference, if it is known to be a local struct. if ((addr->gtFlags & GTF_GLOB_REF) == 0) @@ -6230,30 +6228,16 @@ GenTree* Compiler::gtNewObjNode(CORINFO_CLASS_HANDLE structHnd, GenTree* addr) void Compiler::gtSetObjGcInfo(GenTreeObj* objNode) { - CORINFO_CLASS_HANDLE structHnd = objNode->gtClass; - var_types nodeType = objNode->TypeGet(); - unsigned size = objNode->gtBlkSize; - unsigned slots = 0; - unsigned gcPtrCount = 0; - BYTE* gcPtrs = nullptr; + assert(objNode->OperIs(GT_OBJ, GT_STORE_OBJ)); + assert(varTypeIsStruct(objNode->TypeGet())); + assert(objNode->TypeGet() == impNormStructType(objNode->gtLayout->GetClass())); - assert(varTypeIsStruct(nodeType)); - assert(size == info.compCompHnd->getClassSize(structHnd)); - assert(nodeType == impNormStructType(structHnd)); + objNode->gtLayout->EnsureGCInfoInitialized(this); - if (nodeType == TYP_STRUCT) + if (!objNode->gtLayout->HasGCPtr()) { - if (size >= TARGET_POINTER_SIZE) - { - // Get the GC fields info - var_types simdBaseType; // Dummy argument - slots = roundUp(size, TARGET_POINTER_SIZE) / TARGET_POINTER_SIZE; - gcPtrs = new (this, CMK_ASTNode) BYTE[slots]; - nodeType = impNormStructType(structHnd, gcPtrs, &gcPtrCount, &simdBaseType); - } + objNode->SetOper(objNode->OperIs(GT_OBJ) ? GT_BLK : GT_STORE_BLK); } - objNode->SetGCInfo(gcPtrs, gcPtrCount, slots); - assert(objNode->gtType == nodeType); } //------------------------------------------------------------------------ @@ -6323,7 +6307,7 @@ GenTree* Compiler::gtNewBlockVal(GenTree* addr, unsigned size) } } } - return new (this, GT_BLK) GenTreeBlk(GT_BLK, blkType, addr, size); + return new (this, GT_BLK) GenTreeBlk(GT_BLK, blkType, addr, typGetBlkLayout(size)); } // Creates a new assignment node for a CpObj. @@ -6343,7 +6327,7 @@ GenTree* Compiler::gtNewCpObjNode(GenTree* dstAddr, GenTree* srcAddr, CORINFO_CL if (lhs->OperIsBlk()) { - size = lhs->AsBlk()->gtBlkSize; + size = lhs->AsBlk()->Size(); if (lhs->OperGet() == GT_OBJ) { gtSetObjGcInfo(lhs->AsObj()); @@ -6438,33 +6422,6 @@ void Compiler::gtBlockOpInit(GenTree* result, GenTree* dst, GenTree* srcOrFillVa assert(dst->TypeGet() != TYP_STRUCT); return; } -#ifdef DEBUG - // If the copy involves GC pointers, the caller must have already set - // the node additional members (gtGcPtrs, gtGcPtrCount, gtSlots) on the dst. - if ((dst->gtOper == GT_OBJ) && dst->AsBlk()->HasGCPtr()) - { - GenTreeObj* objNode = dst->AsObj(); - assert(objNode->gtGcPtrs != nullptr); - assert(!IsUninitialized(objNode->gtGcPtrs)); - assert(!IsUninitialized(objNode->gtGcPtrCount)); - assert(!IsUninitialized(objNode->gtSlots) && objNode->gtSlots > 0); - - for (unsigned i = 0; i < objNode->gtGcPtrCount; ++i) - { - CorInfoGCType t = (CorInfoGCType)objNode->gtGcPtrs[i]; - switch (t) - { - case TYPE_GC_NONE: - case TYPE_GC_REF: - case TYPE_GC_BYREF: - case TYPE_GC_OTHER: - break; - default: - unreached(); - } - } - } -#endif // DEBUG /* In the case of CpBlk, we want to avoid generating * nodes where the source and destination are the same @@ -7147,14 +7104,13 @@ GenTree* Compiler::gtCloneExpr( break; case GT_OBJ: - copy = new (this, GT_OBJ) - GenTreeObj(tree->TypeGet(), tree->gtOp.gtOp1, tree->AsObj()->gtClass, tree->gtBlk.gtBlkSize); - copy->AsObj()->CopyGCInfo(tree->AsObj()); + copy = new (this, GT_OBJ) GenTreeObj(tree->TypeGet(), tree->gtOp.gtOp1, tree->AsObj()->gtLayout); copy->gtBlk.gtBlkOpGcUnsafe = tree->gtBlk.gtBlkOpGcUnsafe; break; case GT_BLK: - copy = new (this, GT_BLK) GenTreeBlk(GT_BLK, tree->TypeGet(), tree->gtOp.gtOp1, tree->gtBlk.gtBlkSize); + copy = + new (this, GT_BLK) GenTreeBlk(GT_BLK, tree->TypeGet(), tree->gtOp.gtOp1, tree->AsBlk()->gtLayout); copy->gtBlk.gtBlkOpGcUnsafe = tree->gtBlk.gtBlkOpGcUnsafe; break; @@ -9192,7 +9148,7 @@ void Compiler::gtDispNodeName(GenTree* tree) } else if (tree->OperIsBlk() && !tree->OperIsDynBlk()) { - sprintf_s(bufp, sizeof(buf), " %s(%d)", name, tree->AsBlk()->gtBlkSize); + sprintf_s(bufp, sizeof(buf), " %s(%d)", name, tree->AsBlk()->Size()); } else { @@ -9652,6 +9608,23 @@ void Compiler::gtDispNode(GenTree* tree, IndentStack* indentStack, __in __in_z _ if (tree->gtOper != GT_CAST) { printf(" %-6s", varTypeName(tree->TypeGet())); + + StructTypeLayout* layout = nullptr; + + if (tree->OperIsBlk()) + { + layout = tree->AsBlk()->gtLayout; + } + else if (tree->OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR)) + { + layout = lvaGetDesc(tree->AsLclVarCommon())->lvLayout; + } + + if (layout != nullptr) + { + printf("<%d, %s> ", layout->GetSize(), layout->GetClassName()); + } + if (tree->gtOper == GT_LCL_VAR || tree->gtOper == GT_STORE_LCL_VAR) { LclVarDsc* varDsc = &lvaTable[tree->gtLclVarCommon.gtLclNum]; @@ -14432,12 +14405,8 @@ GenTree* Compiler::gtNewTempAssign( if (dstTyp == TYP_UNDEF) { varDsc->lvType = dstTyp = genActualType(valTyp); - if (varTypeIsGC(dstTyp)) - { - varDsc->lvStructGcCount = 1; - } #if FEATURE_SIMD - else if (varTypeIsSIMD(dstTyp)) + if (varTypeIsSIMD(dstTyp)) { varDsc->lvSIMDType = 1; } @@ -15439,7 +15408,7 @@ bool GenTree::DefinesLocal(Compiler* comp, GenTreeLclVarCommon** pLclVarTree, bo if (blkNode != nullptr) { GenTree* destAddr = blkNode->Addr(); - unsigned width = blkNode->gtBlkSize; + unsigned width = blkNode->Size(); // Do we care about whether this assigns the entire variable? if (pIsEntire != nullptr && width == 0) { @@ -16467,7 +16436,7 @@ CORINFO_CLASS_HANDLE Compiler::gtGetStructHandleIfPresent(GenTree* tree) structHnd = impGetRefAnyClass(); break; case GT_OBJ: - structHnd = tree->gtObj.gtClass; + structHnd = tree->gtObj.gtLayout->GetClass(); break; case GT_CALL: structHnd = tree->gtCall.gtRetClsHnd; diff --git a/src/jit/gentree.h b/src/jit/gentree.h index ef3bca26674f..583d55e3ed5d 100644 --- a/src/jit/gentree.h +++ b/src/jit/gentree.h @@ -210,6 +210,159 @@ class AssertionInfo } }; +class StructTypeLayout +{ + const CORINFO_CLASS_HANDLE m_classHandle; + const unsigned m_size; + + unsigned m_isValueClass : 1; + unsigned m_gcPtrsInitialized : 1; + unsigned m_gcPtrCount : 30; + + union { + BYTE* m_gcPtrs; + BYTE m_gcPtrsArray[sizeof(void*)]; + }; + +#ifdef _TARGET_AMD64_ + StructTypeLayout* m_pppQuirkLayout; +#endif + + INDEBUG(const char* m_className;) + +public: + StructTypeLayout(unsigned size) + : m_classHandle(NO_CLASS_HANDLE) + , m_size(size) + , m_isValueClass(false) + , m_gcPtrsInitialized(true) + , m_gcPtrCount(0) + , m_gcPtrs(nullptr) +#ifdef _TARGET_AMD64_ + , m_pppQuirkLayout(nullptr) +#endif +#ifdef DEBUG + , m_className("block") +#endif + { + assert(size != 0); + } + + StructTypeLayout(CORINFO_CLASS_HANDLE classHandle, bool isValueClass, unsigned size DEBUGARG(const char* className)) + : m_classHandle(classHandle) + , m_size(size) + , m_isValueClass(isValueClass) + , m_gcPtrsInitialized(false) + , m_gcPtrCount(0) + , m_gcPtrs(nullptr) +#ifdef _TARGET_AMD64_ + , m_pppQuirkLayout(nullptr) +#endif +#ifdef DEBUG + , m_className(className) +#endif + { + assert(size != 0); + } + + CORINFO_CLASS_HANDLE GetClass() const + { + return m_classHandle; + } + +#ifdef DEBUG + const char* GetClassName() const + { + return m_className; + } +#endif + + bool IsValueClass() const + { + assert(m_classHandle != NO_CLASS_HANDLE); + + return m_isValueClass; + } + + unsigned GetSize() const + { + return m_size; + } + + unsigned GetSlotCount() const + { + return roundUp(m_size, TARGET_POINTER_SIZE) / TARGET_POINTER_SIZE; + } + + bool IsGCInfoInitialized() const + { + return m_gcPtrsInitialized; + } + + void EnsureGCInfoInitialized(Compiler* compiler); + + unsigned GetGCPtrCount() const + { + assert(m_gcPtrsInitialized); + + return m_gcPtrCount; + } + + bool HasGCPtr() const + { + assert(m_gcPtrsInitialized); + + return m_gcPtrCount != 0; + } + +private: + const BYTE* GetGCPtrs() const + { + assert(m_gcPtrsInitialized); + assert(m_classHandle != NO_CLASS_HANDLE); + + return (GetSlotCount() > sizeof(m_gcPtrsArray)) ? m_gcPtrs : m_gcPtrsArray; + } + + CorInfoGCType GetGCPtr(unsigned slot) const + { + assert(m_gcPtrsInitialized); + assert(slot < GetSlotCount()); + + if (m_gcPtrCount == 0) + { + return TYPE_GC_NONE; + } + + return static_cast(GetGCPtrs()[slot]); + } + +public: + bool IsGCPtr(unsigned slot) const + { + return GetGCPtr(slot) != TYPE_GC_NONE; + } + + var_types GetGCPtrType(unsigned slot) const + { + switch (GetGCPtr(slot)) + { + case TYPE_GC_NONE: + return TYP_I_IMPL; + case TYPE_GC_REF: + return TYP_REF; + case TYPE_GC_BYREF: + return TYP_BYREF; + default: + unreached(); + } + } + +#ifdef _TARGET_AMD64_ + StructTypeLayout* GetPPPQuirkLayout(CompAllocator alloc); +#endif // _TARGET_AMD64_ +}; + /*****************************************************************************/ // GT_FIELD nodes will be lowered into more "code-gen-able" representations, like @@ -4572,6 +4725,8 @@ struct GenTreeIndir : public GenTreeOp struct GenTreeBlk : public GenTreeIndir { public: + StructTypeLayout* gtLayout; + // The data to be stored (null for GT_BLK) GenTree*& Data() { @@ -4585,13 +4740,15 @@ struct GenTreeBlk : public GenTreeIndir // The size of the buffer to be copied. unsigned Size() const { - return gtBlkSize; + assert((gtLayout != nullptr) || OperIs(GT_DYN_BLK, GT_STORE_DYN_BLK)); + return (gtLayout != nullptr) ? gtLayout->GetSize() : 0; } - unsigned gtBlkSize; - // Return true iff the object being copied contains one or more GC pointers. - bool HasGCPtr(); + bool HasGCPtr() + { + return OperIs(GT_OBJ, GT_STORE_OBJ) && gtLayout->HasGCPtr(); + } // True if this BlkOpNode is a volatile memory operation. bool IsVolatile() const @@ -4617,20 +4774,22 @@ struct GenTreeBlk : public GenTreeIndir bool gtBlkOpGcUnsafe; - GenTreeBlk(genTreeOps oper, var_types type, GenTree* addr, unsigned size) + GenTreeBlk(genTreeOps oper, var_types type, GenTree* addr, StructTypeLayout* layout) : GenTreeIndir(oper, type, addr, nullptr) - , gtBlkSize(size) + , gtLayout(layout) , gtBlkOpKind(BlkOpKindInvalid) , gtBlkOpGcUnsafe(false) { assert(OperIsBlk(oper)); + assert((layout != nullptr) || OperIs(GT_DYN_BLK, GT_STORE_DYN_BLK)); gtFlags |= (addr->gtFlags & GTF_ALL_EFFECT); } - GenTreeBlk(genTreeOps oper, var_types type, GenTree* addr, GenTree* data, unsigned size) - : GenTreeIndir(oper, type, addr, data), gtBlkSize(size), gtBlkOpKind(BlkOpKindInvalid), gtBlkOpGcUnsafe(false) + GenTreeBlk(genTreeOps oper, var_types type, GenTree* addr, GenTree* data, StructTypeLayout* layout) + : GenTreeIndir(oper, type, addr, data), gtLayout(layout), gtBlkOpKind(BlkOpKindInvalid), gtBlkOpGcUnsafe(false) { assert(OperIsBlk(oper)); + assert((layout != nullptr) || OperIs(GT_DYN_BLK, GT_STORE_DYN_BLK)); gtFlags |= (addr->gtFlags & GTF_ALL_EFFECT); gtFlags |= (data->gtFlags & GTF_ALL_EFFECT); } @@ -4650,84 +4809,19 @@ struct GenTreeBlk : public GenTreeIndir struct GenTreeObj : public GenTreeBlk { - CORINFO_CLASS_HANDLE gtClass; // the class of the object - - // If non-null, this array represents the gc-layout of the class. - // This may be simply copied when cloning this node, because it is not changed once computed. - BYTE* gtGcPtrs; - - // If non-zero, this is the number of slots in the class layout that - // contain gc-pointers. - __declspec(property(get = GetGcPtrCount)) unsigned gtGcPtrCount; - unsigned GetGcPtrCount() const - { - assert(_gtGcPtrCount != UINT32_MAX); - return _gtGcPtrCount; - } - unsigned _gtGcPtrCount; - - // If non-zero, the number of pointer-sized slots that constitutes the class token. - unsigned gtSlots; - - bool IsGCInfoInitialized() - { - return (_gtGcPtrCount != UINT32_MAX); - } - - void SetGCInfo(BYTE* gcPtrs, unsigned gcPtrCount, unsigned slots) - { - gtGcPtrs = gcPtrs; - _gtGcPtrCount = gcPtrCount; - gtSlots = slots; - if (gtGcPtrCount != 0) - { - // We assume that we cannot have a struct with GC pointers that is not a multiple - // of the register size. - // The EE currently does not allow this, but it could change. - // Let's assert it just to be safe. - noway_assert(roundUp(gtBlkSize, REGSIZE_BYTES) == gtBlkSize); - } - else - { - genTreeOps newOper = GT_BLK; - if (gtOper == GT_STORE_OBJ) - { - newOper = GT_STORE_BLK; - } - else - { - assert(gtOper == GT_OBJ); - } - SetOper(newOper); - } - } - - void CopyGCInfo(GenTreeObj* srcObj) - { - if (srcObj->IsGCInfoInitialized()) - { - gtGcPtrs = srcObj->gtGcPtrs; - _gtGcPtrCount = srcObj->gtGcPtrCount; - gtSlots = srcObj->gtSlots; - } - } - - GenTreeObj(var_types type, GenTree* addr, CORINFO_CLASS_HANDLE cls, unsigned size) - : GenTreeBlk(GT_OBJ, type, addr, size), gtClass(cls) + GenTreeObj(var_types type, GenTree* addr, StructTypeLayout* layout) : GenTreeBlk(GT_OBJ, type, addr, layout) { // By default, an OBJ is assumed to be a global reference. gtFlags |= GTF_GLOB_REF; - noway_assert(cls != NO_CLASS_HANDLE); - _gtGcPtrCount = UINT32_MAX; + noway_assert(layout->GetClass() != NO_CLASS_HANDLE); } - GenTreeObj(var_types type, GenTree* addr, GenTree* data, CORINFO_CLASS_HANDLE cls, unsigned size) - : GenTreeBlk(GT_STORE_OBJ, type, addr, data, size), gtClass(cls) + GenTreeObj(var_types type, GenTree* addr, GenTree* data, StructTypeLayout* layout) + : GenTreeBlk(GT_STORE_OBJ, type, addr, data, layout) { // By default, an OBJ is assumed to be a global reference. gtFlags |= GTF_GLOB_REF; - noway_assert(cls != NO_CLASS_HANDLE); - _gtGcPtrCount = UINT32_MAX; + noway_assert(layout->GetClass() != NO_CLASS_HANDLE); } #if DEBUGGABLE_GENTREE @@ -4749,7 +4843,7 @@ struct GenTreeDynBlk : public GenTreeBlk bool gtEvalSizeFirst; GenTreeDynBlk(GenTree* addr, GenTree* dynamicSize) - : GenTreeBlk(GT_DYN_BLK, TYP_STRUCT, addr, 0), gtDynamicSize(dynamicSize), gtEvalSizeFirst(false) + : GenTreeBlk(GT_DYN_BLK, TYP_STRUCT, addr, nullptr), gtDynamicSize(dynamicSize), gtEvalSizeFirst(false) { // Conservatively the 'addr' could be null or point into the global heap. gtFlags |= GTF_EXCEPT | GTF_GLOB_REF; @@ -5038,8 +5132,7 @@ struct GenTreePutArgStk : public GenTreeUnOp #ifdef FEATURE_PUT_STRUCT_ARG_STK , gtPutArgStkKind(Kind::Invalid) , gtNumSlots(numSlots) - , gtNumberReferenceSlots(0) - , gtGcPtrs(nullptr) + , gtLayout(nullptr) #endif // FEATURE_PUT_STRUCT_ARG_STK #if defined(DEBUG) || defined(UNIX_X86_ABI) , gtCall(callNode) @@ -5102,11 +5195,10 @@ struct GenTreePutArgStk : public GenTreeUnOp } //------------------------------------------------------------------------ - // setGcPointers: Sets the number of references and the layout of the struct object returned by the VM. + // setLayout: Set the layout of the stack argument. // // Arguments: - // numPointers - Number of pointer references. - // pointers - layout of the struct (with pointers marked.) + // layout - The struct type layout // // Return Value: // None @@ -5117,10 +5209,11 @@ struct GenTreePutArgStk : public GenTreeUnOp // Otherwise the pointer reference slots are copied atomically in a way that gcinfo is emitted. // Any non pointer references between the pointer reference slots are copied in block fashion. // - void setGcPointers(unsigned numPointers, BYTE* pointers) + void setLayout(StructTypeLayout* layout) { - gtNumberReferenceSlots = numPointers; - gtGcPtrs = pointers; + assert(layout->IsGCInfoInitialized()); + + gtLayout = layout; } // Instruction selection: during codegen time, what code sequence we will be using @@ -5138,9 +5231,8 @@ struct GenTreePutArgStk : public GenTreeUnOp return (gtPutArgStkKind == Kind::Push) || (gtPutArgStkKind == Kind::PushAllSlots); } - unsigned gtNumSlots; // Number of slots for the argument to be passed on stack - unsigned gtNumberReferenceSlots; // Number of reference slots. - BYTE* gtGcPtrs; // gcPointers + unsigned gtNumSlots; // Number of slots for the argument to be passed on stack + StructTypeLayout* gtLayout; #else // !FEATURE_PUT_STRUCT_ARG_STK unsigned getArgSize(); @@ -6470,27 +6562,6 @@ inline var_types& GenTree::CastToType() return this->gtCast.gtCastType; } -//----------------------------------------------------------------------------------- -// HasGCPtr: determine whether this block op involves GC pointers -// -// Arguments: -// None -// -// Return Value: -// Returns true iff the object being copied contains one or more GC pointers. -// -// Notes: -// Of the block nodes, only GT_OBJ and ST_STORE_OBJ are allowed to have GC pointers. -// -inline bool GenTreeBlk::HasGCPtr() -{ - if ((gtOper == GT_OBJ) || (gtOper == GT_STORE_OBJ)) - { - return (AsObj()->gtGcPtrCount != 0); - } - return false; -} - inline bool GenTree::isUsedFromSpillTemp() const { // If spilled and no reg at use, then it is used from the spill temp location rather than being reloaded. diff --git a/src/jit/gschecks.cpp b/src/jit/gschecks.cpp index 8fe14b9d9d8b..236286d79337 100644 --- a/src/jit/gschecks.cpp +++ b/src/jit/gschecks.cpp @@ -427,7 +427,7 @@ void Compiler::gsParamsToShadows() lvaTable[shadowVar].lvLiveAcrossUCall = varDsc->lvLiveAcrossUCall; #endif lvaTable[shadowVar].lvVerTypeInfo = varDsc->lvVerTypeInfo; - lvaTable[shadowVar].lvGcLayout = varDsc->lvGcLayout; + lvaTable[shadowVar].lvLayout = varDsc->lvLayout; lvaTable[shadowVar].lvIsUnsafeBuffer = varDsc->lvIsUnsafeBuffer; lvaTable[shadowVar].lvIsPtr = varDsc->lvIsPtr; diff --git a/src/jit/hwintrinsicArm64.cpp b/src/jit/hwintrinsicArm64.cpp index 31df5b045627..4eebe2cf56ba 100644 --- a/src/jit/hwintrinsicArm64.cpp +++ b/src/jit/hwintrinsicArm64.cpp @@ -367,7 +367,9 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, return nullptr; } - retNode = impSIMDPopStack(getSIMDTypeForSize(simdSizeBytes), /* expectAddr: */ false, sig->retTypeClass); + StructTypeLayout* retTypeLayout = typGetObjLayout(sig->retTypeClass); + + retNode = impSIMDPopStack(getSIMDTypeForSize(simdSizeBytes), /* expectAddr: */ false, retTypeLayout); SetOpLclRelatedToSIMDIntrinsic(retNode); assert(retNode->gtType == getSIMDTypeForSize(getSIMDTypeSizeInBytes(sig->retTypeSigClass))); diff --git a/src/jit/hwintrinsicxarch.cpp b/src/jit/hwintrinsicxarch.cpp index fcc20e04c642..db2aa4321e2b 100644 --- a/src/jit/hwintrinsicxarch.cpp +++ b/src/jit/hwintrinsicxarch.cpp @@ -1054,7 +1054,9 @@ GenTree* Compiler::impBaseIntrinsic(NamedIntrinsic intrinsic, assert(sig->numArgs == 1); - retNode = impSIMDPopStack(retType, /* expectAddr: */ false, sig->retTypeClass); + StructTypeLayout* retTypeLayout = typGetObjLayout(sig->retTypeClass); + + retNode = impSIMDPopStack(retType, /* expectAddr: */ false, retTypeLayout); SetOpLclRelatedToSIMDIntrinsic(retNode); assert(retNode->gtType == getSIMDTypeForSize(getSIMDTypeSizeInBytes(sig->retTypeSigClass))); break; diff --git a/src/jit/importer.cpp b/src/jit/importer.cpp index 439938185603..4422aa3751af 100644 --- a/src/jit/importer.cpp +++ b/src/jit/importer.cpp @@ -627,8 +627,7 @@ inline void Compiler::impAppendStmt(GenTree* stmt, unsigned chkLevel) } } else if ((lhs->OperIsBlk() && !lhs->AsBlk()->HasGCPtr()) || - ((lhs->OperGet() == GT_LCL_VAR) && - (lvaTable[lhs->AsLclVarCommon()->gtLclNum].lvStructGcCount == 0))) + (lhs->OperIs(GT_LCL_VAR) && !lvaTypeIsGC(lhs->AsLclVar()->GetLclNum()))) { // TODO-1stClassStructs: Previously, spillGlobEffects was set to true for // GT_INITBLK and GT_COPYBLK, but this is overly conservative, and should be @@ -1311,7 +1310,7 @@ GenTree* Compiler::impAssignStructPtr(GenTree* destAddr, asgType = impNormStructType(structHnd); if (src->gtOper == GT_OBJ) { - assert(src->gtObj.gtClass == structHnd); + assert(src->gtObj.gtLayout->GetClass() == structHnd); } } else if (src->gtOper == GT_INDEX) @@ -1393,7 +1392,7 @@ GenTree* Compiler::impAssignStructPtr(GenTree* destAddr, } else if (varTypeIsStruct(asgType)) { - dest = new (this, GT_BLK) GenTreeBlk(GT_BLK, asgType, destAddr, genTypeSize(asgType)); + dest = new (this, GT_BLK) GenTreeBlk(GT_BLK, asgType, destAddr, typGetBlkLayout(genTypeSize(asgType))); } else { @@ -1441,7 +1440,7 @@ GenTree* Compiler::impGetStructAddr(GenTree* structVal, if (oper == GT_OBJ && willDeref) { - assert(structVal->gtObj.gtClass == structHnd); + assert(structVal->gtObj.gtLayout->GetClass() == structHnd); return (structVal->gtObj.Addr()); } else if (oper == GT_CALL || oper == GT_RET_EXPR || oper == GT_OBJ || oper == GT_MKREFANY || @@ -1483,36 +1482,23 @@ GenTree* Compiler::impGetStructAddr(GenTree* structVal, } //------------------------------------------------------------------------ -// impNormStructType: Given a (known to be) struct class handle structHnd, normalize its type, -// and optionally determine the GC layout of the struct. +// impNormStructType: Given a (known to be) struct class handle structHnd and normalize its type. // // Arguments: // structHnd - The class handle for the struct type of interest. -// gcLayout - (optional, default nullptr) - a BYTE pointer, allocated by the caller, -// into which the gcLayout will be written. -// pNumGCVars - (optional, default nullptr) - if non-null, a pointer to an unsigned, -// which will be set to the number of GC fields in the struct. // pSimdBaseType - (optional, default nullptr) - if non-null, and the struct is a SIMD // type, set to the SIMD base type // // Return Value: // The JIT type for the struct (e.g. TYP_STRUCT, or TYP_SIMD*). -// The gcLayout will be returned using the pointers provided by the caller, if non-null. // It may also modify the compFloatingPointUsed flag if the type is a SIMD type. // -// Assumptions: -// The caller must set gcLayout to nullptr OR ensure that it is large enough -// (see ICorStaticInfo::getClassGClayout in corinfo.h). -// // Notes: // Normalizing the type involves examining the struct type to determine if it should // be modified to one that is handled specially by the JIT, possibly being a candidate // for full enregistration, e.g. TYP_SIMD16. -var_types Compiler::impNormStructType(CORINFO_CLASS_HANDLE structHnd, - BYTE* gcLayout, - unsigned* pNumGCVars, - var_types* pSimdBaseType) +var_types Compiler::impNormStructType(CORINFO_CLASS_HANDLE structHnd, var_types* pSimdBaseType) { assert(structHnd != NO_CLASS_HANDLE); @@ -1551,31 +1537,6 @@ var_types Compiler::impNormStructType(CORINFO_CLASS_HANDLE structHnd, } #endif // FEATURE_SIMD - // Fetch GC layout info if requested - if (gcLayout != nullptr) - { - unsigned numGCVars = info.compCompHnd->getClassGClayout(structHnd, gcLayout); - - // Verify that the quick test up above via the class attributes gave a - // safe view of the type's GCness. - // - // Note there are cases where mayContainGCPtrs is true but getClassGClayout - // does not report any gc fields. - - assert(mayContainGCPtrs || (numGCVars == 0)); - - if (pNumGCVars != nullptr) - { - *pNumGCVars = numGCVars; - } - } - else - { - // Can't safely ask for number of GC pointers without also - // asking for layout. - assert(pNumGCVars == nullptr); - } - return structType; } @@ -9162,7 +9123,6 @@ GenTree* Compiler::impFixupStructReturnType(GenTree* op, CORINFO_CLASS_HANDLE re op = op1->gtOp.gtOp1; goto REDO_RETURN_NODE; } - op->gtObj.gtClass = NO_CLASS_HANDLE; op->ChangeOperUnchecked(GT_IND); op->gtFlags |= GTF_IND_TGTANYWHERE; } @@ -15668,7 +15628,7 @@ void Compiler::impImportBlockCode(BasicBlock* block) if (op3->IsCnsIntOrI()) { size = (unsigned)op3->AsIntConCommon()->IconValue(); - op1 = new (this, GT_BLK) GenTreeBlk(GT_BLK, TYP_STRUCT, op1, size); + op1 = new (this, GT_BLK) GenTreeBlk(GT_BLK, TYP_STRUCT, op1, typGetBlkLayout(size)); } else { @@ -15692,7 +15652,7 @@ void Compiler::impImportBlockCode(BasicBlock* block) if (op3->IsCnsIntOrI()) { size = (unsigned)op3->AsIntConCommon()->IconValue(); - op1 = new (this, GT_BLK) GenTreeBlk(GT_BLK, TYP_STRUCT, op1, size); + op1 = new (this, GT_BLK) GenTreeBlk(GT_BLK, TYP_STRUCT, op1, typGetBlkLayout(size)); } else { diff --git a/src/jit/lclvars.cpp b/src/jit/lclvars.cpp index b9cec032b210..30e8adaa9970 100644 --- a/src/jit/lclvars.cpp +++ b/src/jit/lclvars.cpp @@ -400,7 +400,7 @@ void Compiler::lvaInitThisPtr(InitVarDscInfo* varDscInfo) if (featureSIMD) { var_types simdBaseType = TYP_UNKNOWN; - var_types type = impNormStructType(info.compClassHnd, nullptr, nullptr, &simdBaseType); + var_types type = impNormStructType(info.compClassHnd, &simdBaseType); if (simdBaseType != TYP_UNKNOWN) { assert(varTypeIsSIMD(type)); @@ -1258,11 +1258,6 @@ void Compiler::lvaInitVarDsc(LclVarDsc* varDsc, varDsc->lvOverlappingFields = StructHasOverlappingFields(cFlags); } - if (varTypeIsGC(type)) - { - varDsc->lvStructGcCount = 1; - } - // Set the lvType (before this point it is TYP_UNDEF). if ((varTypeIsStruct(type))) { @@ -2458,41 +2453,15 @@ void Compiler::lvaSetStruct(unsigned varNum, CORINFO_CLASS_HANDLE typeHnd, bool } if (varDsc->lvExactSize == 0) { - BOOL isValueClass = info.compCompHnd->isValueClass(typeHnd); - - if (isValueClass) - { - varDsc->lvExactSize = info.compCompHnd->getClassSize(typeHnd); - } - else - { - varDsc->lvExactSize = info.compCompHnd->getHeapClassSize(typeHnd); - } - - size_t lvSize = varDsc->lvSize(); - assert((lvSize % TARGET_POINTER_SIZE) == - 0); // The struct needs to be a multiple of TARGET_POINTER_SIZE bytes for getClassGClayout() to be valid. - varDsc->lvGcLayout = getAllocator(CMK_LvaTable).allocate(lvSize / TARGET_POINTER_SIZE); - unsigned numGCVars; - var_types simdBaseType = TYP_UNKNOWN; - if (isValueClass) - { - varDsc->lvType = impNormStructType(typeHnd, varDsc->lvGcLayout, &numGCVars, &simdBaseType); - } - else - { - numGCVars = info.compCompHnd->getClassGClayout(typeHnd, varDsc->lvGcLayout); - } - - // We only save the count of GC vars in a struct up to 7. - if (numGCVars >= 8) - { - numGCVars = 7; - } - varDsc->lvStructGcCount = numGCVars; + StructTypeLayout* layout = typGetObjLayout(typeHnd); + layout->EnsureGCInfoInitialized(this); + varDsc->lvLayout = layout; + varDsc->lvExactSize = layout->GetSize(); - if (isValueClass) + if (layout->IsValueClass()) { + var_types simdBaseType = TYP_UNKNOWN; + varDsc->lvType = impNormStructType(typeHnd, &simdBaseType); #if FEATURE_SIMD if (simdBaseType != TYP_UNKNOWN) { @@ -2512,7 +2481,7 @@ void Compiler::lvaSetStruct(unsigned varNum, CORINFO_CLASS_HANDLE typeHnd, bool varDsc->lvSetHfaTypeIsFloat(hfaType == TYP_FLOAT); // hfa variables can never contain GC pointers - assert(varDsc->lvStructGcCount == 0); + assert(!layout->HasGCPtr()); // The size of this struct should be evenly divisible by 4 or 8 assert((varDsc->lvExactSize % genTypeSize(hfaType)) == 0); // The number of elements in the HFA should fit into our MAX_ARG_REG_COUNT limit @@ -2776,17 +2745,6 @@ void Compiler::lvaUpdateClass(unsigned varNum, GenTree* tree, CORINFO_CLASS_HAND } } -/***************************************************************************** - * Returns the array of BYTEs containing the GC layout information - */ - -BYTE* Compiler::lvaGetGcLayout(unsigned varNum) -{ - assert(varTypeIsStruct(lvaTable[varNum].lvType) && (lvaTable[varNum].lvExactSize >= TARGET_POINTER_SIZE)); - - return lvaTable[varNum].lvGcLayout; -} - //------------------------------------------------------------------------ // lvaLclSize: returns size of a local variable, in bytes // @@ -3556,25 +3514,8 @@ var_types LclVarDsc::lvaArgType() type = TYP_INT; break; case 8: - switch (*lvGcLayout) - { - case TYPE_GC_NONE: - type = TYP_I_IMPL; - break; - - case TYPE_GC_REF: - type = TYP_REF; - break; - - case TYPE_GC_BYREF: - type = TYP_BYREF; - break; - - default: - unreached(); - } + type = lvLayout->GetGCPtrType(0); break; - default: type = TYP_BYREF; break; diff --git a/src/jit/lower.cpp b/src/jit/lower.cpp index fc507c4c5020..a31d4c8d40df 100644 --- a/src/jit/lower.cpp +++ b/src/jit/lower.cpp @@ -1058,27 +1058,15 @@ GenTree* Lowering::NewPutArg(GenTreeCall* call, GenTree* arg, fgArgTabEntry* inf if (arg->OperGet() == GT_OBJ) { - BYTE* gcLayout = nullptr; - unsigned numRefs = 0; - GenTreeObj* argObj = arg->AsObj(); - - if (argObj->IsGCInfoInitialized()) - { - gcLayout = argObj->gtGcPtrs; - numRefs = argObj->GetGcPtrCount(); - } - else - { - // Set GC Pointer info - gcLayout = new (comp, CMK_Codegen) BYTE[info->numSlots + info->numRegs]; - numRefs = comp->info.compCompHnd->getClassGClayout(arg->gtObj.gtClass, gcLayout); - argSplit->setGcPointers(numRefs, gcLayout); - } + GenTreeObj* argObj = arg->AsObj(); + StructTypeLayout* layout = argObj->gtLayout; + layout->EnsureGCInfoInitialized(comp); + argSplit->setLayout(layout); // Set type of registers for (unsigned index = 0; index < info->numRegs; index++) { - var_types regType = comp->getJitGCType(gcLayout[index]); + var_types regType = layout->GetGCPtrType(index); // Account for the possibility that float fields may be passed in integer registers. if (varTypeIsFloating(regType) && !genIsValidFloatReg(argSplit->GetRegNumByIdx(index))) { @@ -1185,19 +1173,19 @@ GenTree* Lowering::NewPutArg(GenTreeCall* call, GenTree* arg, fgArgTabEntry* inf // pair copying using XMM registers or rep mov instructions. if (info->isStruct) { + putArg->AsPutArgStk()->setLayout(info->layout); + // We use GT_OBJ only for non-lclVar, non-SIMD, non-FIELD_LIST struct arguments. if (arg->OperIsLocal()) { // This must have a type with a known size (SIMD or has been morphed to a primitive type). assert(arg->TypeGet() != TYP_STRUCT); + // assert(comp->lvaGetDesc(arg->AsLclVar())->lvLayout == info->layout); } else if (arg->OperIs(GT_OBJ)) { - unsigned numRefs = 0; - BYTE* gcLayout = new (comp, CMK_Codegen) BYTE[info->numSlots]; assert(!varTypeIsSIMD(arg)); - numRefs = comp->info.compCompHnd->getClassGClayout(arg->gtObj.gtClass, gcLayout); - putArg->AsPutArgStk()->setGcPointers(numRefs, gcLayout); + assert(arg->AsObj()->gtLayout == info->layout); #ifdef _TARGET_X86_ // On x86 VM lies about the type of a struct containing a pointer sized @@ -1383,6 +1371,8 @@ void Lowering::LowerArg(GenTreeCall* call, GenTree** ppArg) GenTree* putArg = NewPutArg(call, fieldList, info, type); putArg->gtRegNum = info->regNum; + putArg->AsPutArgStk()->setLayout(comp->typGetBlkLayout(8)); + // We can't call ReplaceArgWithPutArgOrBitcast here because it presumes that we are keeping the original // arg. BlockRange().InsertBefore(arg, fieldList, putArg); @@ -1409,6 +1399,13 @@ void Lowering::LowerArg(GenTreeCall* call, GenTree** ppArg) GenTree* putArg = NewPutArg(call, arg, info, type); +#ifdef FEATURE_PUT_STRUCT_ARG_STK + if (putArg->OperIs(GT_PUTARG_STK) && info->isStruct) + { + putArg->AsPutArgStk()->setLayout(info->layout); + } +#endif + // In the case of register passable struct (in one or two registers) // the NewPutArg returns a new node (GT_PUTARG_REG or a GT_FIELD_LIST with two GT_PUTARG_REGs.) // If an extra node is returned, splice it in the right place in the tree. diff --git a/src/jit/lowerarmarch.cpp b/src/jit/lowerarmarch.cpp index 4b28ce260a1f..931d7f226fc5 100644 --- a/src/jit/lowerarmarch.cpp +++ b/src/jit/lowerarmarch.cpp @@ -226,7 +226,7 @@ void Lowering::LowerStoreIndir(GenTreeIndir* node) void Lowering::LowerBlockStore(GenTreeBlk* blkNode) { GenTree* dstAddr = blkNode->Addr(); - unsigned size = blkNode->gtBlkSize; + unsigned size = blkNode->Size(); GenTree* source = blkNode->Data(); Compiler* compiler = comp; @@ -237,7 +237,8 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) if (!isInitBlk) { // CopyObj or CopyBlk - if ((blkNode->OperGet() == GT_STORE_OBJ) && ((blkNode->AsObj()->gtGcPtrCount == 0) || blkNode->gtBlkOpGcUnsafe)) + if ((blkNode->OperGet() == GT_STORE_OBJ) && + (!blkNode->AsObj()->gtLayout->HasGCPtr() || blkNode->gtBlkOpGcUnsafe)) { blkNode->SetOper(GT_STORE_BLK); } @@ -305,17 +306,16 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) // CopyObj GenTreeObj* objNode = blkNode->AsObj(); - unsigned slots = objNode->gtSlots; + unsigned slots = objNode->gtLayout->GetSlotCount(); #ifdef DEBUG // CpObj must always have at least one GC-Pointer as a member. - assert(objNode->gtGcPtrCount > 0); + assert(objNode->gtLayout->HasGCPtr()); assert(dstAddr->gtType == TYP_BYREF || dstAddr->gtType == TYP_I_IMPL); - CORINFO_CLASS_HANDLE clsHnd = objNode->gtClass; - size_t classSize = compiler->info.compCompHnd->getClassSize(clsHnd); - size_t blkSize = roundUp(classSize, TARGET_POINTER_SIZE); + size_t classSize = objNode->gtLayout->GetSize(); + size_t blkSize = roundUp(classSize, TARGET_POINTER_SIZE); // Currently, the EE always round up a class data structure so // we are not handling the case where we have a non multiple of pointer sized diff --git a/src/jit/lowerxarch.cpp b/src/jit/lowerxarch.cpp index 292fb93c537d..4d467c880a68 100644 --- a/src/jit/lowerxarch.cpp +++ b/src/jit/lowerxarch.cpp @@ -146,7 +146,7 @@ void Lowering::LowerStoreIndir(GenTreeIndir* node) void Lowering::LowerBlockStore(GenTreeBlk* blkNode) { GenTree* dstAddr = blkNode->Addr(); - unsigned size = blkNode->gtBlkSize; + unsigned size = blkNode->Size(); GenTree* source = blkNode->Data(); Compiler* compiler = comp; GenTree* srcAddrOrFill = nullptr; @@ -155,7 +155,8 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) if (!isInitBlk) { // CopyObj or CopyBlk - if ((blkNode->OperGet() == GT_STORE_OBJ) && ((blkNode->AsObj()->gtGcPtrCount == 0) || blkNode->gtBlkOpGcUnsafe)) + if ((blkNode->OperGet() == GT_STORE_OBJ) && + (!blkNode->AsObj()->gtLayout->HasGCPtr() || blkNode->gtBlkOpGcUnsafe)) { blkNode->SetOper(GT_STORE_BLK); } @@ -248,15 +249,15 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) GenTreeObj* cpObjNode = blkNode->AsObj(); - unsigned slots = cpObjNode->gtSlots; + unsigned slots = cpObjNode->gtLayout->GetSlotCount(); #ifdef DEBUG // CpObj must always have at least one GC-Pointer as a member. - assert(cpObjNode->gtGcPtrCount > 0); + assert(cpObjNode->gtLayout->HasGCPtr()); assert(dstAddr->gtType == TYP_BYREF || dstAddr->gtType == TYP_I_IMPL); - CORINFO_CLASS_HANDLE clsHnd = cpObjNode->gtClass; + CORINFO_CLASS_HANDLE clsHnd = cpObjNode->gtLayout->GetClass(); size_t classSize = comp->info.compCompHnd->getClassSize(clsHnd); size_t blkSize = roundUp(classSize, TARGET_POINTER_SIZE); @@ -280,20 +281,20 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) // Let's inspect the struct/class layout and determine if it's profitable // to use rep movsq for copying non-gc memory instead of using single movsq // instructions for each memory slot. - unsigned i = 0; - BYTE* gcPtrs = cpObjNode->gtGcPtrs; + unsigned i = 0; + StructTypeLayout* layout = cpObjNode->gtLayout; do { unsigned nonGCSlots = 0; // Measure a contiguous non-gc area inside the struct and note the maximum. - while (i < slots && gcPtrs[i] == TYPE_GC_NONE) + while ((i < slots) && !layout->IsGCPtr(i)) { nonGCSlots++; i++; } - while (i < slots && gcPtrs[i] != TYPE_GC_NONE) + while ((i < slots) && layout->IsGCPtr(i)) { i++; } @@ -413,8 +414,8 @@ void Lowering::LowerPutArgStk(GenTreePutArgStk* putArgStk) #ifdef _TARGET_X86_ if (putArgStk->gtOp1->gtOper == GT_FIELD_LIST) { - putArgStk->gtNumberReferenceSlots = 0; - putArgStk->gtPutArgStkKind = GenTreePutArgStk::Kind::Invalid; + unsigned gcPtrCount = 0; + putArgStk->gtPutArgStkKind = GenTreePutArgStk::Kind::Invalid; GenTreeFieldList* fieldList = putArgStk->gtOp1->AsFieldList(); @@ -503,7 +504,7 @@ void Lowering::LowerPutArgStk(GenTreePutArgStk* putArgStk) if (varTypeIsGC(fieldType)) { - putArgStk->gtNumberReferenceSlots++; + gcPtrCount++; } // For x86 we must mark all integral fields as contained or reg-optional, and handle them @@ -541,6 +542,8 @@ void Lowering::LowerPutArgStk(GenTreePutArgStk* putArgStk) prevOffset = fieldOffset; } + assert((gcPtrCount == 0) || (gcPtrCount == putArgStk->gtLayout->GetGCPtrCount())); + // Set the copy kind. // TODO-X86-CQ: Even if we are using push, if there are contiguous floating point fields, we should // adjust the stack once for those fields. The latter is really best done in code generation, but @@ -625,7 +628,7 @@ void Lowering::LowerPutArgStk(GenTreePutArgStk* putArgStk) // If we have a buffer between XMM_REGSIZE_BYTES and CPBLK_UNROLL_LIMIT bytes, we'll use SSE2. // Structs and buffer with sizes <= CPBLK_UNROLL_LIMIT bytes are occurring in more than 95% of // our framework assemblies, so this is the main code generation scheme we'll use. - if (size <= CPBLK_UNROLL_LIMIT && putArgStk->gtNumberReferenceSlots == 0) + if ((size <= CPBLK_UNROLL_LIMIT) && !putArgStk->gtLayout->HasGCPtr()) { #ifdef _TARGET_X86_ if (size < XMM_REGSIZE_BYTES) @@ -639,7 +642,7 @@ void Lowering::LowerPutArgStk(GenTreePutArgStk* putArgStk) } } #ifdef _TARGET_X86_ - else if (putArgStk->gtNumberReferenceSlots != 0) + else if (putArgStk->gtLayout->HasGCPtr()) { // On x86, we must use `push` to store GC references to the stack in order for the emitter to properly update // the function's GC info. These `putargstk` nodes will generate a sequence of `push` instructions. diff --git a/src/jit/lsraarmarch.cpp b/src/jit/lsraarmarch.cpp index 611d858d6d8a..b066788b092b 100644 --- a/src/jit/lsraarmarch.cpp +++ b/src/jit/lsraarmarch.cpp @@ -570,7 +570,7 @@ int LinearScan::BuildPutArgSplit(GenTreePutArgSplit* argNode) int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) { GenTree* dstAddr = blkNode->Addr(); - unsigned size = blkNode->gtBlkSize; + unsigned size = blkNode->Size(); GenTree* source = blkNode->Data(); int srcCount = 0; diff --git a/src/jit/lsrabuild.cpp b/src/jit/lsrabuild.cpp index 38dfca69859c..5edea719a3d6 100644 --- a/src/jit/lsrabuild.cpp +++ b/src/jit/lsrabuild.cpp @@ -906,7 +906,7 @@ regMaskTP LinearScan::getKillSetForBlockStore(GenTreeBlk* blkNode) if ((blkNode->OperGet() == GT_STORE_OBJ) && blkNode->OperIsCopyBlkOp()) { - assert(blkNode->AsObj()->gtGcPtrCount != 0); + assert(blkNode->AsObj()->gtLayout->HasGCPtr()); killMask = compiler->compHelperCallKillSet(CORINFO_HELP_ASSIGN_BYREF); } else @@ -3186,7 +3186,7 @@ int LinearScan::BuildPutArgReg(GenTreeUnOp* node) { GenTreeObj* obj = op1->AsObj(); GenTree* addr = obj->Addr(); - unsigned size = obj->gtBlkSize; + unsigned size = obj->Size(); assert(size <= TARGET_POINTER_SIZE); if (addr->OperIsLocalAddr()) { diff --git a/src/jit/lsraxarch.cpp b/src/jit/lsraxarch.cpp index 364a4b7103a2..f5e37310633b 100644 --- a/src/jit/lsraxarch.cpp +++ b/src/jit/lsraxarch.cpp @@ -1280,7 +1280,7 @@ int LinearScan::BuildCall(GenTreeCall* call) int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) { GenTree* dstAddr = blkNode->Addr(); - unsigned size = blkNode->gtBlkSize; + unsigned size = blkNode->Size(); GenTree* source = blkNode->Data(); int srcCount = 0; @@ -1497,6 +1497,7 @@ int LinearScan::BuildPutArgStk(GenTreePutArgStk* putArgStk) RefPosition* simdTemp = nullptr; RefPosition* intTemp = nullptr; unsigned prevOffset = putArgStk->getArgSize(); + unsigned gcPtrCount = 0; // We need to iterate over the fields twice; once to determine the need for internal temps, // and once to actually build the uses. for (GenTreeFieldList* current = putArgStk->gtOp1->AsFieldList(); current != nullptr; current = current->Rest()) @@ -1542,11 +1543,13 @@ int LinearScan::BuildPutArgStk(GenTreePutArgStk* putArgStk) if (varTypeIsGC(fieldType)) { - putArgStk->gtNumberReferenceSlots++; + gcPtrCount++; } prevOffset = fieldOffset; } + assert((gcPtrCount == 0) || (putArgStk->gtLayout->GetGCPtrCount() == gcPtrCount)); + for (GenTreeFieldList* current = putArgStk->gtOp1->AsFieldList(); current != nullptr; current = current->Rest()) { GenTree* const fieldNode = current->Current(); @@ -1598,7 +1601,7 @@ int LinearScan::BuildPutArgStk(GenTreePutArgStk* putArgStk) // x86 specific note: if the size is odd, the last copy operation would be of size 1 byte. // But on x86 only RBM_BYTE_REGS could be used as byte registers. Therefore, exclude // RBM_NON_BYTE_REGS from internal candidates. - if ((putArgStk->gtNumberReferenceSlots == 0) && (size & (XMM_REGSIZE_BYTES - 1)) != 0) + if (!putArgStk->gtLayout->HasGCPtr() && (size & (XMM_REGSIZE_BYTES - 1)) != 0) { regMaskTP regMask = allRegs(TYP_INT); diff --git a/src/jit/morph.cpp b/src/jit/morph.cpp index 48d97cfe4c63..157ee89c6e82 100644 --- a/src/jit/morph.cpp +++ b/src/jit/morph.cpp @@ -1617,9 +1617,8 @@ void fgArgInfo::ArgsComplete() // else if (argx->OperGet() == GT_OBJ) { - GenTreeObj* argObj = argx->AsObj(); - CORINFO_CLASS_HANDLE objClass = argObj->gtClass; - unsigned structSize = compiler->info.compCompHnd->getClassSize(objClass); + GenTreeObj* argObj = argx->AsObj(); + unsigned structSize = argObj->gtLayout->GetSize(); switch (structSize) { case 3: @@ -3164,7 +3163,6 @@ void Compiler::fgInitArgInfo(GenTreeCall* call) unsigned structSize = 0; bool passStructByRef = false; - bool isStructArg; GenTree* actualArg = argx->gtEffectiveVal(true /* Commas only */); // @@ -3172,8 +3170,9 @@ void Compiler::fgInitArgInfo(GenTreeCall* call) // TARGET_POINTER_SIZE stack slots, or the sum of these if the argument is split between the registers and // the stack. // - isStructArg = varTypeIsStruct(argx); - CORINFO_CLASS_HANDLE objClass = NO_CLASS_HANDLE; + const bool isStructArg = varTypeIsStruct(argx); + CORINFO_CLASS_HANDLE objClass = NO_CLASS_HANDLE; + StructTypeLayout* layout = nullptr; if (isStructArg) { objClass = gtGetStructHandle(argx); @@ -3183,15 +3182,14 @@ void Compiler::fgInitArgInfo(GenTreeCall* call) switch (actualArg->OperGet()) { case GT_OBJ: - // Get the size off the OBJ node. - structSize = actualArg->AsObj()->gtBlkSize; - assert(structSize == info.compCompHnd->getClassSize(objClass)); + layout = actualArg->AsObj()->gtLayout; break; case GT_LCL_VAR: - structSize = lvaGetDesc(actualArg->AsLclVarCommon())->lvExactSize; + layout = lvaGetDesc(actualArg->AsLclVar())->lvLayout; + assert(layout->GetSize() == lvaGetDesc(actualArg->AsLclVar())->lvExactSize); break; case GT_MKREFANY: - structSize = info.compCompHnd->getClassSize(objClass); + layout = typGetObjLayout(objClass); break; default: BADCODE("illegal argument tree in fgInitArgInfo"); @@ -3200,20 +3198,24 @@ void Compiler::fgInitArgInfo(GenTreeCall* call) } else { - structSize = genTypeSize(argx); - assert(structSize == info.compCompHnd->getClassSize(objClass)); + layout = typGetObjLayout(objClass); + assert(layout->GetSize() == genTypeSize(argx)); } + + layout->EnsureGCInfoInitialized(this); + structSize = layout->GetSize(); } + #if defined(_TARGET_AMD64_) #ifdef UNIX_AMD64_ABI - if (!isStructArg) + if (isStructArg) { - size = 1; // On AMD64, all primitives fit in a single (64-bit) 'slot' + size = layout->GetSlotCount(); + eeGetSystemVAmd64PassStructInRegisterDescriptor(objClass, &structDesc); } else { - size = (unsigned)(roundUp(structSize, TARGET_POINTER_SIZE)) / TARGET_POINTER_SIZE; - eeGetSystemVAmd64PassStructInRegisterDescriptor(objClass, &structDesc); + size = 1; // On AMD64, all primitives fit in a single (64-bit) 'slot' } #else // !UNIX_AMD64_ABI size = 1; // On AMD64 Windows, all args fit in a single (64-bit) 'slot' @@ -3234,7 +3236,7 @@ void Compiler::fgInitArgInfo(GenTreeCall* call) // if sufficient registers are available. // Structs that are larger than 2 pointers (except for HFAs) are passed by // reference (to a copy) - size = (unsigned)(roundUp(structSize, TARGET_POINTER_SIZE)) / TARGET_POINTER_SIZE; + size = layout->GetSlotCount(); if (size > 2) { @@ -3251,7 +3253,7 @@ void Compiler::fgInitArgInfo(GenTreeCall* call) #elif defined(_TARGET_ARM_) || defined(_TARGET_X86_) if (isStructArg) { - size = (unsigned)(roundUp(structSize, TARGET_POINTER_SIZE)) / TARGET_POINTER_SIZE; + size = layout->GetSlotCount(); } else { @@ -3650,6 +3652,10 @@ void Compiler::fgInitArgInfo(GenTreeCall* call) newArgEntry->argType = argx->TypeGet(); } +#ifdef FEATURE_PUT_STRUCT_ARG_STK + newArgEntry->layout = layout; +#endif + argSlots += size; } // end foreach argument loop @@ -3826,7 +3832,7 @@ GenTreeCall* Compiler::fgMorphArgs(GenTreeCall* call) if (argObj->OperIs(GT_OBJ)) { // Get the size off the OBJ node. - originalSize = argObj->AsObj()->gtBlkSize; + originalSize = argObj->AsObj()->gtLayout->GetSize(); assert(originalSize == info.compCompHnd->getClassSize(objClass)); } else @@ -4503,9 +4509,8 @@ GenTree* Compiler::fgMorphMultiregStructArg(GenTree* arg, fgArgTabEntry* fgEntry if (arg->OperGet() == GT_OBJ) { GenTreeObj* argObj = arg->AsObj(); - objClass = argObj->gtClass; - structSize = argObj->Size(); - assert(structSize == info.compCompHnd->getClassSize(objClass)); + objClass = argObj->gtLayout->GetClass(); + structSize = argObj->gtLayout->GetSize(); // If we have a GT_OBJ of a GT_ADDR then we set argValue to the child node of the GT_ADDR. GenTree* op1 = argObj->gtOp1; @@ -4685,14 +4690,12 @@ GenTree* Compiler::fgMorphMultiregStructArg(GenTree* arg, fgArgTabEntry* fgEntry for (unsigned inx = 0; inx < elemCount; inx++) { - CorInfoGCType currentGcLayoutType = (CorInfoGCType)varDsc->lvGcLayout[inx]; - // We setup the type[inx] value above using the GC info from 'objClass' // This GT_LCL_VAR must have the same GC layout info // - if (currentGcLayoutType != TYPE_GC_NONE) + if (varDsc->lvLayout->IsGCPtr(inx)) { - noway_assert(type[inx] == getJitGCType((BYTE)currentGcLayoutType)); + noway_assert(type[inx] == varDsc->lvLayout->GetGCPtrType(inx)); } else { @@ -4846,21 +4849,21 @@ GenTree* Compiler::fgMorphMultiregStructArg(GenTree* arg, fgArgTabEntry* fgEntry // The allocated size of our LocalVar must be at least as big as lastOffset assert(varDsc->lvSize() >= lastOffset); - if (varDsc->lvStructGcCount > 0) + if (varDsc->lvLayout->HasGCPtr()) { // alignment of the baseOffset is required noway_assert((baseOffset % TARGET_POINTER_SIZE) == 0); #ifndef UNIX_AMD64_ABI noway_assert(elemSize == TARGET_POINTER_SIZE); #endif - unsigned baseIndex = baseOffset / TARGET_POINTER_SIZE; - const BYTE* gcPtrs = varDsc->lvGcLayout; // Get the GC layout for the local variable + unsigned baseIndex = baseOffset / TARGET_POINTER_SIZE; + StructTypeLayout* layout = varDsc->lvLayout; for (unsigned inx = 0; (inx < elemCount); inx++) { // The GC information must match what we setup using 'objClass' - if ((gcPtrs[baseIndex + inx] != TYPE_GC_NONE) || varTypeGCtype(type[inx])) + if (layout->IsGCPtr(baseIndex + inx) || varTypeIsGC(type[inx])) { - noway_assert(type[inx] == getJitGCType(gcPtrs[baseIndex + inx])); + noway_assert(type[inx] == layout->GetGCPtrType(baseIndex + inx)); } } } @@ -7063,7 +7066,7 @@ bool Compiler::fgCanFastTailCall(GenTreeCall* callee) if (argx->OperGet() == GT_OBJ) { - objClass = argx->AsObj()->gtClass; + objClass = argx->AsObj()->gtLayout->GetClass(); } else if (argx->IsLocal()) { @@ -7858,7 +7861,7 @@ void Compiler::fgMorphRecursiveFastTailCallIntoLoop(BasicBlock* block, GenTreeCa { var_types lclType = varDsc->TypeGet(); bool isUserLocal = (varNum < info.compLocalsCount); - bool structWithGCFields = ((lclType == TYP_STRUCT) && (varDsc->lvStructGcCount > 0)); + bool structWithGCFields = ((lclType == TYP_STRUCT) && varDsc->lvLayout->HasGCPtr()); if (isUserLocal || structWithGCFields) { GenTree* lcl = gtNewLclvNode(varNum, lclType); @@ -8988,8 +8991,8 @@ GenTree* Compiler::fgMorphOneAsgBlockOp(GenTree* tree) bool isCopyBlock = asg->OperIsCopyBlkOp(); bool isInitBlock = !isCopyBlock; - unsigned size; - CORINFO_CLASS_HANDLE clsHnd = NO_CLASS_HANDLE; + unsigned size; + StructTypeLayout* layout = nullptr; #ifdef FEATURE_SIMD // importer introduces cpblk nodes with src = GT_ADDR(GT_SIMD/GT_HWIntrinsic) // The SIMD type in question could be Vector2f which is 8-bytes in size. @@ -9010,16 +9013,13 @@ GenTree* Compiler::fgMorphOneAsgBlockOp(GenTree* tree) if (dest->gtEffectiveVal()->OperIsBlk()) { GenTreeBlk* lhsBlk = dest->gtEffectiveVal()->AsBlk(); + layout = lhsBlk->gtLayout; size = lhsBlk->Size(); if (impIsAddressInLocal(lhsBlk->Addr(), &destLclVarTree)) { destVarNum = destLclVarTree->AsLclVarCommon()->gtLclNum; destVarDsc = &(lvaTable[destVarNum]); } - if (lhsBlk->OperGet() == GT_OBJ) - { - clsHnd = lhsBlk->AsObj()->gtClass; - } } else { @@ -9035,12 +9035,12 @@ GenTree* Compiler::fgMorphOneAsgBlockOp(GenTree* tree) destVarDsc = &(lvaTable[destVarNum]); if (isCopyBlock) { - clsHnd = destVarDsc->lvVerTypeInfo.GetClassHandle(); - size = info.compCompHnd->getClassSize(clsHnd); + layout = destVarDsc->lvLayout; + size = destVarDsc->lvLayout->GetSize(); } else { - size = destVarDsc->lvExactSize; + size = destVarDsc->lvLayout->GetSize(); } } @@ -9058,16 +9058,14 @@ GenTree* Compiler::fgMorphOneAsgBlockOp(GenTree* tree) { if (size == REGSIZE_BYTES) { - if (clsHnd == NO_CLASS_HANDLE) + if ((layout == nullptr) || (layout->GetClass() == NO_CLASS_HANDLE)) { // A register-sized cpblk can be treated as an integer asignment. asgType = TYP_I_IMPL; } else { - BYTE gcPtr; - info.compCompHnd->getClassGClayout(clsHnd, &gcPtr); - asgType = getJitGCType(gcPtr); + asgType = layout->GetGCPtrType(0); } } else @@ -9856,7 +9854,7 @@ GenTree* Compiler::fgMorphBlkNode(GenTree* tree, bool isDest) } else { - tree = new (this, GT_BLK) GenTreeBlk(GT_BLK, structType, addr, genTypeSize(structType)); + tree = new (this, GT_BLK) GenTreeBlk(GT_BLK, structType, addr, typGetBlkLayout(genTypeSize(structType))); } gtUpdateNodeSideEffects(tree); @@ -9881,7 +9879,7 @@ GenTree* Compiler::fgMorphBlkNode(GenTree* tree, bool isDest) { blkNode->AsDynBlk()->gtDynamicSize = nullptr; blkNode->ChangeOper(GT_BLK); - blkNode->gtBlkSize = size; + blkNode->gtLayout = typGetBlkLayout(size); } else { @@ -10026,7 +10024,7 @@ GenTree* Compiler::fgMorphBlockOperand(GenTree* tree, var_types asgType, unsigne CORINFO_CLASS_HANDLE clsHnd = gtGetStructHandleIfPresent(effectiveVal); if (clsHnd == NO_CLASS_HANDLE) { - newTree = new (this, GT_BLK) GenTreeBlk(GT_BLK, TYP_STRUCT, addr, blockWidth); + newTree = new (this, GT_BLK) GenTreeBlk(GT_BLK, TYP_STRUCT, addr, typGetBlkLayout(blockWidth)); } else { @@ -10075,8 +10073,8 @@ GenTree* Compiler::fgMorphBlockOperand(GenTree* tree, var_types asgType, unsigne void Compiler::fgMorphUnsafeBlk(GenTreeObj* dest) { #if defined(CPBLK_UNROLL_LIMIT) && !defined(JIT32_GCENCODER) - assert(dest->gtGcPtrCount != 0); - unsigned blockWidth = dest->AsBlk()->gtBlkSize; + assert(dest->gtLayout->HasGCPtr()); + unsigned blockWidth = dest->gtLayout->GetSize(); #ifdef DEBUG bool destOnStack = false; GenTree* destAddr = dest->Addr(); @@ -10198,7 +10196,7 @@ GenTree* Compiler::fgMorphCopyBlock(GenTree* tree) { blockWidth = genTypeSize(destLclVar->lvType); } - hasGCPtrs = destLclVar->lvStructGcCount != 0; + hasGCPtrs = lvaTypeIsGC(destLclNum); } else { @@ -10227,7 +10225,7 @@ GenTree* Compiler::fgMorphCopyBlock(GenTree* tree) assert(effectiveDest->OperIsBlk()); GenTreeBlk* blk = effectiveDest->AsBlk(); - blockWidth = blk->gtBlkSize; + blockWidth = blk->Size(); blockWidthIsConst = (blk->gtOper != GT_DYN_BLK); if ((dest == effectiveDest) && ((dest->gtFlags & GTF_IND_ARR_INDEX) == 0)) { @@ -18621,7 +18619,7 @@ class LocalAddressVisitor final : public GenTreeVisitor m_compiler->info.compCompHnd->getFieldClass(indir->AsField()->gtFldHnd)); case GT_BLK: case GT_OBJ: - return indir->AsBlk()->gtBlkSize; + return indir->AsBlk()->Size(); default: assert(indir->OperIs(GT_IND, GT_DYN_BLK)); return 0; diff --git a/src/jit/rationalize.cpp b/src/jit/rationalize.cpp index 1cdb54f2140f..d2bc234e49f7 100644 --- a/src/jit/rationalize.cpp +++ b/src/jit/rationalize.cpp @@ -366,37 +366,29 @@ void Rationalizer::RewriteAssignment(LIR::Use& use) { if ((location->OperGet() == GT_LCL_VAR)) { + LclVarDsc* varDsc = comp->lvaGetDesc(location->AsLclVar()); + // We need to construct a block node for the location. // Modify lcl to be the address form. location->SetOper(addrForm(locationOp)); - LclVarDsc* varDsc = &(comp->lvaTable[location->AsLclVarCommon()->gtLclNum]); - location->gtType = TYP_BYREF; - GenTreeBlk* storeBlk = nullptr; - unsigned int size = varDsc->lvExactSize; + location->gtType = TYP_BYREF; - if (varDsc->lvStructGcCount != 0) - { - CORINFO_CLASS_HANDLE structHnd = varDsc->lvVerTypeInfo.GetClassHandle(); - GenTreeObj* objNode = comp->gtNewObjNode(structHnd, location)->AsObj(); - unsigned int slots = roundUp(size, TARGET_POINTER_SIZE) / TARGET_POINTER_SIZE; - - objNode->SetGCInfo(varDsc->lvGcLayout, varDsc->lvStructGcCount, slots); - objNode->ChangeOper(GT_STORE_OBJ); - objNode->SetData(value); - comp->fgMorphUnsafeBlk(objNode); - storeBlk = objNode; - } - else + assignment->ChangeOper(varDsc->lvLayout->HasGCPtr() ? GT_STORE_OBJ : GT_STORE_BLK); + GenTreeBlk* store = assignment->AsBlk(); + + store->gtFlags |= GTF_IND_NONFAULTING; + store->gtFlags &= ~(GTF_GLOB_REF | GTF_REVERSE_OPS); + assert(store->Addr() == location); + assert(store->Data() == value); + store->gtLayout = varDsc->lvLayout; + store->gtBlkOpKind = GenTreeBlk::BlkOpKindInvalid; + store->gtBlkOpGcUnsafe = false; + + if (store->OperIs(GT_STORE_OBJ)) { - storeBlk = new (comp, GT_STORE_BLK) GenTreeBlk(GT_STORE_BLK, TYP_STRUCT, location, value, size); + comp->fgMorphUnsafeBlk(store->AsObj()); } - storeBlk->gtFlags |= GTF_ASG; - storeBlk->gtFlags |= ((location->gtFlags | value->gtFlags) & GTF_ALL_EFFECT); - GenTree* insertionPoint = location->gtNext; - BlockRange().InsertBefore(insertionPoint, storeBlk); - use.ReplaceWith(comp, storeBlk); - BlockRange().Remove(assignment); JITDUMP("After transforming local struct assignment into a block op:\n"); DISPTREERANGE(BlockRange(), use.Def()); JITDUMP("\n"); diff --git a/src/jit/simd.cpp b/src/jit/simd.cpp index 4f3f8eb04c0b..c544293314ba 100644 --- a/src/jit/simd.cpp +++ b/src/jit/simd.cpp @@ -1022,17 +1022,17 @@ const SIMDIntrinsicInfo* Compiler::getSIMDIntrinsicInfo(CORINFO_CLASS_HANDLE* in // Normalizes TYP_STRUCT value in case of GT_CALL, GT_RET_EXPR and arg nodes. // // Arguments: -// type - the type of value that the caller expects to be popped off the stack. -// expectAddr - if true indicates we are expecting type stack entry to be a TYP_BYREF. -// structHandle - the class handle to use when normalizing if it is not the same as the stack entry class handle; -// this can happen for certain scenarios, such as folding away a static cast, where we want the -// value popped to have the type that would have been returned. +// type - the type of value that the caller expects to be popped off the stack. +// expectAddr - if true indicates we are expecting type stack entry to be a TYP_BYREF. +// layout - the struct layout to use when normalizing if it is not the same as the stack entry layout; +// this can happen for certain scenarios, such as folding away a static cast, where we want the +// value popped to have the type that would have been returned. // // Notes: // If the popped value is a struct, and the expected type is a simd type, it will be set // to that type, otherwise it will assert if the type being popped is not the expected type. -GenTree* Compiler::impSIMDPopStack(var_types type, bool expectAddr, CORINFO_CLASS_HANDLE structHandle) +GenTree* Compiler::impSIMDPopStack(var_types type, bool expectAddr, StructTypeLayout* layout) { StackEntry se = impPopStack(); typeInfo ti = se.seTypeInfo; @@ -1058,10 +1058,10 @@ GenTree* Compiler::impSIMDPopStack(var_types type, bool expectAddr, CORINFO_CLAS // If we have a ldobj of a SIMD local we need to transform it. if (tree->OperGet() == GT_OBJ) { - if (tree->AsObj()->gtClass != structHandle) + if (tree->AsObj()->gtLayout != layout) { // In this case we need to retain the GT_OBJ to retype the value. - tree->AsObj()->gtClass = structHandle; + tree->AsObj()->gtLayout = layout; } else { @@ -1085,12 +1085,12 @@ GenTree* Compiler::impSIMDPopStack(var_types type, bool expectAddr, CORINFO_CLAS { assert(ti.IsType(TI_STRUCT)); - if (structHandle == nullptr) + if (layout == nullptr) { - structHandle = ti.GetClassHandleForValueClass(); + layout = typGetObjLayout(ti.GetClassHandleForValueClass()); } - tree = impNormStructVal(tree, structHandle, (unsigned)CHECK_SPILL_ALL); + tree = impNormStructVal(tree, layout->GetClass(), (unsigned)CHECK_SPILL_ALL); } // Now set the type of the tree to the specialized SIMD struct type, if applicable. @@ -3118,18 +3118,18 @@ GenTree* Compiler::impSIMDIntrinsic(OPCODE opcode, GenTree* dupOp1 = fgInsertCommaFormTemp(&op1, gtGetStructHandleForSIMD(simdType, baseType)); // Widen the lower half and assign it to dstAddrLo. - simdTree = gtNewSIMDNode(simdType, op1, nullptr, SIMDIntrinsicWidenLo, baseType, size); - GenTree* loDest = - new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, dstAddrLo, getSIMDTypeSizeInBytes(clsHnd)); + simdTree = gtNewSIMDNode(simdType, op1, nullptr, SIMDIntrinsicWidenLo, baseType, size); + GenTree* loDest = new (this, GT_BLK) + GenTreeBlk(GT_BLK, simdType, dstAddrLo, typGetBlkLayout(getSIMDTypeSizeInBytes(clsHnd))); GenTree* loAsg = gtNewBlkOpNode(loDest, simdTree, getSIMDTypeSizeInBytes(clsHnd), false, // not volatile true); // copyBlock loAsg->gtFlags |= ((simdTree->gtFlags | dstAddrLo->gtFlags) & GTF_ALL_EFFECT); // Widen the upper half and assign it to dstAddrHi. - simdTree = gtNewSIMDNode(simdType, dupOp1, nullptr, SIMDIntrinsicWidenHi, baseType, size); - GenTree* hiDest = - new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, dstAddrHi, getSIMDTypeSizeInBytes(clsHnd)); + simdTree = gtNewSIMDNode(simdType, dupOp1, nullptr, SIMDIntrinsicWidenHi, baseType, size); + GenTree* hiDest = new (this, GT_BLK) + GenTreeBlk(GT_BLK, simdType, dstAddrHi, typGetBlkLayout(getSIMDTypeSizeInBytes(clsHnd))); GenTree* hiAsg = gtNewBlkOpNode(hiDest, simdTree, getSIMDTypeSizeInBytes(clsHnd), false, // not volatile true); // copyBlock @@ -3167,7 +3167,8 @@ GenTree* Compiler::impSIMDIntrinsic(OPCODE opcode, // block ops. if (doCopyBlk) { - GenTree* dest = new (this, GT_BLK) GenTreeBlk(GT_BLK, simdType, copyBlkDst, getSIMDTypeSizeInBytes(clsHnd)); + GenTree* dest = new (this, GT_BLK) + GenTreeBlk(GT_BLK, simdType, copyBlkDst, typGetBlkLayout(getSIMDTypeSizeInBytes(clsHnd))); dest->gtFlags |= GTF_GLOB_REF; retVal = gtNewBlkOpNode(dest, simdTree, getSIMDTypeSizeInBytes(clsHnd), false, // not volatile