From 9f02ecd1a5cabb7f3bfb6ec3a85d2d8492bcb1dd Mon Sep 17 00:00:00 2001 From: Joe Groff Date: Fri, 10 Aug 2018 11:27:13 -0700 Subject: [PATCH] IRGen: Use any field of structs for extra inhabitants. This allows us to layout-optimize Optional when T is a struct with an extra-inhabitant-bearing field anywhere in its definition, not only at the beginning. rdar://problem/43019427 --- lib/IRGen/FixedTypeInfo.h | 6 +- lib/IRGen/GenEnum.cpp | 58 ++-- lib/IRGen/GenEnum.h | 6 +- lib/IRGen/GenExistential.cpp | 19 +- lib/IRGen/GenFunc.cpp | 11 +- lib/IRGen/GenHeap.cpp | 17 +- lib/IRGen/GenObjC.cpp | 6 +- lib/IRGen/GenStruct.cpp | 289 +++++++++++++++--- lib/IRGen/GenTuple.cpp | 12 +- lib/IRGen/GenType.cpp | 11 +- lib/IRGen/GenValueWitness.cpp | 6 +- lib/IRGen/HeapTypeInfo.h | 5 +- lib/IRGen/NonFixedTypeInfo.h | 6 +- lib/IRGen/ResilientTypeInfo.h | 6 +- lib/IRGen/TypeInfo.h | 6 +- stdlib/public/runtime/Metadata.cpp | 28 +- ...itants_ExtraInhabitantResilientTypes.swift | 7 + .../struct_extra_inhabitants.swift | 169 ++++++++++ 18 files changed, 562 insertions(+), 106 deletions(-) create mode 100644 test/Interpreter/Inputs/struct_extra_inhabitants_ExtraInhabitantResilientTypes.swift create mode 100644 test/Interpreter/struct_extra_inhabitants.swift diff --git a/lib/IRGen/FixedTypeInfo.h b/lib/IRGen/FixedTypeInfo.h index ac362534424bb..68411cd338a6c 100644 --- a/lib/IRGen/FixedTypeInfo.h +++ b/lib/IRGen/FixedTypeInfo.h @@ -167,7 +167,8 @@ class FixedTypeInfo : public TypeInfo { /// Map an extra inhabitant representation in memory to a unique 31-bit /// identifier, and map a valid representation of the type to -1. llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF, - Address src, SILType T) const override { + Address src, SILType T, + bool isOutlined) const override { return getSpareBitExtraInhabitantIndex(IGF, src); } @@ -180,7 +181,8 @@ class FixedTypeInfo : public TypeInfo { /// to memory. void storeExtraInhabitant(IRGenFunction &IGF, llvm::Value *index, - Address dest, SILType T) const override { + Address dest, SILType T, + bool isOutlined) const override { storeSpareBitExtraInhabitant(IGF, index, dest); } diff --git a/lib/IRGen/GenEnum.cpp b/lib/IRGen/GenEnum.cpp index b190e6b6d4836..2a3ffa2da6bfe 100644 --- a/lib/IRGen/GenEnum.cpp +++ b/lib/IRGen/GenEnum.cpp @@ -641,7 +641,8 @@ namespace { } llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF, - Address src, SILType T) + Address src, SILType T, + bool isOutlined) const override { if (!getSingleton()) { // Any empty value is a valid value. @@ -650,19 +651,22 @@ namespace { return getSingleton()->getExtraInhabitantIndex(IGF, getSingletonAddress(IGF, src), - getSingletonType(IGF.IGM, T)); + getSingletonType(IGF.IGM, T), + isOutlined); } void storeExtraInhabitant(IRGenFunction &IGF, llvm::Value *index, - Address dest, SILType T) const override { + Address dest, SILType T, + bool isOutlined) const override { if (!getSingleton()) { // Nothing to store for empty singletons. return; } getSingleton()->storeExtraInhabitant(IGF, index, getSingletonAddress(IGF, dest), - getSingletonType(IGF.IGM, T)); + getSingletonType(IGF.IGM, T), + isOutlined); } unsigned getFixedExtraInhabitantCount(IRGenModule &IGM) const override { @@ -1014,7 +1018,8 @@ namespace { } llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF, - Address src, SILType T) + Address src, SILType T, + bool isOutlined) const override { auto &C = IGF.IGM.getLLVMContext(); @@ -1044,7 +1049,8 @@ namespace { void storeExtraInhabitant(IRGenFunction &IGF, llvm::Value *index, - Address dest, SILType T) const override { + Address dest, SILType T, + bool isOutlined) const override { auto &C = IGF.IGM.getLLVMContext(); auto payloadTy = llvm::IntegerType::get(C, cast(TI)->getFixedSize().getValueInBits()); @@ -1119,13 +1125,15 @@ namespace { } llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF, - Address src, SILType T) const override { + Address src, SILType T, + bool isOutlined) const override { llvm_unreachable("no extra inhabitants"); } void storeExtraInhabitant(IRGenFunction &IGF, llvm::Value *index, - Address dest, SILType T) const override { + Address dest, SILType T, + bool isOutlined) const override { llvm_unreachable("no extra inhabitants"); } @@ -2915,11 +2923,13 @@ namespace { llvm::Value * getExtraInhabitantIndex(IRGenFunction &IGF, - Address src, SILType T) const override { + Address src, SILType T, + bool isOutlined) const override { auto payload = projectPayloadData(IGF, src); llvm::Value *index = getPayloadTypeInfo().getExtraInhabitantIndex(IGF, payload, - getPayloadType(IGF.IGM, T)); + getPayloadType(IGF.IGM, T), + isOutlined); // Offset the payload extra inhabitant index by the number of inhabitants // we used. If less than zero, it's a valid value of the enum type. @@ -2935,14 +2945,16 @@ namespace { void storeExtraInhabitant(IRGenFunction &IGF, llvm::Value *index, - Address dest, SILType T) const override { + Address dest, SILType T, + bool isOutlined) const override { // Offset the index to skip the extra inhabitants we used. index = IGF.Builder.CreateAdd(index, llvm::ConstantInt::get(IGF.IGM.Int32Ty, ElementsWithNoPayload.size())); auto payload = projectPayloadData(IGF, dest); getPayloadTypeInfo().storeExtraInhabitant(IGF, index, payload, - getPayloadType(IGF.IGM, T)); + getPayloadType(IGF.IGM, T), + isOutlined); } APInt @@ -4811,14 +4823,16 @@ namespace { llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF, Address src, - SILType T) const override { + SILType T, + bool isOutlined) const override { llvm_unreachable("extra inhabitants for multi-payload enums not implemented"); } void storeExtraInhabitant(IRGenFunction &IGF, llvm::Value *index, Address dest, - SILType T) const override { + SILType T, + bool isOutlined) const override { llvm_unreachable("extra inhabitants for multi-payload enums not implemented"); } @@ -5204,14 +5218,16 @@ namespace { llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF, Address src, - SILType T) const override { + SILType T, + bool isOutlined) const override { return emitGetExtraInhabitantIndexCall(IGF, T, src); } void storeExtraInhabitant(IRGenFunction &IGF, llvm::Value *index, Address dest, - SILType T) const override { + SILType T, + bool isOutlined) const override { emitStoreExtraInhabitantCall(IGF, T, index, dest); } @@ -5436,14 +5452,16 @@ namespace { } llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF, Address src, - SILType T) const override { - return Strategy.getExtraInhabitantIndex(IGF, src, T); + SILType T, + bool isOutlined) const override { + return Strategy.getExtraInhabitantIndex(IGF, src, T, isOutlined); } void storeExtraInhabitant(IRGenFunction &IGF, llvm::Value *index, Address dest, - SILType T) const override { - return Strategy.storeExtraInhabitant(IGF, index, dest, T); + SILType T, + bool isOutlined) const override { + return Strategy.storeExtraInhabitant(IGF, index, dest, T, isOutlined); } bool isSingleRetainablePointer(ResilienceExpansion expansion, ReferenceCounting *rc) const override { diff --git a/lib/IRGen/GenEnum.h b/lib/IRGen/GenEnum.h index c8c29aa6d0f84..50f56e03e3121 100644 --- a/lib/IRGen/GenEnum.h +++ b/lib/IRGen/GenEnum.h @@ -375,11 +375,13 @@ class EnumImplStrategy { virtual llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF, Address src, - SILType T) const = 0; + SILType T, + bool isOutlined) const = 0; virtual void storeExtraInhabitant(IRGenFunction &IGF, llvm::Value *index, Address dest, - SILType T) const = 0; + SILType T, + bool isOutlined) const = 0; /// \group Delegated FixedTypeInfo operations diff --git a/lib/IRGen/GenExistential.cpp b/lib/IRGen/GenExistential.cpp index c20caa188a460..bfaaa58b93e73 100644 --- a/lib/IRGen/GenExistential.cpp +++ b/lib/IRGen/GenExistential.cpp @@ -554,20 +554,21 @@ namespace { } llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF, Address src, - SILType T) + SILType T, bool isOutlined) const override { // NB: We assume that the witness table slots are zero if an extra // inhabitant is stored in the container. src = projectValue(IGF, src); return asDerived().getValueTypeInfoForExtraInhabitants(IGF.IGM) - .getExtraInhabitantIndex(IGF, src, SILType()); + .getExtraInhabitantIndex(IGF, src, SILType(), isOutlined); } void storeExtraInhabitant(IRGenFunction &IGF, llvm::Value *index, - Address dest, SILType T) const override { + Address dest, SILType T, bool isOutlined) + const override { Address valueDest = projectValue(IGF, dest); asDerived().getValueTypeInfoForExtraInhabitants(IGF.IGM) - .storeExtraInhabitant(IGF, index, valueDest, SILType()); + .storeExtraInhabitant(IGF, index, valueDest, SILType(), isOutlined); } APInt getFixedExtraInhabitantMask(IRGenModule &IGM) const override { @@ -615,23 +616,25 @@ namespace { } \ } \ llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF, Address src, \ - SILType T) const override { \ + SILType T, bool isOutlined) \ + const override { \ Address valueSrc = projectValue(IGF, src); \ if (shouldStoreExtraInhabitantsInRef(IGF.IGM)) { \ return IGF.getReferenceStorageExtraInhabitantIndex(valueSrc, \ ReferenceOwnership::Name, Refcounting); \ } else { \ - return Super::getExtraInhabitantIndex(IGF, src, T); \ + return Super::getExtraInhabitantIndex(IGF, src, T, isOutlined); \ } \ } \ void storeExtraInhabitant(IRGenFunction &IGF, llvm::Value *index, \ - Address dest, SILType T) const override { \ + Address dest, SILType T, bool isOutlined) \ + const override { \ Address valueDest = projectValue(IGF, dest); \ if (shouldStoreExtraInhabitantsInRef(IGF.IGM)) { \ return IGF.storeReferenceStorageExtraInhabitant(index, valueDest, \ ReferenceOwnership::Name, Refcounting); \ } else { \ - return Super::storeExtraInhabitant(IGF, index, dest, T); \ + return Super::storeExtraInhabitant(IGF, index, dest, T, isOutlined); \ } \ } \ APInt getFixedExtraInhabitantMask(IRGenModule &IGM) const override { \ diff --git a/lib/IRGen/GenFunc.cpp b/lib/IRGen/GenFunc.cpp index 7ab471a4da5f4..703fe248f9cda 100644 --- a/lib/IRGen/GenFunc.cpp +++ b/lib/IRGen/GenFunc.cpp @@ -165,13 +165,14 @@ namespace { } llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF, Address src, - SILType T) + SILType T, bool isOutlined) const override { return getFunctionPointerExtraInhabitantIndex(IGF, src); } void storeExtraInhabitant(IRGenFunction &IGF, llvm::Value *index, - Address dest, SILType T) const override { + Address dest, SILType T, bool isOutlined) + const override { return storeFunctionPointerExtraInhabitant(IGF, index, dest); } }; @@ -411,7 +412,8 @@ namespace { } llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF, Address src, - SILType T) const override { + SILType T, bool isOutlined) + const override { src = projectFunction(IGF, src); return getFunctionPointerExtraInhabitantIndex(IGF, src); } @@ -425,7 +427,8 @@ namespace { } void storeExtraInhabitant(IRGenFunction &IGF, llvm::Value *index, - Address dest, SILType T) const override { + Address dest, SILType T, bool isOutlined) + const override { dest = projectFunction(IGF, dest); return storeFunctionPointerExtraInhabitant(IGF, index, dest); } diff --git a/lib/IRGen/GenHeap.cpp b/lib/IRGen/GenHeap.cpp index 1acad47604f5f..6432e89ca9d28 100644 --- a/lib/IRGen/GenHeap.cpp +++ b/lib/IRGen/GenHeap.cpp @@ -97,13 +97,15 @@ namespace { ReferenceCounting::Nativeness); \ } \ llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF, Address src, \ - SILType T) const override { \ + SILType T, bool isOutlined) \ + const override { \ return IGF.getReferenceStorageExtraInhabitantIndex(src, \ ReferenceOwnership::Name, \ ReferenceCounting::Nativeness); \ } \ void storeExtraInhabitant(IRGenFunction &IGF, llvm::Value *index, \ - Address dest, SILType T) const override { \ + Address dest, SILType T, bool isOutlined) \ + const override { \ return IGF.storeReferenceStorageExtraInhabitant(index, dest, \ ReferenceOwnership::Name, \ ReferenceCounting::Nativeness); \ @@ -166,13 +168,15 @@ namespace { ReferenceCounting::Nativeness); \ } \ llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF, Address src, \ - SILType T) const override { \ + SILType T, bool isOutlined) \ + const override { \ return IGF.getReferenceStorageExtraInhabitantIndex(src, \ ReferenceOwnership::Name, \ ReferenceCounting::Nativeness); \ } \ void storeExtraInhabitant(IRGenFunction &IGF, llvm::Value *index, \ - Address dest, SILType T) const override { \ + Address dest, SILType T, bool isOutlined) \ + const override { \ return IGF.storeReferenceStorageExtraInhabitant(index, dest, \ ReferenceOwnership::Name, \ ReferenceCounting::Nativeness); \ @@ -217,12 +221,13 @@ namespace { index + IsOptional, 0); \ } \ llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF, Address src, \ - SILType T) \ + SILType T, bool isOutlined) \ const override { \ return getHeapObjectExtraInhabitantIndex(IGF, src); \ } \ void storeExtraInhabitant(IRGenFunction &IGF, llvm::Value *index, \ - Address dest, SILType T) const override { \ + Address dest, SILType T, bool isOutlined) \ + const override { \ return storeHeapObjectExtraInhabitant(IGF, index, dest); \ } \ }; diff --git a/lib/IRGen/GenObjC.cpp b/lib/IRGen/GenObjC.cpp index 4fe452d249c12..8eeef901a0b42 100644 --- a/lib/IRGen/GenObjC.cpp +++ b/lib/IRGen/GenObjC.cpp @@ -278,7 +278,8 @@ namespace { return APInt(bits, 0); } llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF, Address src, - SILType T) const override { + SILType T, + bool isOutlined) const override { src = IGF.Builder.CreateBitCast(src, IGF.IGM.SizeTy->getPointerTo()); auto val = IGF.Builder.CreateLoad(src); auto isNonzero = IGF.Builder.CreateICmpNE(val, @@ -288,7 +289,8 @@ namespace { return IGF.Builder.CreateSExt(isNonzero, IGF.IGM.Int32Ty); } void storeExtraInhabitant(IRGenFunction &IGF, llvm::Value *index, - Address dest, SILType T) const override { + Address dest, SILType T, bool isOutlined) + const override { // There's only one extra inhabitant, 0. dest = IGF.Builder.CreateBitCast(dest, IGF.IGM.SizeTy->getPointerTo()); IGF.Builder.CreateStore(llvm::ConstantInt::get(IGF.IGM.SizeTy, 0), dest); diff --git a/lib/IRGen/GenStruct.cpp b/lib/IRGen/GenStruct.cpp index 6a4ebcf16ea5a..ebbb2f3dfebae 100644 --- a/lib/IRGen/GenStruct.cpp +++ b/lib/IRGen/GenStruct.cpp @@ -108,7 +108,8 @@ namespace { class StructTypeInfoBase : public RecordTypeInfo { using super = RecordTypeInfo; - + mutable Optional ExtraInhabitantProvidingField; + mutable Optional MayHaveExtraInhabitants; protected: template StructTypeInfoBase(StructTypeInfoKind kind, As &&...args) @@ -201,37 +202,48 @@ namespace { return fieldInfo.getStructIndex(); } - // For now, just use extra inhabitants from the first field. - // FIXME: generalize bool mayHaveExtraInhabitants(IRGenModule &IGM) const override { - if (asImpl().getFields().empty()) return false; - return asImpl().getFields()[0].getTypeInfo().mayHaveExtraInhabitants(IGM); + if (!MayHaveExtraInhabitants.hasValue()) { + MayHaveExtraInhabitants = false; + for (auto &field : asImpl().getFields()) + if (field.getTypeInfo().mayHaveExtraInhabitants(IGM)) { + MayHaveExtraInhabitants = true; + break; + } + } + return *MayHaveExtraInhabitants; } // This is dead code in NonFixedStructTypeInfo. unsigned getFixedExtraInhabitantCount(IRGenModule &IGM) const { - if (asImpl().getFields().empty()) return 0; - auto &fieldTI = cast(asImpl().getFields()[0].getTypeInfo()); - return fieldTI.getFixedExtraInhabitantCount(IGM); + if (auto field = asImpl().getFixedExtraInhabitantProvidingField(IGM)) { + auto &fieldTI = cast(field->getTypeInfo()); + return fieldTI.getFixedExtraInhabitantCount(IGM); + } + + return 0; } // This is dead code in NonFixedStructTypeInfo. APInt getFixedExtraInhabitantValue(IRGenModule &IGM, unsigned bits, unsigned index) const { - auto &fieldTI = cast(asImpl().getFields()[0].getTypeInfo()); - return fieldTI.getFixedExtraInhabitantValue(IGM, bits, index); + // We are only called if the type is known statically to have extra + // inhabitants. + auto &field = *asImpl().getFixedExtraInhabitantProvidingField(IGM); + auto &fieldTI = cast(field.getTypeInfo()); + APInt fieldValue = fieldTI.getFixedExtraInhabitantValue(IGM, bits, index); + return fieldValue.shl(field.getFixedByteOffset().getValueInBits()); } // This is dead code in NonFixedStructTypeInfo. APInt getFixedExtraInhabitantMask(IRGenModule &IGM) const { - if (asImpl().getFields().empty()) + auto field = asImpl().getFixedExtraInhabitantProvidingField(IGM); + if (!field) return APInt(); - // Currently we only use the first field's extra inhabitants. The other - // fields can be ignored. const FixedTypeInfo &fieldTI - = cast(asImpl().getFields()[0].getTypeInfo()); + = cast(field->getTypeInfo()); auto targetSize = asImpl().getFixedSize().getValueInBits(); if (fieldTI.isKnownEmpty(ResilienceExpansion::Maximal)) @@ -240,28 +252,164 @@ namespace { APInt fieldMask = fieldTI.getFixedExtraInhabitantMask(IGM); if (targetSize > fieldMask.getBitWidth()) fieldMask = fieldMask.zext(targetSize); + fieldMask = fieldMask.shl(field->getFixedByteOffset().getValueInBits()); return fieldMask; } + + // Perform an operation using the field that provides extra inhabitants for + // the aggregate, whether that field is known statically or dynamically. + llvm::Value *withExtraInhabitantProvidingField(IRGenFunction &IGF, + Address structAddr, + SILType structType, + bool isOutlined, + llvm::Type *resultTy, + llvm::function_ref body, + llvm::function_ref outline) const { + // If we know one field consistently provides extra inhabitants, delegate + // to that field. + if (auto field = asImpl().getFixedExtraInhabitantProvidingField(IGF.IGM)){ + return body(*field); + } + + // Otherwise, we have to figure out which field at runtime. + // The decision tree could be rather large, so invoke the value witness + // unless we're emitting the value witness. + if (!isOutlined) + return outline(); + + // The number of extra inhabitants the instantiated type has can be used + // to figure out which field the runtime chose. The runtime uses the same + // algorithm as above--use the field with the most extra inhabitants, + // favoring the earliest field in a tie. If we test the number of extra + // inhabitants in the struct against each field type's, then the first + // match should indicate which field we chose. + // + // We can reduce the decision space somewhat if there are fixed-layout + // fields, since we know the only possible runtime choices are + // either the fixed field with the most extra inhabitants (if any), or + // one of the unknown-layout fields. + // + // See whether we have a fixed candidate. + const FieldInfoType *fixedCandidate = nullptr; + unsigned fixedCount = 0; + for (auto &field : asImpl().getFields()) { + if (!field.getTypeInfo().mayHaveExtraInhabitants(IGF.IGM)) + continue; + + if (const FixedTypeInfo *fixed = + dyn_cast(&field.getTypeInfo())) { + auto fieldCount = fixed->getFixedExtraInhabitantCount(IGF.IGM); + if (fieldCount > fixedCount) { + fixedCandidate = &field; + fixedCount = fieldCount; + } + } + } + + // Loop through checking to see whether we picked the fixed candidate + // (if any) or one of the unknown-layout fields. + llvm::Value *instantiatedCount + = emitLoadOfExtraInhabitantCount(IGF, structType); + + auto contBB = IGF.createBasicBlock("chose_field_for_xi"); + llvm::PHINode *contPhi = nullptr; + if (resultTy != IGF.IGM.VoidTy) + contPhi = llvm::PHINode::Create(resultTy, + asImpl().getFields().size()); + + // If two fields have the same type, they have the same extra inhabitant + // count, and we'll pick the first. We don't have to check both. + SmallPtrSet visitedTypes; + + for (auto &field : asImpl().getFields()) { + if (!field.getTypeInfo().mayHaveExtraInhabitants(IGF.IGM)) + continue; + + ConditionalDominanceScope condition(IGF); + + llvm::Value *fieldCount; + if (isa(field.getTypeInfo())) { + // Skip fixed fields except for the candidate with the most known + // extra inhabitants we picked above. + if (&field != fixedCandidate) + continue; + + fieldCount = llvm::ConstantInt::get(IGF.IGM.SizeTy, fixedCount); + } else { + auto fieldTy = field.getType(IGF.IGM, structType); + // If this field has the same type as a field we already tested, + // we'll never pick this one, since they both have the same count. + if (!visitedTypes.insert(fieldTy).second) + continue; + + fieldCount = emitLoadOfExtraInhabitantCount(IGF, fieldTy); + } + auto equalsCount = IGF.Builder.CreateICmpEQ(instantiatedCount, + fieldCount); + + auto yesBB = IGF.createBasicBlock(""); + auto noBB = IGF.createBasicBlock(""); + + IGF.Builder.CreateCondBr(equalsCount, yesBB, noBB); + + IGF.Builder.emitBlock(yesBB); + auto value = body(field); + if (contPhi) + contPhi->addIncoming(value, IGF.Builder.GetInsertBlock()); + IGF.Builder.CreateBr(contBB); + + IGF.Builder.emitBlock(noBB); + } + + // We shouldn't have picked a number of extra inhabitants inconsistent + // with any individual field. + IGF.Builder.CreateUnreachable(); + + IGF.Builder.emitBlock(contBB); + if (contPhi) + IGF.Builder.Insert(contPhi); + + return contPhi; + } llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF, Address structAddr, - SILType structType) const override { - auto &field = asImpl().getFields()[0]; - Address fieldAddr = - asImpl().projectFieldAddress(IGF, structAddr, structType, field.Field); - return field.getTypeInfo().getExtraInhabitantIndex(IGF, fieldAddr, - field.getType(IGF.IGM, structType)); + SILType structType, + bool isOutlined) const override { + return withExtraInhabitantProvidingField(IGF, structAddr, structType, + isOutlined, + IGF.IGM.Int32Ty, + [&](const FieldInfoType &field) -> llvm::Value* { + Address fieldAddr = asImpl().projectFieldAddress( + IGF, structAddr, structType, field.Field); + return field.getTypeInfo().getExtraInhabitantIndex(IGF, fieldAddr, + field.getType(IGF.IGM, structType), + false /*not outlined for field*/); + }, + [&]() -> llvm::Value * { + return emitGetExtraInhabitantIndexCall(IGF, structType, structAddr); + }); } void storeExtraInhabitant(IRGenFunction &IGF, llvm::Value *index, Address structAddr, - SILType structType) const override { - auto &field = asImpl().getFields()[0]; - Address fieldAddr = - asImpl().projectFieldAddress(IGF, structAddr, structType, field.Field); - field.getTypeInfo().storeExtraInhabitant(IGF, index, fieldAddr, - field.getType(IGF.IGM, structType)); + SILType structType, + bool isOutlined) const override { + withExtraInhabitantProvidingField(IGF, structAddr, structType, isOutlined, + IGF.IGM.VoidTy, + [&](const FieldInfoType &field) -> llvm::Value* { + Address fieldAddr = asImpl().projectFieldAddress( + IGF, structAddr, structType, field.Field); + field.getTypeInfo().storeExtraInhabitant(IGF, index, fieldAddr, + field.getType(IGF.IGM, structType), + false /*not outlined for field*/); + return nullptr; + }, + [&]() -> llvm::Value * { + emitStoreExtraInhabitantCall(IGF, structType, index, structAddr); + return nullptr; + }); } bool isSingleRetainablePointer(ResilienceExpansion expansion, @@ -285,6 +433,7 @@ namespace { : public StructMetadataScanner { public: VarDecl *FieldToFind; + Size AddressPoint = Size::invalid(); Size FieldOffset = Size::invalid(); FindOffsetOfFieldOffsetVector(IRGenModule &IGM, VarDecl *Field) @@ -292,6 +441,10 @@ namespace { IGM, cast(Field->getDeclContext())), FieldToFind(Field) {} + void noteAddressPoint() { + AddressPoint = this->NextOffset; + } + void addFieldOffset(VarDecl *Field) { if (Field == FieldToFind) { FieldOffset = this->NextOffset; @@ -304,7 +457,8 @@ namespace { FindOffsetOfFieldOffsetVector scanner(IGF.IGM, field.Field); scanner.layout(); - if (scanner.FieldOffset == Size::invalid()) + if (scanner.FieldOffset == Size::invalid() + || scanner.AddressPoint == Size::invalid()) continue; // Load the offset from the field offset vector and ensure it matches @@ -313,13 +467,14 @@ namespace { IGF.Builder.CreateBitCast(metadata, IGF.IGM.Int8PtrTy); auto fieldOffsetPtr = IGF.Builder.CreateInBoundsGEP(metadataBytes, - IGF.IGM.getSize(scanner.FieldOffset)); + IGF.IGM.getSize(scanner.FieldOffset - scanner.AddressPoint)); fieldOffsetPtr = IGF.Builder.CreateBitCast(fieldOffsetPtr, - IGF.IGM.SizeTy->getPointerTo()); - auto fieldOffset = - IGF.Builder.CreateLoad(fieldOffsetPtr, - IGF.IGM.getPointerAlignment()); + IGF.IGM.Int32Ty->getPointerTo()); + llvm::Value *fieldOffset = + IGF.Builder.CreateLoad(fieldOffsetPtr, Alignment(4)); + fieldOffset = IGF.Builder.CreateZExtOrBitCast(fieldOffset, + IGF.IGM.SizeTy); IGF.verifyValues(metadata, fieldOffset, IGF.IGM.getSize(field.getFixedByteOffset()), @@ -333,8 +488,70 @@ namespace { } } } + + const FieldInfoType * + getFixedExtraInhabitantProvidingField(IRGenModule &IGM) const { + if (!ExtraInhabitantProvidingField.hasValue()) { + unsigned mostExtraInhabitants = 0; + const FieldInfoType *fieldWithMost = nullptr; + const FieldInfoType *singleNonFixedField = nullptr; + + // TODO: If two fields have the same type, they have the same extra + // inhabitant count, and we'll pick the first. We don't have to check + // both. However, we don't always have access to the substituted struct + // type from this context, which would be necessary to make that + // judgment reliably. + + for (auto &field : asImpl().getFields()) { + auto &ti = field.getTypeInfo(); + if (!ti.mayHaveExtraInhabitants(IGM)) + continue; + + auto *fixed = dyn_cast(&field.getTypeInfo()); + // If any field is non-fixed, we can't definitively pick a best one, + // unless it happens to be the only non-fixed field and none of the + // other fields have extra inhabitants. + if (!fixed) { + // If we already saw a non-fixed field, then we can't pick one + // at compile time. + if (singleNonFixedField) { + singleNonFixedField = fieldWithMost = nullptr; + break; + } + + // Otherwise, note this field for later. If we have no fixed + // candidates, it may be the only choice for extra inhabitants. + singleNonFixedField = &field; + continue; + } + + unsigned count = fixed->getFixedExtraInhabitantCount(IGM); + if (count > mostExtraInhabitants) { + mostExtraInhabitants = count; + fieldWithMost = &field; + } + } + + if (fieldWithMost) { + if (singleNonFixedField) { + // If we have a non-fixed and fixed candidate, we can't know for + // sure now. + ExtraInhabitantProvidingField = nullptr; + } else { + // If we had all fixed fields, pick the one with the most extra + // inhabitants. + ExtraInhabitantProvidingField = fieldWithMost; + } + } else { + // If there were no fixed candidates, but we had a single non-fixed + // field with potential extra inhabitants, then it's our only choice. + ExtraInhabitantProvidingField = singleNonFixedField; + } + } + return *ExtraInhabitantProvidingField; + } }; - + /// A type implementation for loadable record types imported from Clang. class ClangRecordTypeInfo final : public StructTypeInfoBase { public: - // FIXME: Spare bits between struct members. LoadableStructTypeInfo(ArrayRef fields, unsigned explosionSize, llvm::Type *storageType, Size size, @@ -893,7 +1110,7 @@ TypeConverter::convertResilientStruct(IsABIAccessible_t abiAccessible) { } const TypeInfo *TypeConverter::convertStructType(TypeBase *key, CanType type, - StructDecl *D) { + StructDecl *D){ // All resilient structs have the same opaque lowering, since they are // indistinguishable as values --- except that we have to track // ABI-accessibility. diff --git a/lib/IRGen/GenTuple.cpp b/lib/IRGen/GenTuple.cpp index c24b78533dc0f..197634ab20725 100644 --- a/lib/IRGen/GenTuple.cpp +++ b/lib/IRGen/GenTuple.cpp @@ -204,23 +204,27 @@ namespace { llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF, Address tupleAddr, - SILType tupleType) const override { + SILType tupleType, + bool isOutlined) const override { Address eltAddr = asImpl().projectElementAddress(IGF, tupleAddr, tupleType, 0); auto &elt = asImpl().getFields()[0]; return elt.getTypeInfo().getExtraInhabitantIndex(IGF, eltAddr, - elt.getType(IGF.IGM, tupleType)); + elt.getType(IGF.IGM, tupleType), + false /*not outlined for field*/); } void storeExtraInhabitant(IRGenFunction &IGF, llvm::Value *index, Address tupleAddr, - SILType tupleType) const override { + SILType tupleType, + bool isOutlined) const override { Address eltAddr = asImpl().projectElementAddress(IGF, tupleAddr, tupleType, 0); auto &elt = asImpl().getFields()[0]; elt.getTypeInfo().storeExtraInhabitant(IGF, index, eltAddr, - elt.getType(IGF.IGM, tupleType)); + elt.getType(IGF.IGM, tupleType), + false /*not outlined for field*/); } void verify(IRGenTypeVerifierFunction &IGF, diff --git a/lib/IRGen/GenType.cpp b/lib/IRGen/GenType.cpp index 127fac8980a5c..0ea631c80e60a 100644 --- a/lib/IRGen/GenType.cpp +++ b/lib/IRGen/GenType.cpp @@ -510,7 +510,7 @@ llvm::Value *FixedTypeInfo::getEnumTagSinglePayload(IRGenFunction &IGF, // If there are extra inhabitants, see whether the payload is valid. llvm::Value *result0; if (mayHaveExtraInhabitants(IGM)) { - result0 = getExtraInhabitantIndex(IGF, enumAddr, T); + result0 = getExtraInhabitantIndex(IGF, enumAddr, T, false); noExtraTagBitsBB = Builder.GetInsertBlock(); } else { result0 = llvm::ConstantInt::getSigned(IGM.Int32Ty, -1); @@ -665,7 +665,8 @@ void FixedTypeInfo::storeEnumTagSinglePayload(IRGenFunction &IGF, if (mayHaveExtraInhabitants(IGM)) { // Store an index in the range [0..ElementsWithNoPayload-1]. auto *nonPayloadElementIndex = Builder.CreateSub(whichCase, one); - storeExtraInhabitant(IGF, nonPayloadElementIndex, enumAddr, T); + storeExtraInhabitant(IGF, nonPayloadElementIndex, enumAddr, T, + /*outlined*/ false); } Builder.CreateBr(returnBB); @@ -868,7 +869,8 @@ namespace { llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF, Address src, - SILType T) const override { + SILType T, + bool isOutlined) const override { // Copied from BridgeObjectTypeInfo. src = IGF.Builder.CreateBitCast(src, IGF.IGM.IntPtrTy->getPointerTo()); auto val = IGF.Builder.CreateLoad(src); @@ -880,7 +882,8 @@ namespace { } void storeExtraInhabitant(IRGenFunction &IGF, llvm::Value *index, - Address dest, SILType T) const override { + Address dest, SILType T, + bool isOutlined) const override { // Copied from BridgeObjectTypeInfo. // There's only one extra inhabitant, 0. dest = IGF.Builder.CreateBitCast(dest, IGF.IGM.IntPtrTy->getPointerTo()); diff --git a/lib/IRGen/GenValueWitness.cpp b/lib/IRGen/GenValueWitness.cpp index c9a78be1eae20..ab3749ab90972 100644 --- a/lib/IRGen/GenValueWitness.cpp +++ b/lib/IRGen/GenValueWitness.cpp @@ -515,7 +515,8 @@ static void buildValueWitnessFunction(IRGenModule &IGM, llvm::Value *index = getArg(argv, "index"); getArgAsLocalSelfTypeMetadata(IGF, argv, abstractType); - type.storeExtraInhabitant(IGF, index, dest, concreteType); + type.storeExtraInhabitant(IGF, index, dest, concreteType, + /*outlined*/ true); IGF.Builder.CreateRetVoid(); return; } @@ -524,7 +525,8 @@ static void buildValueWitnessFunction(IRGenModule &IGM, Address src = getArgAs(IGF, argv, type, "src"); getArgAsLocalSelfTypeMetadata(IGF, argv, abstractType); - llvm::Value *idx = type.getExtraInhabitantIndex(IGF, src, concreteType); + llvm::Value *idx = type.getExtraInhabitantIndex(IGF, src, concreteType, + /*outlined*/ true); IGF.Builder.CreateRet(idx); return; } diff --git a/lib/IRGen/HeapTypeInfo.h b/lib/IRGen/HeapTypeInfo.h index 718856fb23a91..cbf07b38898c3 100644 --- a/lib/IRGen/HeapTypeInfo.h +++ b/lib/IRGen/HeapTypeInfo.h @@ -237,13 +237,14 @@ class HeapTypeInfo : public SingleScalarTypeInfo { } llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF, Address src, - SILType T) + SILType T, bool isOutlined) const override { return getHeapObjectExtraInhabitantIndex(IGF, src); } void storeExtraInhabitant(IRGenFunction &IGF, llvm::Value *index, - Address dest, SILType T) const override { + Address dest, SILType T, bool isOutlined) + const override { return storeHeapObjectExtraInhabitant(IGF, index, dest); } }; diff --git a/lib/IRGen/NonFixedTypeInfo.h b/lib/IRGen/NonFixedTypeInfo.h index 43e4e574ab8ea..9a3656b2a485f 100644 --- a/lib/IRGen/NonFixedTypeInfo.h +++ b/lib/IRGen/NonFixedTypeInfo.h @@ -118,12 +118,14 @@ class WitnessSizedTypeInfo : public IndirectTypeInfo { /// FIXME: Dynamic extra inhabitant lookup. bool mayHaveExtraInhabitants(IRGenModule &) const override { return false; } llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF, - Address src, SILType T) const override { + Address src, SILType T, + bool isOutlined) const override { llvm_unreachable("dynamic extra inhabitants not supported"); } void storeExtraInhabitant(IRGenFunction &IGF, llvm::Value *index, - Address dest, SILType T) const override { + Address dest, SILType T, + bool isOutlined) const override { llvm_unreachable("dynamic extra inhabitants not supported"); } diff --git a/lib/IRGen/ResilientTypeInfo.h b/lib/IRGen/ResilientTypeInfo.h index 0e495030bc342..65418e5407384 100644 --- a/lib/IRGen/ResilientTypeInfo.h +++ b/lib/IRGen/ResilientTypeInfo.h @@ -139,13 +139,15 @@ class ResilientTypeInfo : public WitnessSizedTypeInfo { } llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF, Address src, - SILType T) const override { + SILType T, + bool isOutlined) const override { return emitGetExtraInhabitantIndexCall(IGF, T, src); } void storeExtraInhabitant(IRGenFunction &IGF, llvm::Value *index, Address dest, - SILType T) const override { + SILType T, + bool isOutlined) const override { emitStoreExtraInhabitantCall(IGF, T, index, dest); } diff --git a/lib/IRGen/TypeInfo.h b/lib/IRGen/TypeInfo.h index b2b2b6eaa3d04..cd123f1308797 100644 --- a/lib/IRGen/TypeInfo.h +++ b/lib/IRGen/TypeInfo.h @@ -409,7 +409,8 @@ class TypeInfo { /// has extra inhabitants. virtual llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF, Address src, - SILType T) const = 0; + SILType T, + bool isOutlined) const = 0; /// Store the extra inhabitant representation indexed by a 31-bit identifier /// to memory. @@ -419,7 +420,8 @@ class TypeInfo { virtual void storeExtraInhabitant(IRGenFunction &IGF, llvm::Value *index, Address dest, - SILType T) const = 0; + SILType T, + bool isOutlined) const = 0; /// Get the tag of a single payload enum with a payload of this type (\p T) e.g /// Optional. diff --git a/stdlib/public/runtime/Metadata.cpp b/stdlib/public/runtime/Metadata.cpp index 19773a4b6a99f..6e9f17ef0c7f4 100644 --- a/stdlib/public/runtime/Metadata.cpp +++ b/stdlib/public/runtime/Metadata.cpp @@ -1939,7 +1939,7 @@ static ValueWitnessTable *getMutableVWTableForInit(StructMetadata *self, } /// Initialize the value witness table and struct field offset vector for a -/// struct, using the "Universal" layout strategy. +/// struct. void swift::swift_initStructMetadata(StructMetadata *structType, StructLayoutFlags layoutFlags, size_t numFields, @@ -1952,17 +1952,29 @@ void swift::swift_initStructMetadata(StructMetadata *structType, assignUnlessEqual(fieldOffsets[i], offset); }); - bool hasExtraInhabitants = fieldTypes[0]->flags.hasExtraInhabitants(); - + // We have extra inhabitants if any element does. Use the field with the most. + unsigned extraInhabitantField = ~0u; + unsigned extraInhabitantCount = 0; + for (unsigned i = 0; i < numFields; ++i) { + if (!fieldTypes[i]->flags.hasExtraInhabitants()) + continue; + unsigned fieldExtraInhabitantCount = + fieldTypes[i]->getExtraInhabitantFlags().getNumExtraInhabitants(); + if (fieldExtraInhabitantCount > extraInhabitantCount) { + extraInhabitantField = i; + extraInhabitantCount = fieldExtraInhabitantCount; + } + } + auto vwtable = - getMutableVWTableForInit(structType, layoutFlags, hasExtraInhabitants); + getMutableVWTableForInit(structType, layoutFlags, + extraInhabitantCount > 0); - // We have extra inhabitants if the first element does. - // FIXME: generalize this. - if (hasExtraInhabitants) { + if (extraInhabitantCount > 0) { layout.flags = layout.flags.withExtraInhabitants(true); auto xiVWT = static_cast(vwtable); - xiVWT->extraInhabitantFlags = fieldTypes[0]->getExtraInhabitantFlags(); + xiVWT->extraInhabitantFlags = + fieldTypes[extraInhabitantField]->getExtraInhabitantFlags(); // The compiler should already have initialized these. assert(xiVWT->storeExtraInhabitant); diff --git a/test/Interpreter/Inputs/struct_extra_inhabitants_ExtraInhabitantResilientTypes.swift b/test/Interpreter/Inputs/struct_extra_inhabitants_ExtraInhabitantResilientTypes.swift new file mode 100644 index 0000000000000..088379fdf8b70 --- /dev/null +++ b/test/Interpreter/Inputs/struct_extra_inhabitants_ExtraInhabitantResilientTypes.swift @@ -0,0 +1,7 @@ +public struct ResilientXI { + var x: AnyObject +} + +public struct ResilientNoXI { + var x: Int +} diff --git a/test/Interpreter/struct_extra_inhabitants.swift b/test/Interpreter/struct_extra_inhabitants.swift new file mode 100644 index 0000000000000..5c11bfed65d68 --- /dev/null +++ b/test/Interpreter/struct_extra_inhabitants.swift @@ -0,0 +1,169 @@ +// RUN: %empty-directory(%t) + +// -- build resilient library +// RUN: %target-build-swift -force-single-frontend-invocation -Xfrontend -enable-resilience -module-name ExtraInhabitantResilientTypes -emit-module-path %t/ExtraInhabitantResilientTypes.swiftmodule -parse-as-library -c -o %t/ExtraInhabitantResilientTypes.o %S/Inputs/struct_extra_inhabitants_ExtraInhabitantResilientTypes.swift + +// -- run tests +// RUN: %target-build-swift -parse-stdlib -Xfrontend -verify-type-layout -Xfrontend PairWithPointerFirst -Xfrontend -verify-type-layout -Xfrontend PairWithPointerSecond -Xfrontend -verify-type-layout -Xfrontend PairWithPointerSecondAndPhantomParam_Int -Xfrontend -verify-type-layout -Xfrontend GenericPairWithPointerFirst_Int -Xfrontend -verify-type-layout -Xfrontend GenericPairWithPointerFirst_AnyObject -Xfrontend -verify-type-layout -Xfrontend GenericPairWithPointerSecond_Int -Xfrontend -verify-type-layout -Xfrontend GenericPairWithPointerSecond_AnyObject -Xfrontend -verify-type-layout -Xfrontend StringAlike32 -Xfrontend -verify-type-layout -Xfrontend StringAlike64 -I %t -o %t/a.out.tests %s %t/ExtraInhabitantResilientTypes.o +// RUN: %target-run %t/a.out.tests 2>&1 + +// Type layout verifier is only compiled into the runtime in asserts builds. +// REQUIRES: swift_stdlib_asserts + +// CHECK-NOT: Type verification + +import Swift +import ExtraInhabitantResilientTypes +import StdlibUnittest + +// Enum layout should use extra inhabitants from any struct field +// for fixed-layout types. +struct PairWithPointerFirst { + var a: AnyObject + var b: Int +} + +struct PairWithPointerSecond { + var a: Int + var b: AnyObject +} + +struct PairWithPointerSecondAndPhantomParam { + var a: Int + var b: AnyObject +} + +struct StringAlike64 { + var a: Int + var b: Builtin.BridgeObject +} +struct StringAlike32 { + var a,b,c: Int + var d: Builtin.BridgeObject +} + +// TODO: Runtime struct instantiation still only considers the first argument + +struct GenericPair { + var a: T + var b: U +} + +struct GenericSamePair { + var a: T + var b: T +} + +struct GenericPairPlusPointer { + var a: T + var b: U + var c: UnsafeRawPointer +} + +struct GenericSamePairPlusPointer { + var a: T + var b: T + var c: UnsafeRawPointer +} + +struct GenericPairWithPointerFirst { + var a: AnyObject + var b: T +} + +struct GenericPairWithPointerSecond { + var a: T + var b: AnyObject +} + +struct GenericFullHouse { + var a, b, c: T + var d, e: U +} + +struct ResilientPairWithXIFirst { + var a: ResilientXI + var b: ResilientNoXI +} + +struct ResilientPairWithXISecond { + var a: ResilientNoXI + var b: ResilientXI +} + +// Typealiases for the type layout verifier + +typealias PairWithPointerSecondAndPhantomParam_Int + = PairWithPointerSecondAndPhantomParam + +typealias GenericPairWithPointerFirst_Int + = GenericPairWithPointerFirst + +typealias GenericPairWithPointerFirst_AnyObject + = GenericPairWithPointerFirst + +typealias GenericPairWithPointerSecond_Int + = GenericPairWithPointerSecond + +typealias GenericPairWithPointerSecond_AnyObject + = GenericPairWithPointerSecond + +var tests = TestSuite("extra inhabitants of structs") + +@inline(never) +func expectHasExtraInhabitant(_: T.Type, nil: T?, + file: String = #file, line: UInt = #line) { + expectEqual(MemoryLayout.size, MemoryLayout.size, + "\(T.self) has extra inhabitant", + file: file, line: line) + + expectNil(Optional.none, + "\(T.self) extra inhabitant should agree in generic and concrete " + + "context") +} + +func expectHasNoExtraInhabitant(_: T.Type, + file: String = #file, line: UInt = #line) { + expectNotEqual(MemoryLayout.size, MemoryLayout.size, + "\(T.self) does not have extra inhabitant", + file: file, line: line) +} + +tests.test("types that have extra inhabitants") { + expectHasExtraInhabitant(PairWithPointerFirst.self, nil: nil) + expectHasExtraInhabitant(PairWithPointerSecond.self, nil: nil) + expectHasExtraInhabitant(PairWithPointerSecondAndPhantomParam.self, nil: nil) + expectHasExtraInhabitant(PairWithPointerSecondAndPhantomParam.self, nil: nil) + expectHasExtraInhabitant(GenericPairWithPointerFirst.self, nil: nil) + expectHasExtraInhabitant(GenericPairWithPointerFirst.self, nil: nil) + expectHasExtraInhabitant(GenericPairWithPointerSecond.self, nil: nil) + expectHasExtraInhabitant(GenericPairWithPointerSecond.self, nil: nil) + expectHasExtraInhabitant(ResilientPairWithXIFirst.self, nil: nil) + expectHasExtraInhabitant(ResilientPairWithXISecond.self, nil: nil) + expectHasExtraInhabitant(StringAlike64.self, nil: nil) + expectHasExtraInhabitant(StringAlike32.self, nil: nil) + expectHasExtraInhabitant(GenericPair.self, nil: nil) + expectHasExtraInhabitant(GenericPair.self, nil: nil) + expectHasExtraInhabitant(GenericPair.self, nil: nil) + expectHasExtraInhabitant(GenericPair.self, nil: nil) + expectHasExtraInhabitant(GenericPair.self, nil: nil) + expectHasExtraInhabitant(GenericPair.self, nil: nil) + expectHasExtraInhabitant(GenericPairPlusPointer.self, nil: nil) + expectHasExtraInhabitant(GenericPairPlusPointer.self, nil: nil) + expectHasExtraInhabitant(GenericPairPlusPointer.self, nil: nil) + expectHasExtraInhabitant(GenericPairPlusPointer.self, nil: nil) + expectHasExtraInhabitant(GenericSamePair.self, nil: nil) + expectHasExtraInhabitant(GenericSamePair.self, nil: nil) + expectHasExtraInhabitant(GenericSamePairPlusPointer.self, nil: nil) + expectHasExtraInhabitant(GenericSamePairPlusPointer.self, nil: nil) + expectHasExtraInhabitant(GenericSamePairPlusPointer.self, nil: nil) + expectHasExtraInhabitant(GenericFullHouse.self, nil: nil) + expectHasExtraInhabitant(GenericFullHouse.self, nil: nil) + expectHasExtraInhabitant(GenericFullHouse.self, nil: nil) + expectHasExtraInhabitant(GenericFullHouse.self, nil: nil) + expectHasExtraInhabitant(GenericFullHouse.self, nil: nil) + expectHasExtraInhabitant(GenericFullHouse.self, nil: nil) +} + +runAllTests() +