diff --git a/include/swift/ABI/CompactFunctionPointer.h b/include/swift/ABI/CompactFunctionPointer.h new file mode 100644 index 0000000000000..dfcc63c807917 --- /dev/null +++ b/include/swift/ABI/CompactFunctionPointer.h @@ -0,0 +1,62 @@ +//===--- CompactFunctionPointer.h - Compact Function Pointers ---*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Swift's runtime structures often use relative function pointers to reduce the +// size of metadata and also to minimize load-time overhead in PIC. +// This file defines pointer types whose size and interface are compatible with +// the relative pointer types for targets that do not support relative references +// to code from data. +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_ABI_COMPACTFUNCTIONPOINTER_H +#define SWIFT_ABI_COMPACTFUNCTIONPOINTER_H + +namespace swift { + +/// A compact unconditional absolute function pointer that can fit in a 32-bit +/// integer. +/// As a trade-off compared to relative pointers, this has load-time overhead in PIC +/// and is only available on 32-bit targets. +template +class AbsoluteFunctionPointer { + T *Pointer; + static_assert(sizeof(T *) == sizeof(int32_t), + "Function pointer must be 32-bit when using compact absolute pointer"); + +public: + using PointerTy = T *; + + PointerTy get() const & { return Pointer; } + + operator PointerTy() const & { return this->get(); } + + bool isNull() const & { return Pointer == nullptr; } + + /// Resolve a pointer from a `base` pointer and a value loaded from `base`. + template + static PointerTy resolve(BasePtrTy *base, Value value) { + return reinterpret_cast(value); + } + + template + typename std::result_of::type operator()(ArgTy... arg) const { + static_assert(std::is_function::value, + "T must be an explicit function type"); + return this->get()(std::forward(arg)...); + } +}; + +// TODO(katei): Add another pointer structure for 64-bit targets and for efficiency on PIC + +} // namespace swift + +#endif // SWIFT_ABI_COMPACTFUNCTIONPOINTER_H diff --git a/include/swift/ABI/Executor.h b/include/swift/ABI/Executor.h index 47274269211c1..e6e54c26016f6 100644 --- a/include/swift/ABI/Executor.h +++ b/include/swift/ABI/Executor.h @@ -241,7 +241,7 @@ template class AsyncFunctionPointer { public: /// The function to run. - RelativeDirectPointer, + TargetCompactFunctionPointer, /*nullable*/ false, int32_t> Function; diff --git a/include/swift/ABI/Metadata.h b/include/swift/ABI/Metadata.h index 3cc204372a430..50ae7484884e0 100644 --- a/include/swift/ABI/Metadata.h +++ b/include/swift/ABI/Metadata.h @@ -506,9 +506,20 @@ struct TargetMethodDescriptor { MethodDescriptorFlags Flags; /// The method implementation. - TargetRelativeDirectPointer Impl; + union { + TargetCompactFunctionPointer Impl; + TargetRelativeDirectPointer AsyncImpl; + }; // TODO: add method types or anything else needed for reflection. + + void *getImpl() const { + if (Flags.isAsync()) { + return AsyncImpl.get(); + } else { + return Impl.get(); + } + } }; using MethodDescriptor = TargetMethodDescriptor; @@ -578,7 +589,20 @@ struct TargetMethodOverrideDescriptor { TargetRelativeMethodDescriptorPointer Method; /// The implementation of the override. - TargetRelativeDirectPointer Impl; + union { + TargetCompactFunctionPointer Impl; + TargetRelativeDirectPointer AsyncImpl; + }; + + void *getImpl() const { + auto *baseMethod = Method.get(); + assert(baseMethod && "no base method"); + if (baseMethod->Flags.isAsync()) { + return AsyncImpl.get(); + } else { + return Impl.get(); + } + } }; /// Header for a class vtable override descriptor. This is a variable-sized @@ -1523,7 +1547,18 @@ struct TargetProtocolRequirement { // TODO: name, type /// The optional default implementation of the protocol. - RelativeDirectPointer DefaultImplementation; + union { + TargetCompactFunctionPointer DefaultFuncImplementation; + TargetRelativeDirectPointer DefaultImplementation; + }; + + void *getDefaultImplementation() const { + if (Flags.isFunctionImpl()) { + return DefaultFuncImplementation.get(); + } else { + return DefaultImplementation.get(); + } + } }; using ProtocolRequirement = TargetProtocolRequirement; @@ -2170,7 +2205,18 @@ using GenericBoxHeapMetadata = TargetGenericBoxHeapMetadata; template struct TargetResilientWitness { TargetRelativeProtocolRequirementPointer Requirement; - RelativeDirectPointer Witness; + union { + TargetRelativeDirectPointer Impl; + TargetCompactFunctionPointer FuncImpl; + }; + + void *getWitness(ProtocolRequirementFlags flags) const { + if (flags.isFunctionImpl()) { + return FuncImpl.get(); + } else { + return Impl.get(); + } + } }; using ResilientWitness = TargetResilientWitness; @@ -2233,10 +2279,13 @@ struct TargetGenericWitnessTable { uint16_t WitnessTablePrivateSizeInWordsAndRequiresInstantiation; /// The instantiation function, which is called after the template is copied. - RelativeDirectPointer *instantiatedTable, - const TargetMetadata *type, - const void * const *instantiationArgs), - /*nullable*/ true> Instantiator; + TargetCompactFunctionPointer< + Runtime, + void(TargetWitnessTable *instantiatedTable, + const TargetMetadata *type, + const void *const *instantiationArgs), + /*nullable*/ true> + Instantiator; using PrivateDataType = void *[swift::NumGenericMetadataPrivateDataWords]; @@ -2968,12 +3017,12 @@ using MetadataCompleter = template struct TargetGenericMetadataPattern { /// The function to call to instantiate the template. - TargetRelativeDirectPointer + TargetCompactFunctionPointer InstantiationFunction; /// The function to call to complete the instantiation. If this is null, /// the instantiation function must always generate complete metadata. - TargetRelativeDirectPointer + TargetCompactFunctionPointer CompletionFunction; /// Flags describing the layout of this instantiation pattern. @@ -3080,10 +3129,10 @@ struct TargetGenericClassMetadataPattern final : using TargetGenericMetadataPattern::PatternFlags; /// The heap-destructor function. - TargetRelativeDirectPointer Destroy; + TargetCompactFunctionPointer Destroy; /// The ivar-destructor function. - TargetRelativeDirectPointer + TargetCompactFunctionPointer IVarDestroyer; /// The class flags. @@ -3284,7 +3333,7 @@ class MetadataAccessFunction { template struct TargetForeignMetadataInitialization { /// The completion function. The pattern will always be null. - TargetRelativeDirectPointer + TargetCompactFunctionPointer CompletionFunction; }; @@ -3329,14 +3378,14 @@ struct TargetResilientClassMetadataPattern { /// /// If this is null, the runtime instead calls swift_relocateClassMetadata(), /// passing in the class descriptor and this pattern. - TargetRelativeDirectPointer + TargetCompactFunctionPointer RelocationFunction; /// The heap-destructor function. - TargetRelativeDirectPointer Destroy; + TargetCompactFunctionPointer Destroy; /// The ivar-destructor function. - TargetRelativeDirectPointer + TargetCompactFunctionPointer IVarDestroyer; /// The class flags. @@ -3380,7 +3429,7 @@ struct TargetSingletonMetadataInitialization { /// The completion function. The pattern will always be null, even /// for a resilient class. - TargetRelativeDirectPointer + TargetCompactFunctionPointer CompletionFunction; bool hasResilientClassPattern( @@ -3409,7 +3458,7 @@ struct TargetCanonicalSpecializedMetadatasListEntry { template struct TargetCanonicalSpecializedMetadataAccessorsListEntry { - TargetRelativeDirectPointer accessor; + TargetCompactFunctionPointer accessor; }; template @@ -3429,7 +3478,7 @@ class TargetTypeContextDescriptor /// The function type here is a stand-in. You should use getAccessFunction() /// to wrap the function pointer in an accessor that uses the proper calling /// convention for a given number of arguments. - TargetRelativeDirectPointer AccessFunctionPtr; /// A pointer to the field descriptor for the type, if any. @@ -3694,7 +3743,7 @@ class TargetClassDescriptor final using MetadataListEntry = TargetCanonicalSpecializedMetadatasListEntry; using MetadataAccessor = - TargetRelativeDirectPointer; + TargetCompactFunctionPointer; using MetadataAccessorListEntry = TargetCanonicalSpecializedMetadataAccessorsListEntry; using MetadataCachingOnceToken = @@ -4495,12 +4544,23 @@ class DynamicReplacementDescriptor { DynamicReplacementKey * __ptrauth_swift_dynamic_replacement_key>> replacedFunctionKey; - RelativeDirectPointer replacementFunction; + union { + TargetCompactFunctionPointer replacementFunction; + TargetRelativeDirectPointer replacementAsyncFunction; + }; RelativeDirectPointer chainEntry; uint32_t flags; enum : uint32_t { EnableChainingMask = 0x1 }; + void *getReplacementFunction() const { + if (replacedFunctionKey->isAsync()) { + return replacementAsyncFunction.get(); + } else { + return replacementFunction.get(); + } + } + public: /// Enable this replacement by changing the function's replacement chain's /// root entry. diff --git a/include/swift/ABI/MetadataValues.h b/include/swift/ABI/MetadataValues.h index f6be29b27b02f..6a7a01e5fdf9b 100644 --- a/include/swift/ABI/MetadataValues.h +++ b/include/swift/ABI/MetadataValues.h @@ -611,6 +611,21 @@ class ProtocolRequirementFlags { int_type getIntValue() const { return Value; } + /// Is the method implementation is represented as a native function pointer? + bool isFunctionImpl() const { + switch (getKind()) { + case ProtocolRequirementFlags::Kind::Method: + case ProtocolRequirementFlags::Kind::Init: + case ProtocolRequirementFlags::Kind::Getter: + case ProtocolRequirementFlags::Kind::Setter: + case ProtocolRequirementFlags::Kind::ReadCoroutine: + case ProtocolRequirementFlags::Kind::ModifyCoroutine: + return !isAsync(); + default: + return false; + } + } + enum : uintptr_t { /// Bit used to indicate that an associated type witness is a pointer to /// a mangled name (vs. a pointer to metadata). diff --git a/include/swift/ABI/TargetLayout.h b/include/swift/ABI/TargetLayout.h index 661a9189808c2..ae49d395448dd 100644 --- a/include/swift/ABI/TargetLayout.h +++ b/include/swift/ABI/TargetLayout.h @@ -31,6 +31,7 @@ #include "swift/Runtime/Config.h" #include "swift/Basic/RelativePointer.h" +#include "swift/ABI/CompactFunctionPointer.h" namespace swift { @@ -101,6 +102,14 @@ struct InProcess { template using RelativeDirectPointer = RelativeDirectPointer; + template +#if SWIFT_COMPACT_ABSOLUTE_FUNCTION_POINTER + using CompactFunctionPointer = AbsoluteFunctionPointer; +#else + using CompactFunctionPointer = + swift::RelativeDirectPointer; +#endif + template T *getStrippedSignedPointer(const T *pointer) const { return swift_ptrauth_strip(pointer); @@ -163,6 +172,9 @@ struct External { template using RelativeDirectPointer = int32_t; + template + using CompactFunctionPointer = int32_t; + StoredPointer getStrippedSignedPointer(const StoredSignedPointer pointer) const { return swift_ptrauth_strip(pointer); } @@ -191,6 +203,12 @@ template using TargetRelativeIndirectablePointer = typename Runtime::template RelativeIndirectablePointer; +template +using TargetCompactFunctionPointer = + typename Runtime::template CompactFunctionPointer; + } // end namespace swift #endif diff --git a/include/swift/Basic/RelativePointer.h b/include/swift/Basic/RelativePointer.h index 00bc13a32b67c..a1c8ce7002c36 100644 --- a/include/swift/Basic/RelativePointer.h +++ b/include/swift/Basic/RelativePointer.h @@ -386,6 +386,10 @@ class RelativeIndirectablePointerIntPair { /// position-independent constant data. template class RelativeDirectPointerImpl { +#if SWIFT_COMPACT_ABSOLUTE_FUNCTION_POINTER + static_assert(!std::is_function::value, + "relative direct function pointer should not be used under absolute function pointer mode"); +#endif private: /// The relative offset of the function's entry point from *this. Offset RelativeOffset; @@ -454,19 +458,25 @@ class RelativeDirectPointerImpl { /// Apply the offset to a parameter, instead of `this`. PointerTy getRelative(void *base) const & { - // Check for null. - if (Nullable && RelativeOffset == 0) - return nullptr; - - // The value is addressed relative to `base`. - uintptr_t absolute = detail::applyRelativeOffset(base, RelativeOffset); - return reinterpret_cast(absolute); + return resolve(base, RelativeOffset); } /// A zero relative offset encodes a null reference. bool isNull() const & { return RelativeOffset == 0; } + + /// Resolve a pointer from a `base` pointer and a value loaded from `base`. + template + static PointerTy resolve(BasePtrTy *base, Offset value) { + // Check for null. + if (Nullable && value == 0) + return nullptr; + + // The value is addressed relative to `base`. + uintptr_t absolute = detail::applyRelativeOffset(base, value); + return reinterpret_cast(absolute); + } }; template UnsafeRawPointer { +#if SWIFT_COMPACT_ABSOLUTE_FUNCTION_POINTER + return UnsafeRawPointer(bitPattern: Int(offset)).unsafelyUnwrapped +#else + return _resolveRelativeAddress(base, offset) +#endif +} + internal func _loadRelativeAddress(at: UnsafeRawPointer, fromByteOffset: Int = 0, as: T.Type) -> T { @@ -2780,12 +2790,12 @@ internal func _walkKeyPathPattern( let idValue = _pop(from: &componentBuffer, as: Int32.self) let getterBase = componentBuffer.baseAddress.unsafelyUnwrapped let getterRef = _pop(from: &componentBuffer, as: Int32.self) - let getter = _resolveRelativeAddress(getterBase, getterRef) + let getter = _resolveCompactFunctionPointer(getterBase, getterRef) let setter: UnsafeRawPointer? if header.isComputedSettable { let setterBase = componentBuffer.baseAddress.unsafelyUnwrapped let setterRef = _pop(from: &componentBuffer, as: Int32.self) - setter = _resolveRelativeAddress(setterBase, setterRef) + setter = _resolveCompactFunctionPointer(setterBase, setterRef) } else { setter = nil } @@ -2799,7 +2809,7 @@ internal func _walkKeyPathPattern( if header.hasComputedArguments { let getLayoutBase = componentBuffer.baseAddress.unsafelyUnwrapped let getLayoutRef = _pop(from: &componentBuffer, as: Int32.self) - let getLayoutRaw = _resolveRelativeAddress(getLayoutBase, getLayoutRef) + let getLayoutRaw = _resolveCompactFunctionPointer(getLayoutBase, getLayoutRef) let getLayoutSigned = _PtrAuth.sign(pointer: getLayoutRaw, key: .processIndependentCode, discriminator: _PtrAuth.discriminator(for: KeyPathComputedArgumentLayoutFn.self)) @@ -2817,8 +2827,8 @@ internal func _walkKeyPathPattern( let initializerBase = componentBuffer.baseAddress.unsafelyUnwrapped let initializerRef = _pop(from: &componentBuffer, as: Int32.self) - let initializerRaw = _resolveRelativeAddress(initializerBase, - initializerRef) + let initializerRaw = _resolveCompactFunctionPointer(initializerBase, + initializerRef) let initializerSigned = _PtrAuth.sign(pointer: initializerRaw, key: .processIndependentCode, discriminator: _PtrAuth.discriminator(for: KeyPathComputedArgumentInitializerFn.self)) diff --git a/stdlib/public/runtime/Metadata.cpp b/stdlib/public/runtime/Metadata.cpp index e39416a5fa217..19fbbe06afec9 100644 --- a/stdlib/public/runtime/Metadata.cpp +++ b/stdlib/public/runtime/Metadata.cpp @@ -2762,7 +2762,7 @@ static void initClassVTable(ClassMetadata *self) { for (unsigned i = 0, e = vtable->VTableSize; i < e; ++i) { auto &methodDescription = descriptors[i]; swift_ptrauth_init_code_or_data( - &classWords[vtableOffset + i], methodDescription.Impl.get(), + &classWords[vtableOffset + i], methodDescription.getImpl(), methodDescription.Flags.getExtraDiscriminator(), !methodDescription.Flags.isAsync()); } @@ -2802,9 +2802,8 @@ static void initClassVTable(ClassMetadata *self) { auto baseVTable = baseClass->getVTableDescriptor(); auto offset = (baseVTable->getVTableOffset(baseClass) + (baseMethod - baseClassMethods.data())); - swift_ptrauth_init_code_or_data(&classWords[offset], - descriptor.Impl.get(), + descriptor.getImpl(), baseMethod->Flags.getExtraDiscriminator(), !baseMethod->Flags.isAsync()); } @@ -5139,7 +5138,7 @@ static void initializeResilientWitnessTable( auto &reqt = requirements[reqDescriptor - requirements.begin()]; // This is an unsigned pointer formed from a relative address. - void *impl = witness.Witness.get(); + void *impl = witness.getWitness(reqt.Flags); initProtocolWitness(&table[witnessIndex], impl, reqt); } @@ -5153,7 +5152,7 @@ static void initializeResilientWitnessTable( auto &reqt = requirements[i]; if (!table[witnessIndex]) { // This is an unsigned pointer formed from a relative address. - void *impl = reqt.DefaultImplementation.get(); + void *impl = reqt.getDefaultImplementation(); initProtocolWitness(&table[witnessIndex], impl, reqt); } @@ -5622,7 +5621,7 @@ static const WitnessTable *swift_getAssociatedConformanceWitnessSlowImpl( // Resolve the relative reference to the witness function. int32_t offset; memcpy(&offset, mangledName.data() + 1, 4); - uintptr_t ptr = detail::applyRelativeOffset(mangledName.data() + 1, offset); + void *ptr = TargetCompactFunctionPointer::resolve(mangledName.data() + 1, offset); // Call the witness function. AssociatedWitnessTableAccessFunction *witnessFn; diff --git a/stdlib/public/runtime/MetadataLookup.cpp b/stdlib/public/runtime/MetadataLookup.cpp index 21c7773171860..d0815b7e04736 100644 --- a/stdlib/public/runtime/MetadataLookup.cpp +++ b/stdlib/public/runtime/MetadataLookup.cpp @@ -76,7 +76,16 @@ static uintptr_t resolveSymbolicReferenceOffset(SymbolicReferenceKind kind, Directness isIndirect, int32_t offset, const void *base) { - auto ptr = detail::applyRelativeOffset(base, offset); + uintptr_t ptr; + // Function references may be resolved differently than other data references. + switch (kind) { + case SymbolicReferenceKind::AccessorFunctionReference: + ptr = (uintptr_t)TargetCompactFunctionPointer::resolve(base, offset); + break; + default: + ptr = detail::applyRelativeOffset(base, offset); + break; + } // Indirect references may be authenticated in a way appropriate for the // referent. @@ -2600,10 +2609,10 @@ void DynamicReplacementDescriptor::enableReplacement() const { // Link the replacement entry. chainRoot->next = chainEntry.get(); - // chainRoot->implementationFunction = replacementFunction.get(); + // chainRoot->implementationFunction = getReplacementFunction(); swift_ptrauth_init_code_or_data( reinterpret_cast(&chainRoot->implementationFunction), - reinterpret_cast(replacementFunction.get()), + reinterpret_cast(getReplacementFunction()), replacedFunctionKey->getExtraDiscriminator(), !replacedFunctionKey->isAsync()); }