diff --git a/llvm/include/llvm/Analysis/MemoryLocation.h b/llvm/include/llvm/Analysis/MemoryLocation.h index 85ca84e68a139..b72a27cab86b3 100644 --- a/llvm/include/llvm/Analysis/MemoryLocation.h +++ b/llvm/include/llvm/Analysis/MemoryLocation.h @@ -64,16 +64,19 @@ class Value; // // If asked to represent a pathologically large value, this will degrade to // std::nullopt. +// Store Scalable information in bit 62 of Value. Scalable information is +// required to do Alias Analysis on Scalable quantities class LocationSize { enum : uint64_t { BeforeOrAfterPointer = ~uint64_t(0), - AfterPointer = BeforeOrAfterPointer - 1, + ScalableBit = uint64_t(1) << 62, + AfterPointer = (BeforeOrAfterPointer - 1) & ~ScalableBit, MapEmpty = BeforeOrAfterPointer - 2, MapTombstone = BeforeOrAfterPointer - 3, ImpreciseBit = uint64_t(1) << 63, // The maximum value we can represent without falling back to 'unknown'. - MaxValue = (MapTombstone - 1) & ~ImpreciseBit, + MaxValue = (MapTombstone - 1) & ~(ImpreciseBit | ScalableBit), }; uint64_t Value; @@ -82,12 +85,16 @@ class LocationSize { // public LocationSize ctor goes away. enum DirectConstruction { Direct }; - constexpr LocationSize(uint64_t Raw, DirectConstruction): Value(Raw) {} + constexpr LocationSize(uint64_t Raw, DirectConstruction) : Value(Raw) {} + constexpr LocationSize(uint64_t Raw, bool Scalable) + : Value(Raw > MaxValue ? AfterPointer + : Raw | (Scalable ? ScalableBit : uint64_t(0))) {} static_assert(AfterPointer & ImpreciseBit, "AfterPointer is imprecise by definition."); static_assert(BeforeOrAfterPointer & ImpreciseBit, "BeforeOrAfterPointer is imprecise by definition."); + static_assert(~(MaxValue & ScalableBit), "Max value don't have bit 62 set"); public: // FIXME: Migrate all users to construct via either `precise` or `upperBound`, @@ -98,12 +105,12 @@ class LocationSize { // this assumes the provided value is precise. constexpr LocationSize(uint64_t Raw) : Value(Raw > MaxValue ? AfterPointer : Raw) {} - - static LocationSize precise(uint64_t Value) { return LocationSize(Value); } + // Create non-scalable LocationSize + static LocationSize precise(uint64_t Value) { + return LocationSize(Value, false /*Scalable*/); + } static LocationSize precise(TypeSize Value) { - if (Value.isScalable()) - return afterPointer(); - return precise(Value.getFixedValue()); + return LocationSize(Value.getKnownMinValue(), Value.isScalable()); } static LocationSize upperBound(uint64_t Value) { @@ -150,6 +157,8 @@ class LocationSize { return beforeOrAfterPointer(); if (Value == AfterPointer || Other.Value == AfterPointer) return afterPointer(); + if (isScalable() || Other.isScalable()) + return afterPointer(); return upperBound(std::max(getValue(), Other.getValue())); } @@ -157,19 +166,23 @@ class LocationSize { bool hasValue() const { return Value != AfterPointer && Value != BeforeOrAfterPointer; } - uint64_t getValue() const { + bool isScalable() const { return (Value & ScalableBit); } + + TypeSize getValue() const { assert(hasValue() && "Getting value from an unknown LocationSize!"); - return Value & ~ImpreciseBit; + assert((Value & ~(ImpreciseBit | ScalableBit)) < MaxValue && + "Scalable bit of value should be masked"); + return {Value & ~(ImpreciseBit | ScalableBit), isScalable()}; } // Returns whether or not this value is precise. Note that if a value is // precise, it's guaranteed to not be unknown. - bool isPrecise() const { - return (Value & ImpreciseBit) == 0; - } + bool isPrecise() const { return (Value & ImpreciseBit) == 0; } // Convenience method to check if this LocationSize's value is 0. - bool isZero() const { return hasValue() && getValue() == 0; } + bool isZero() const { + return hasValue() && getValue().getKnownMinValue() == 0; + } /// Whether accesses before the base pointer are possible. bool mayBeBeforePointer() const { return Value == BeforeOrAfterPointer; } @@ -178,9 +191,7 @@ class LocationSize { return Value == Other.Value; } - bool operator!=(const LocationSize &Other) const { - return !(*this == Other); - } + bool operator!=(const LocationSize &Other) const { return !(*this == Other); } // Ordering operators are not provided, since it's unclear if there's only one // reasonable way to compare: @@ -317,9 +328,7 @@ class MemoryLocation { // Specialize DenseMapInfo. template <> struct DenseMapInfo { - static inline LocationSize getEmptyKey() { - return LocationSize::mapEmpty(); - } + static inline LocationSize getEmptyKey() { return LocationSize::mapEmpty(); } static inline LocationSize getTombstoneKey() { return LocationSize::mapTombstone(); } @@ -349,6 +358,6 @@ template <> struct DenseMapInfo { return LHS == RHS; } }; -} +} // namespace llvm #endif diff --git a/llvm/lib/Analysis/BasicAliasAnalysis.cpp b/llvm/lib/Analysis/BasicAliasAnalysis.cpp index 16268bb25306f..2acd5c47b7681 100644 --- a/llvm/lib/Analysis/BasicAliasAnalysis.cpp +++ b/llvm/lib/Analysis/BasicAliasAnalysis.cpp @@ -100,8 +100,8 @@ bool BasicAAResult::invalidate(Function &Fn, const PreservedAnalyses &PA, // Useful predicates //===----------------------------------------------------------------------===// -/// Returns the size of the object specified by V or nullopt if unknown. -static std::optional getObjectSize(const Value *V, +/// Returns the size of the object specified by V or UnknownSize if unknown. +static std::optional getObjectSize(const Value *V, const DataLayout &DL, const TargetLibraryInfo &TLI, bool NullIsValidLoc, @@ -111,13 +111,13 @@ static std::optional getObjectSize(const Value *V, Opts.RoundToAlign = RoundToAlign; Opts.NullIsUnknownSize = NullIsValidLoc; if (getObjectSize(V, Size, DL, &TLI, Opts)) - return Size; + return TypeSize::getFixed(Size); return std::nullopt; } /// Returns true if we can prove that the object specified by V is smaller than /// Size. -static bool isObjectSmallerThan(const Value *V, uint64_t Size, +static bool isObjectSmallerThan(const Value *V, TypeSize Size, const DataLayout &DL, const TargetLibraryInfo &TLI, bool NullIsValidLoc) { @@ -152,16 +152,16 @@ static bool isObjectSmallerThan(const Value *V, uint64_t Size, // This function needs to use the aligned object size because we allow // reads a bit past the end given sufficient alignment. - std::optional ObjectSize = getObjectSize(V, DL, TLI, NullIsValidLoc, + std::optional ObjectSize = getObjectSize(V, DL, TLI, NullIsValidLoc, /*RoundToAlign*/ true); - return ObjectSize && *ObjectSize < Size; + return ObjectSize && TypeSize::isKnownLT(*ObjectSize, Size); } /// Return the minimal extent from \p V to the end of the underlying object, /// assuming the result is used in an aliasing query. E.g., we do use the query /// location size and the fact that null pointers cannot alias here. -static uint64_t getMinimalExtentFrom(const Value &V, +static TypeSize getMinimalExtentFrom(const Value &V, const LocationSize &LocSize, const DataLayout &DL, bool NullIsValidLoc) { @@ -176,14 +176,14 @@ static uint64_t getMinimalExtentFrom(const Value &V, // If queried with a precise location size, we assume that location size to be // accessed, thus valid. if (LocSize.isPrecise()) - DerefBytes = std::max(DerefBytes, LocSize.getValue()); - return DerefBytes; + DerefBytes = std::max(DerefBytes, LocSize.getValue().getKnownMinValue()); + return TypeSize::getFixed(DerefBytes); } /// Returns true if we can prove that the object specified by V has size Size. -static bool isObjectSize(const Value *V, uint64_t Size, const DataLayout &DL, +static bool isObjectSize(const Value *V, TypeSize Size, const DataLayout &DL, const TargetLibraryInfo &TLI, bool NullIsValidLoc) { - std::optional ObjectSize = + std::optional ObjectSize = getObjectSize(V, DL, TLI, NullIsValidLoc); return ObjectSize && *ObjectSize == Size; } @@ -1058,15 +1058,19 @@ AliasResult BasicAAResult::aliasGEP( // If an inbounds GEP would have to start from an out of bounds address // for the two to alias, then we can assume noalias. + // TODO: Remove !isScalable() once BasicAA fully support scalable location + // size if (*DecompGEP1.InBounds && DecompGEP1.VarIndices.empty() && - V2Size.hasValue() && DecompGEP1.Offset.sge(V2Size.getValue()) && + V2Size.hasValue() && !V2Size.isScalable() && + DecompGEP1.Offset.sge(V2Size.getValue()) && isBaseOfObject(DecompGEP2.Base)) return AliasResult::NoAlias; if (isa(V2)) { // Symmetric case to above. if (*DecompGEP2.InBounds && DecompGEP1.VarIndices.empty() && - V1Size.hasValue() && DecompGEP1.Offset.sle(-V1Size.getValue()) && + V1Size.hasValue() && !V1Size.isScalable() && + DecompGEP1.Offset.sle(-V1Size.getValue()) && isBaseOfObject(DecompGEP1.Base)) return AliasResult::NoAlias; } @@ -1090,6 +1094,10 @@ AliasResult BasicAAResult::aliasGEP( return BaseAlias; } + // Bail on analysing scalable LocationSize + if (V1Size.isScalable() || V2Size.isScalable()) + return AliasResult::MayAlias; + // If there is a constant difference between the pointers, but the difference // is less than the size of the associated memory object, then we know // that the objects are partially overlapping. If the difference is diff --git a/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp b/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp index 1c1a0873ac520..49eccde45f317 100644 --- a/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp +++ b/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp @@ -373,7 +373,10 @@ static bool canSkipClobberingStore(const StoreInst *SI, return false; if (MemoryLocation::get(SI).Size != MemLoc.Size) return false; - if (std::min(MemLocAlign, SI->getAlign()).value() < MemLoc.Size.getValue()) + if (MemLoc.Size.isScalable()) + return false; + if (std::min(MemLocAlign, SI->getAlign()).value() < + MemLoc.Size.getValue().getKnownMinValue()) return false; auto *LI = dyn_cast(SI->getValueOperand()); @@ -1099,7 +1102,8 @@ bool MemoryDependenceResults::getNonLocalPointerDepFromBB( // be conservative. ThrowOutEverything = CacheInfo->Size.isPrecise() != Loc.Size.isPrecise() || - CacheInfo->Size.getValue() < Loc.Size.getValue(); + !TypeSize::isKnownGE(CacheInfo->Size.getValue(), + Loc.Size.getValue()); } else { // For our purposes, unknown size > all others. ThrowOutEverything = !Loc.Size.hasValue(); diff --git a/llvm/lib/CodeGen/StackProtector.cpp b/llvm/lib/CodeGen/StackProtector.cpp index 387b653f88153..3ba85bc125a01 100644 --- a/llvm/lib/CodeGen/StackProtector.cpp +++ b/llvm/lib/CodeGen/StackProtector.cpp @@ -178,8 +178,7 @@ static bool HasAddressTaken(const Instruction *AI, TypeSize AllocSize, // the bounds of the allocated object. std::optional MemLoc = MemoryLocation::getOrNone(I); if (MemLoc && MemLoc->Size.hasValue() && - !TypeSize::isKnownGE(AllocSize, - TypeSize::getFixed(MemLoc->Size.getValue()))) + !TypeSize::isKnownGE(AllocSize, MemLoc->Size.getValue())) return true; switch (I->getOpcode()) { case Instruction::Store: diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp index e062acd807688..bbb0cfa0eb05f 100644 --- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp +++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp @@ -2546,7 +2546,8 @@ static int64_t getKnownNonNullAndDerefBytesForUse( } std::optional Loc = MemoryLocation::getOrNone(I); - if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() || I->isVolatile()) + if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() || + Loc->Size.isScalable() || I->isVolatile()) return 0; int64_t Offset; diff --git a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp index 7074dbc2c7b90..75490e984f985 100644 --- a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp +++ b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp @@ -205,7 +205,7 @@ static bool isShortenableAtTheBeginning(Instruction *I) { return isa(I); } -static std::optional getPointerSize(const Value *V, +static std::optional getPointerSize(const Value *V, const DataLayout &DL, const TargetLibraryInfo &TLI, const Function *F) { @@ -214,7 +214,7 @@ static std::optional getPointerSize(const Value *V, Opts.NullIsUnknownSize = NullPointerIsDefined(F); if (getObjectSize(V, Size, DL, &TLI, Opts)) - return Size; + return TypeSize::getFixed(Size); return std::nullopt; } @@ -952,7 +952,7 @@ struct DSEState { // case the size/offset of the dead store does not matter. if (DeadUndObj == KillingUndObj && KillingLocSize.isPrecise() && isIdentifiedObject(KillingUndObj)) { - std::optional KillingUndObjSize = + std::optional KillingUndObjSize = getPointerSize(KillingUndObj, DL, TLI, &F); if (KillingUndObjSize && *KillingUndObjSize == KillingLocSize.getValue()) return OW_Complete; @@ -977,9 +977,15 @@ struct DSEState { return isMaskedStoreOverwrite(KillingI, DeadI, BatchAA); } - const uint64_t KillingSize = KillingLocSize.getValue(); - const uint64_t DeadSize = DeadLoc.Size.getValue(); + const TypeSize KillingSize = KillingLocSize.getValue(); + const TypeSize DeadSize = DeadLoc.Size.getValue(); + // Bail on doing Size comparison which depends on AA for now + // TODO: Remove AnyScalable once Alias Analysis deal with scalable vectors + const bool AnyScalable = + DeadSize.isScalable() || KillingLocSize.isScalable(); + if (AnyScalable) + return OW_Unknown; // Query the alias information AliasResult AAR = BatchAA.alias(KillingLoc, DeadLoc); diff --git a/llvm/test/Analysis/AliasSet/memloc-vscale.ll b/llvm/test/Analysis/AliasSet/memloc-vscale.ll new file mode 100644 index 0000000000000..8a83645ddaf9a --- /dev/null +++ b/llvm/test/Analysis/AliasSet/memloc-vscale.ll @@ -0,0 +1,52 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -S < %s -passes=print-alias-sets 2>&1 | FileCheck %s + +; CHECK-LABEL: Alias sets for function 'sn' +; CHECK: AliasSet[{{.*}}, 1] must alias, Mod Pointers: (ptr %p, unknown after) +define void @sn(ptr %p) {; + store zeroinitializer, ptr %p, align 2 + store i64 0, ptr %p, align 2 + ret void +} + +; CHECK-LABEL: Alias sets for function 'ns' +; CHECK: AliasSet[{{.*}}, 1] must alias, Mod Pointers: (ptr %p, unknown after) +define void @ns(ptr %p) { + store i64 0, ptr %p, align 2 + store zeroinitializer, ptr %p, align 2 + ret void +} + +; CHECK-LABEL: Alias sets for function 'ss': +; CHECK: AliasSet[{{.*}}, 1] must alias, Mod Pointers: (ptr %p, LocationSize::precise(vscale x 16)) +define void @ss(ptr %p) { + store zeroinitializer, ptr %p, align 2 + store zeroinitializer, ptr %p, align 2 + ret void +} + +; CHECK-LABEL: Alias sets for function 'ss2': +; CHECK: AliasSet[{{.*}}, 1] must alias, Mod Pointers: (ptr %p, unknown after) +define void @ss2(ptr %p) { + store zeroinitializer, ptr %p, align 2 + store zeroinitializer, ptr %p, align 2 + store zeroinitializer, ptr %p, align 2 + ret void +} +; CHECK-LABEL: Alias sets for function 'son': +; CHECK: AliasSet[{{.*}}, 2] may alias, Mod Pointers: (ptr %g, LocationSize::precise(vscale x 16)), (ptr %p, LocationSize::precise(8)) +define void @son(ptr %p) { + %g = getelementptr i8, ptr %p, i64 8 + store zeroinitializer, ptr %g, align 2 + store i64 0, ptr %p, align 2 + ret void +} + +; CHECK-LABEL: Alias sets for function 'sno': +; CHECK: AliasSet[{{.*}}, 2] may alias, Mod Pointers: (ptr %p, LocationSize::precise(vscale x 16)), (ptr %g, LocationSize::precise(8)) +define void @sno(ptr %p) { + %g = getelementptr i8, ptr %p, i64 8 + store zeroinitializer, ptr %p, align 2 + store i64 0, ptr %g, align 2 + ret void +} diff --git a/llvm/test/Transforms/GVN/scalable-memloc.ll b/llvm/test/Transforms/GVN/scalable-memloc.ll new file mode 100644 index 0000000000000..23b4c19f280ca --- /dev/null +++ b/llvm/test/Transforms/GVN/scalable-memloc.ll @@ -0,0 +1,29 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -S < %s -passes=gvn | FileCheck %s + +define void @test(i1 %cmp19, ptr %p) { +; CHECK-LABEL: @test( +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[CMP19:%.*]], label [[WHILE_BODY_LR_PH:%.*]], label [[FOR_COND_PREHEADER:%.*]] +; CHECK: while.body.lr.ph: +; CHECK-NEXT: [[DOTPRE1:%.*]] = load , ptr [[P:%.*]], align 16 +; CHECK-NEXT: [[TMP0:%.*]] = extractelement [[DOTPRE1]], i64 0 +; CHECK-NEXT: ret void +; CHECK: for.cond.preheader: +; CHECK-NEXT: [[DOTPRE:%.*]] = load double, ptr [[P]], align 8 +; CHECK-NEXT: [[ADD:%.*]] = fadd double [[DOTPRE]], 0.000000e+00 +; CHECK-NEXT: ret void +; +entry: + br i1 %cmp19, label %while.body.lr.ph, label %for.cond.preheader + +while.body.lr.ph: ; preds = %entry + %.pre1 = load , ptr %p, align 16 + %0 = extractelement %.pre1, i64 0 + ret void + +for.cond.preheader: ; preds = %entry + %.pre = load double, ptr %p, align 8 + %add = fadd double %.pre, 0.000000e+00 + ret void +}