Skip to content

Commit 7aa511d

Browse files
fhahnfhahn
committed
[VPlan] Cast VPIRMetadata.
1 parent 6360bbb commit 7aa511d

File tree

2 files changed

+85
-13
lines changed

2 files changed

+85
-13
lines changed

‎llvm/lib/Transforms/Vectorize/VPlan.h‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3871,6 +3871,77 @@ template <>
38713871
struct CastInfo<VPPhiAccessors, const VPRecipeBase *>
38723872
: CastInfoVPPhiAccessors<const VPRecipeBase *> {};
38733873

3874+
/// Casting from VPRecipeBase -> VPIRMetadata is supported for all recipe types
3875+
/// implementing VPIRMetadata. Used by isa<> & co.
3876+
namespace detail {
3877+
/// Returns const VPIRMetadata* if input is const, and VPIRMetadata* otherwise.
3878+
template <typename RecipeBasePtrTy>
3879+
static inline auto castToVPIRMetadata(RecipeBasePtrTy R) -> std::conditional_t<
3880+
std::is_const_v<std::remove_pointer_t<RecipeBasePtrTy>>,
3881+
const VPIRMetadata *, VPIRMetadata *> {
3882+
switch (R->getVPDefID()) {
3883+
case VPDef::VPInstructionSC:
3884+
return cast<VPInstruction>(R);
3885+
case VPDef::VPWidenSC:
3886+
return cast<VPWidenRecipe>(R);
3887+
case VPDef::VPWidenCastSC:
3888+
return cast<VPWidenCastRecipe>(R);
3889+
case VPDef::VPWidenIntrinsicSC:
3890+
return cast<VPWidenIntrinsicRecipe>(R);
3891+
case VPDef::VPWidenCallSC:
3892+
return cast<VPWidenCallRecipe>(R);
3893+
case VPDef::VPWidenSelectSC:
3894+
return cast<VPWidenSelectRecipe>(R);
3895+
case VPDef::VPReplicateSC:
3896+
return cast<VPReplicateRecipe>(R);
3897+
case VPDef::VPInterleaveSC:
3898+
return cast<VPInterleaveRecipe>(R);
3899+
case VPDef::VPInterleaveEVLSC:
3900+
return cast<VPInterleaveEVLRecipe>(R);
3901+
case VPDef::VPWidenLoadSC:
3902+
return cast<VPWidenLoadRecipe>(R);
3903+
case VPDef::VPWidenLoadEVLSC:
3904+
return cast<VPWidenLoadEVLRecipe>(R);
3905+
case VPDef::VPWidenStoreSC:
3906+
return cast<VPWidenStoreRecipe>(R);
3907+
case VPDef::VPWidenStoreEVLSC:
3908+
return cast<VPWidenStoreEVLRecipe>(R);
3909+
default:
3910+
llvm_unreachable("invalid recipe for VPIRMetadata cast");
3911+
}
3912+
}
3913+
} // namespace detail
3914+
3915+
/// Support casting from VPRecipeBase -> VPIRMetadata, by down-casting to the
3916+
/// recipe types implementing VPIRMetadata. Used by cast<>, dyn_cast<> & co.
3917+
template <typename SrcTy>
3918+
struct CastInfoVPIRMetadata : public CastIsPossible<VPIRMetadata, SrcTy> {
3919+
static inline bool isPossible(SrcTy R) {
3920+
return isa<VPInstruction, VPWidenRecipe, VPWidenCastRecipe,
3921+
VPWidenIntrinsicRecipe, VPWidenCallRecipe, VPWidenSelectRecipe,
3922+
VPReplicateRecipe, VPInterleaveRecipe, VPInterleaveEVLRecipe,
3923+
VPWidenLoadRecipe, VPWidenLoadEVLRecipe, VPWidenStoreRecipe,
3924+
VPWidenStoreEVLRecipe>(R);
3925+
}
3926+
3927+
using Self = CastInfo<VPIRMetadata, SrcTy>;
3928+
using RetTy = decltype(detail::castToVPIRMetadata(std::declval<SrcTy>()));
3929+
3930+
static inline RetTy doCast(SrcTy R) { return detail::castToVPIRMetadata(R); }
3931+
3932+
static inline RetTy doCastIfPossible(SrcTy R) {
3933+
if (!Self::isPossible(R))
3934+
return nullptr;
3935+
return doCast(R);
3936+
}
3937+
};
3938+
template <>
3939+
struct CastInfo<VPIRMetadata, VPRecipeBase *>
3940+
: CastInfoVPIRMetadata<VPRecipeBase *> {};
3941+
template <>
3942+
struct CastInfo<VPIRMetadata, const VPRecipeBase *>
3943+
: CastInfoVPIRMetadata<const VPRecipeBase *> {};
3944+
38743945
/// VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph. It
38753946
/// holds a sequence of zero or more VPRecipe's each representing a sequence of
38763947
/// output IR instructions. All PHI-like recipes must come before any non-PHI recipes.

‎llvm/unittests/Transforms/Vectorize/VPlanTest.cpp‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -996,7 +996,7 @@ TEST_F(VPRecipeTest, CastVPInstructionToVPUser) {
996996
VPValue *Op2 = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2));
997997
VPInstruction Recipe(Instruction::Add, {Op1, Op2});
998998

999-
checkVPRecipeCastImpl<VPInstruction, VPUser>(&Recipe);
999+
checkVPRecipeCastImpl<VPInstruction, VPUser, VPIRMetadata>(&Recipe);
10001000
}
10011001

10021002
TEST_F(VPRecipeTest, CastVPWidenRecipeToVPUser) {
@@ -1011,7 +1011,7 @@ TEST_F(VPRecipeTest, CastVPWidenRecipeToVPUser) {
10111011
Args.push_back(Op2);
10121012
VPWidenRecipe WidenR(*AI, make_range(Args.begin(), Args.end()));
10131013

1014-
checkVPRecipeCastImpl<VPWidenRecipe, VPUser>(&WidenR);
1014+
checkVPRecipeCastImpl<VPWidenRecipe, VPUser, VPIRMetadata>(&WidenR);
10151015
delete AI;
10161016
}
10171017

@@ -1030,7 +1030,7 @@ TEST_F(VPRecipeTest, CastVPWidenCallRecipeToVPUserAndVPDef) {
10301030
Args.push_back(CalledFn);
10311031
VPWidenCallRecipe Recipe(Call, Fn, Args);
10321032

1033-
checkVPRecipeCastImpl<VPWidenCallRecipe, VPUser>(&Recipe);
1033+
checkVPRecipeCastImpl<VPWidenCallRecipe, VPUser, VPIRMetadata>(&Recipe);
10341034

10351035
VPValue *VPV = &Recipe;
10361036
EXPECT_TRUE(VPV->getDefiningRecipe());
@@ -1056,7 +1056,8 @@ TEST_F(VPRecipeTest, CastVPWidenSelectRecipeToVPUserAndVPDef) {
10561056
VPWidenSelectRecipe WidenSelectR(*SelectI,
10571057
make_range(Args.begin(), Args.end()));
10581058

1059-
checkVPRecipeCastImpl<VPWidenSelectRecipe, VPUser>(&WidenSelectR);
1059+
checkVPRecipeCastImpl<VPWidenSelectRecipe, VPUser, VPIRMetadata>(
1060+
&WidenSelectR);
10601061

10611062
VPValue *VPV = &WidenSelectR;
10621063
EXPECT_EQ(&WidenSelectR, VPV->getDefiningRecipe());
@@ -1094,7 +1095,7 @@ TEST_F(VPRecipeTest, CastVPWidenCastRecipeToVPUser) {
10941095
VPValue *Op1 = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
10951096
VPWidenCastRecipe Recipe(Instruction::ZExt, Op1, Int64, *Cast);
10961097

1097-
checkVPRecipeCastImpl<VPWidenCastRecipe, VPUser>(&Recipe);
1098+
checkVPRecipeCastImpl<VPWidenCastRecipe, VPUser, VPIRMetadata>(&Recipe);
10981099
delete Cast;
10991100
}
11001101

@@ -1105,7 +1106,7 @@ TEST_F(VPRecipeTest, CastVPWidenIntrinsicRecipeToVPUser) {
11051106
VPValue *Op2 = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2));
11061107
VPWidenIntrinsicRecipe Recipe(Intrinsic::smax, {Op1, Op2}, Int32);
11071108

1108-
checkVPRecipeCastImpl<VPWidenIntrinsicRecipe, VPUser>(&Recipe);
1109+
checkVPRecipeCastImpl<VPWidenIntrinsicRecipe, VPUser, VPIRMetadata>(&Recipe);
11091110
}
11101111

11111112
TEST_F(VPRecipeTest, CastVPBlendRecipeToVPUser) {
@@ -1135,7 +1136,7 @@ TEST_F(VPRecipeTest, CastVPInterleaveRecipeToVPUser) {
11351136
InterleaveGroup<Instruction> IG(4, false, Align(4));
11361137
VPInterleaveRecipe Recipe(&IG, Addr, {}, Mask, false, {}, DebugLoc());
11371138

1138-
checkVPRecipeCastImpl<VPInterleaveRecipe, VPUser>(&Recipe);
1139+
checkVPRecipeCastImpl<VPInterleaveRecipe, VPUser, VPIRMetadata>(&Recipe);
11391140
}
11401141

11411142
TEST_F(VPRecipeTest, CastVPReplicateRecipeToVPUser) {
@@ -1151,7 +1152,7 @@ TEST_F(VPRecipeTest, CastVPReplicateRecipeToVPUser) {
11511152
auto *Call = CallInst::Create(FTy, PoisonValue::get(FTy));
11521153
VPReplicateRecipe Recipe(Call, make_range(Args.begin(), Args.end()), true);
11531154

1154-
checkVPRecipeCastImpl<VPReplicateRecipe, VPUser>(&Recipe);
1155+
checkVPRecipeCastImpl<VPReplicateRecipe, VPUser, VPIRMetadata>(&Recipe);
11551156

11561157
delete Call;
11571158
}
@@ -1175,7 +1176,7 @@ TEST_F(VPRecipeTest, CastVPWidenMemoryRecipeToVPUserAndVPDef) {
11751176
VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2));
11761177
VPWidenLoadRecipe Recipe(*Load, Addr, Mask, true, false, {}, {});
11771178

1178-
checkVPRecipeCastImpl<VPWidenLoadRecipe, VPUser>(&Recipe);
1179+
checkVPRecipeCastImpl<VPWidenLoadRecipe, VPUser, VPIRMetadata>(&Recipe);
11791180

11801181
VPValue *VPV = Recipe.getVPSingleValue();
11811182
EXPECT_TRUE(isa<VPRecipeBase>(VPV->getDefiningRecipe()));
@@ -1194,7 +1195,7 @@ TEST_F(VPRecipeTest, CastVPInterleaveEVLRecipeToVPUser) {
11941195
VPInterleaveRecipe BaseRecipe(&IG, Addr, {}, Mask, false, {}, DebugLoc());
11951196
VPInterleaveEVLRecipe Recipe(BaseRecipe, *EVL, Mask);
11961197

1197-
checkVPRecipeCastImpl<VPInterleaveEVLRecipe, VPUser>(&Recipe);
1198+
checkVPRecipeCastImpl<VPInterleaveEVLRecipe, VPUser, VPIRMetadata>(&Recipe);
11981199
}
11991200

12001201
TEST_F(VPRecipeTest, CastVPWidenLoadEVLRecipeToVPUser) {
@@ -1209,7 +1210,7 @@ TEST_F(VPRecipeTest, CastVPWidenLoadEVLRecipeToVPUser) {
12091210
VPWidenLoadRecipe BaseLoad(*Load, Addr, Mask, true, false, {}, {});
12101211
VPWidenLoadEVLRecipe Recipe(BaseLoad, Addr, *EVL, Mask);
12111212

1212-
checkVPRecipeCastImpl<VPWidenLoadEVLRecipe, VPUser>(&Recipe);
1213+
checkVPRecipeCastImpl<VPWidenLoadEVLRecipe, VPUser, VPIRMetadata>(&Recipe);
12131214

12141215
delete Load;
12151216
}
@@ -1225,7 +1226,7 @@ TEST_F(VPRecipeTest, CastVPWidenStoreRecipeToVPUser) {
12251226
VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2));
12261227
VPWidenStoreRecipe Recipe(*Store, Addr, StoredVal, Mask, true, false, {}, {});
12271228

1228-
checkVPRecipeCastImpl<VPWidenStoreRecipe, VPUser>(&Recipe);
1229+
checkVPRecipeCastImpl<VPWidenStoreRecipe, VPUser, VPIRMetadata>(&Recipe);
12291230

12301231
delete Store;
12311232
}
@@ -1244,7 +1245,7 @@ TEST_F(VPRecipeTest, CastVPWidenStoreEVLRecipeToVPUser) {
12441245
{});
12451246
VPWidenStoreEVLRecipe Recipe(BaseStore, Addr, *EVL, Mask);
12461247

1247-
checkVPRecipeCastImpl<VPWidenStoreEVLRecipe, VPUser>(&Recipe);
1248+
checkVPRecipeCastImpl<VPWidenStoreEVLRecipe, VPUser, VPIRMetadata>(&Recipe);
12481249

12491250
delete Store;
12501251
}

0 commit comments

Comments
 (0)