Unverified Commit 5f2854f1 authored by Florian Hahn's avatar Florian Hahn
Browse files

[LV] Always create VPWidenCanonicalIVRecipe, optimize away later.

This patch updates createBlockInMask to always generate
VPWidenCanonicalIVRecipe and adds a transform to optimize it away later,
if it is not needed.

This is a step towards breaking up VPWidenIntOrFpInductionRecipe and
explicitly distinguishing between vector phis and scalarizing.

Split off from D116123.

Reviewed By: Ayal

Differential Revision: https://reviews.llvm.org/D117140
parent 00d68c38
......@@ -8418,15 +8418,8 @@ VPValue *VPRecipeBuilder::createBlockInMask(BasicBlock *BB, VPlanPtr &Plan) {
assert(CM.foldTailByMasking() && "must fold the tail");
VPBasicBlock *HeaderVPBB = Plan->getEntry()->getEntryBasicBlock();
auto NewInsertionPoint = HeaderVPBB->getFirstNonPhi();
VPValue *IV = nullptr;
if (Legal->getPrimaryInduction())
IV = Plan->getOrAddVPValue(Legal->getPrimaryInduction());
else {
auto *IVRecipe = new VPWidenCanonicalIVRecipe(Plan->getCanonicalIV());
HeaderVPBB->insert(IVRecipe, NewInsertionPoint);
IV = IVRecipe;
}
auto *IV = new VPWidenCanonicalIVRecipe(Plan->getCanonicalIV());
HeaderVPBB->insert(IV, HeaderVPBB->getFirstNonPhi());
VPBuilder::InsertPointGuard Guard(Builder);
Builder.setInsertPoint(HeaderVPBB, NewInsertionPoint);
......@@ -9201,6 +9194,7 @@ VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes(
}
}
VPlanTransforms::removeRedundantCanonicalIVs(*Plan);
VPlanTransforms::removeRedundantInductionCasts(*Plan);
// Now that sink-after is done, move induction recipes for optimized truncates
......
......@@ -1076,6 +1076,12 @@ public:
/// Returns true if the induction is canonical, i.e. starting at 0 and
/// incremented by UF * VF (= the original IV is incremented by 1).
bool isCanonical() const;
/// Returns the scalar type of the induction.
const Type *getScalarType() const {
const TruncInst *TruncI = getTruncInst();
return TruncI ? TruncI->getType() : IV->getType();
}
};
/// A pure virtual base class for all recipes modeling header phis, including
......@@ -1675,6 +1681,11 @@ public:
void print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const override;
#endif
/// Returns the scalar type of the induction.
const Type *getScalarType() const {
return getOperand(0)->getLiveInIRValue()->getType();
}
};
/// A Recipe for widening the canonical induction variable of the vector loop.
......@@ -1691,6 +1702,16 @@ public:
return D->getVPDefID() == VPRecipeBase::VPWidenCanonicalIVSC;
}
/// Extra classof implementations to allow directly casting from VPUser ->
/// VPWidenCanonicalIVRecipe.
static inline bool classof(const VPUser *U) {
auto *R = dyn_cast<VPRecipeBase>(U);
return R && R->getVPDefID() == VPRecipeBase::VPWidenCanonicalIVSC;
}
static inline bool classof(const VPRecipeBase *R) {
return R->getVPDefID() == VPRecipeBase::VPWidenCanonicalIVSC;
}
/// Generate a canonical vector induction variable of the vector loop, with
/// start = {<Part*VF, Part*VF+1, ..., Part*VF+VF-1> for 0 <= Part < UF}, and
/// step = <VF*UF, VF*UF, ..., VF*UF>.
......@@ -1701,6 +1722,12 @@ public:
void print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const override;
#endif
/// Returns the scalar type of the induction.
const Type *getScalarType() const {
return cast<VPCanonicalIVPHIRecipe>(getOperand(0)->getDef())
->getScalarType();
}
};
/// VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph. It
......
......@@ -324,3 +324,30 @@ void VPlanTransforms::removeRedundantInductionCasts(VPlan &Plan) {
E.first->eraseFromParent();
}
}
void VPlanTransforms::removeRedundantCanonicalIVs(VPlan &Plan) {
VPCanonicalIVPHIRecipe *CanonicalIV = Plan.getCanonicalIV();
VPWidenCanonicalIVRecipe *WidenNewIV = nullptr;
for (VPUser *U : CanonicalIV->users()) {
WidenNewIV = dyn_cast<VPWidenCanonicalIVRecipe>(U);
if (WidenNewIV)
break;
}
if (!WidenNewIV)
return;
VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
auto *WidenOriginalIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
// If the induction recipe is canonical and the types match, use it
// directly.
if (WidenOriginalIV && WidenOriginalIV->isCanonical() &&
WidenOriginalIV->getScalarType() == WidenNewIV->getScalarType()) {
WidenNewIV->replaceAllUsesWith(WidenOriginalIV);
WidenNewIV->eraseFromParent();
return;
}
}
}
......@@ -45,6 +45,10 @@ struct VPlanTransforms {
/// in the vectorized loop. There is no need to vectorize the cast - the same
/// value can be used for both the phi and casts in the vector loop.
static void removeRedundantInductionCasts(VPlan &Plan);
/// Try to replace VPWidenCanonicalIVRecipes with a widened canonical IV
/// recipe, if it exists.
static void removeRedundantCanonicalIVs(VPlan &Plan);
};
} // namespace llvm
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment