Unverified Commit 992210aa authored by Florian Hahn's avatar Florian Hahn Committed by GitHub
Browse files

[VPlan] Run removeDeadRecipes early. (#190191) 

The initial  simplifyRecipes run can leave dead recipes, which
removeDeadRecipes can clean up, similar for dead instructions in the
input.

PR: https://github.com/llvm/llvm-project/pull/190191
parent 696406fb

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

+6 −3
Original line number Diff line number Diff line
@@ -6853,6 +6853,7 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF, 
    return;



  RUN_VPLAN_PASS(VPlanTransforms::simplifyRecipes, *VPlan0);

  RUN_VPLAN_PASS(VPlanTransforms::removeDeadRecipes, *VPlan0);

  // If we're vectorizing a loop with an uncountable exit, make sure that the

  // recipes are safe to handle.

  // TODO: Remove this once we can properly check the VPlan itself for both
@@ -7107,6 +7108,10 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VPlanPtr Plan, 
                   Range);

  }



  // Ensure scalar VF plans only contain VF=1, as required by hasScalarVFOnly.

  if (Range.Start.isScalar())

    Range.End = Range.Start * 2;



  for (ElementCount VF : Range)

    Plan->addVF(VF);

  Plan->setName("Initial VPlan");
@@ -7192,9 +7197,7 @@ void LoopVectorizationPlanner::addReductionResultComputation( 
  for (VPRecipeBase &R :

       Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {

    VPReductionPHIRecipe *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);

    // TODO: Remove check for constant incoming value once removeDeadRecipes is

    // used on VPlan0.

    if (!PhiR || isa<VPIRValue>(PhiR->getOperand(1)))

    if (!PhiR)

      continue;



    RecurKind RecurrenceKind = PhiR->getRecurrenceKind();

llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

+10 −1
Original line number Diff line number Diff line
@@ -650,6 +650,7 @@ static void removeRedundantInductionCasts(VPlan &Plan) { 
      }

      FindMyCast = FoundUserCast;

    }

    if (FindMyCast)

      FindMyCast->replaceAllUsesWith(IV);

  }

}
@@ -3523,6 +3524,14 @@ void VPlanTransforms::createInterleaveGroups( 
  // single VPInterleaveRecipe at its insertion point.

  VPDominatorTree VPDT(Plan);

  for (const auto *IG : InterleaveGroups) {

    // Skip interleave groups where members don't have recipes. This can happen

    // when removeDeadRecipes removes recipes that are part of interleave groups

    // but have no users.

    if (llvm::any_of(IG->members(), [&IRMemberToRecipe](Instruction *Member) {

          return !IRMemberToRecipe.contains(Member);

        }))

      continue;



    auto *Start = IRMemberToRecipe.lookup(IG->getMember(0));

    VPIRMetadata InterleaveMD(*Start);

    SmallVector<VPValue *, 4> StoredValues;

llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll

+1 −1
Original line number Diff line number Diff line
@@ -324,7 +324,7 @@ define void @latch_branch_cost(ptr %dst) { 
; PRED-NEXT:    br label %[[PRED_STORE_CONTINUE6]]

; PRED:       [[PRED_STORE_CONTINUE6]]:

; PRED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4

; PRED-NEXT:    [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 4)

; PRED-NEXT:    [[VEC_IND_NEXT]] = add nuw <4 x i8> [[VEC_IND]], splat (i8 4)

; PRED-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100

; PRED-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]

; PRED:       [[MIDDLE_BLOCK]]:

llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll

+1 −1
Original line number Diff line number Diff line
@@ -641,7 +641,7 @@ define void @forced_scalar_instr(ptr %gep.dst) { 
; COMMON-NEXT:    br label %[[PRED_STORE_CONTINUE6]]

; COMMON:       [[PRED_STORE_CONTINUE6]]:

; COMMON-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4

; COMMON-NEXT:    [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 4)

; COMMON-NEXT:    [[VEC_IND_NEXT]] = add nuw <4 x i8> [[VEC_IND]], splat (i8 4)

; COMMON-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], 8

; COMMON-NEXT:    br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]

; COMMON:       [[MIDDLE_BLOCK]]:

llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-epilogue.ll

+1 −1
Original line number Diff line number Diff line
@@ -181,8 +181,8 @@ define i32 @dotp_predicated(i64 %N, ptr %a, ptr %b) { 
; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]

; CHECK:       vector.body:

; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE62:%.*]] ]

; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <16 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_LOAD_CONTINUE62]] ]

; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[PRED_LOAD_CONTINUE62]] ]

; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <16 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_LOAD_CONTINUE62]] ]

; CHECK-NEXT:    [[TMP16:%.*]] = icmp ule <16 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]

; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <16 x i1> [[TMP16]], i64 0

; CHECK-NEXT:    br i1 [[TMP17]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]