Unverified Commit 38233414 authored by Florian Hahn's avatar Florian Hahn Committed by GitHub
Browse files

Reapply "[VPlan] Run removeDeadRecipes early." (#195325) (#195445) 

This reverts commit 2a9699cc.

Recommit contains a small fix for skipping dead recipes when finding
induction casts.

Original message:
The initial simplifyRecipes run can leave dead recipes, which
removeDeadRecipes can clean up, similar for dead instructions in the
input.

PR: https://github.com/llvm/llvm-project/pull/190191
parent 82f9618d

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

+6 −3
Original line number Diff line number Diff line
@@ -6853,6 +6853,7 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF, 
    return;



  RUN_VPLAN_PASS(VPlanTransforms::simplifyRecipes, *VPlan0);

  RUN_VPLAN_PASS(VPlanTransforms::removeDeadRecipes, *VPlan0);

  // If we're vectorizing a loop with an uncountable exit, make sure that the

  // recipes are safe to handle.

  // TODO: Remove this once we can properly check the VPlan itself for both
@@ -7107,6 +7108,10 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VPlanPtr Plan, 
                   Range);

  }



  // Ensure scalar VF plans only contain VF=1, as required by hasScalarVFOnly.

  if (Range.Start.isScalar())

    Range.End = Range.Start * 2;



  for (ElementCount VF : Range)

    Plan->addVF(VF);

  Plan->setName("Initial VPlan");
@@ -7192,9 +7197,7 @@ void LoopVectorizationPlanner::addReductionResultComputation( 
  for (VPRecipeBase &R :

       Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {

    VPReductionPHIRecipe *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);

    // TODO: Remove check for constant incoming value once removeDeadRecipes is

    // used on VPlan0.

    if (!PhiR || isa<VPIRValue>(PhiR->getOperand(1)))

    if (!PhiR)

      continue;



    RecurKind RecurrenceKind = PhiR->getRecurrenceKind();

llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

+13 −1
Original line number Diff line number Diff line
@@ -648,8 +648,12 @@ static void removeRedundantInductionCasts(VPlan &Plan) { 
          break;

        }

      }

      // A cast recipe in the chain may have been removed by earlier DCE.

      if (!FoundUserCast)

        break;

      FindMyCast = FoundUserCast;

    }

    if (FindMyCast != IV)

      FindMyCast->replaceAllUsesWith(IV);

  }

}
@@ -3521,6 +3525,14 @@ void VPlanTransforms::createInterleaveGroups( 
  // single VPInterleaveRecipe at its insertion point.

  VPDominatorTree VPDT(Plan);

  for (const auto *IG : InterleaveGroups) {

    // Skip interleave groups where members don't have recipes. This can happen

    // when removeDeadRecipes removes recipes that are part of interleave groups

    // but have no users.

    if (llvm::any_of(IG->members(), [&IRMemberToRecipe](Instruction *Member) {

          return !IRMemberToRecipe.contains(Member);

        }))

      continue;



    auto *Start = IRMemberToRecipe.lookup(IG->getMember(0));

    VPIRMetadata InterleaveMD(*Start);

    SmallVector<VPValue *, 4> StoredValues;

llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll

+1 −1
Original line number Diff line number Diff line
@@ -324,7 +324,7 @@ define void @latch_branch_cost(ptr %dst) { 
; PRED-NEXT:    br label %[[PRED_STORE_CONTINUE6]]

; PRED:       [[PRED_STORE_CONTINUE6]]:

; PRED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4

; PRED-NEXT:    [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 4)

; PRED-NEXT:    [[VEC_IND_NEXT]] = add nuw <4 x i8> [[VEC_IND]], splat (i8 4)

; PRED-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100

; PRED-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]

; PRED:       [[MIDDLE_BLOCK]]:

llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll

+1 −1
Original line number Diff line number Diff line
@@ -641,7 +641,7 @@ define void @forced_scalar_instr(ptr %gep.dst) { 
; COMMON-NEXT:    br label %[[PRED_STORE_CONTINUE6]]

; COMMON:       [[PRED_STORE_CONTINUE6]]:

; COMMON-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4

; COMMON-NEXT:    [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 4)

; COMMON-NEXT:    [[VEC_IND_NEXT]] = add nuw <4 x i8> [[VEC_IND]], splat (i8 4)

; COMMON-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], 8

; COMMON-NEXT:    br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]

; COMMON:       [[MIDDLE_BLOCK]]:

llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-epilogue.ll

+1 −1
Original line number Diff line number Diff line
@@ -181,8 +181,8 @@ define i32 @dotp_predicated(i64 %N, ptr %a, ptr %b) { 
; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]

; CHECK:       vector.body:

; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE62:%.*]] ]

; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <16 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_LOAD_CONTINUE62]] ]

; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[PRED_LOAD_CONTINUE62]] ]

; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <16 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_LOAD_CONTINUE62]] ]

; CHECK-NEXT:    [[TMP16:%.*]] = icmp ule <16 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]

; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <16 x i1> [[TMP16]], i64 0

; CHECK-NEXT:    br i1 [[TMP17]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]