161#define LV_NAME "loop-vectorize"
162#define DEBUG_TYPE LV_NAME
168STATISTIC(LoopsVectorized,
"Number of loops vectorized");
169STATISTIC(LoopsAnalyzed,
"Number of loops analyzed for vectorization");
170STATISTIC(LoopsEpilogueVectorized,
"Number of epilogues vectorized");
171STATISTIC(LoopsEarlyExitVectorized,
"Number of early exit loops vectorized");
175 cl::desc(
"Enable vectorization of epilogue loops."));
179 cl::desc(
"When epilogue vectorization is enabled, and a value greater than "
180 "1 is specified, forces the given VF for all applicable epilogue "
184 "epilogue-vectorization-minimum-VF",
cl::Hidden,
185 cl::desc(
"Only loops with vectorization factor equal to or larger than "
186 "the specified value are considered for epilogue vectorization."));
192 cl::desc(
"Loops with a constant trip count that is smaller than this "
193 "value are vectorized only if no scalar iteration overheads "
198 cl::desc(
"The maximum allowed number of runtime memory checks"));
214 "prefer-predicate-over-epilogue",
217 cl::desc(
"Tail-folding and predication preferences over creating a scalar "
221 "Don't tail-predicate loops, create scalar epilogue"),
223 "predicate-else-scalar-epilogue",
224 "prefer tail-folding, create scalar epilogue if tail "
227 "predicate-dont-vectorize",
228 "prefers tail-folding, don't attempt vectorization if "
229 "tail-folding fails.")));
232 "force-tail-folding-style",
cl::desc(
"Force the tail folding style"),
238 "Create lane mask for data only, using active.lane.mask intrinsic"),
240 "data-without-lane-mask",
241 "Create lane mask with compare/stepvector"),
243 "Create lane mask using active.lane.mask intrinsic, and use "
244 "it for both data and control flow"),
246 "data-and-control-without-rt-check",
247 "Similar to data-and-control, but remove the runtime check"),
249 "Use predicated EVL instructions for tail folding. If EVL "
250 "is unsupported, fallback to data-without-lane-mask.")));
254 cl::desc(
"Maximize bandwidth when selecting vectorization factor which "
255 "will be determined by the smallest type in loop."));
259 cl::desc(
"Enable vectorization on interleaved memory accesses in a loop"));
265 cl::desc(
"Enable vectorization on masked interleaved memory accesses in a loop"));
269 cl::desc(
"A flag that overrides the target's number of scalar registers."));
273 cl::desc(
"A flag that overrides the target's number of vector registers."));
277 cl::desc(
"A flag that overrides the target's max interleave factor for "
282 cl::desc(
"A flag that overrides the target's max interleave factor for "
283 "vectorized loops."));
287 cl::desc(
"A flag that overrides the target's expected cost for "
288 "an instruction to a single constant value. Mostly "
289 "useful for getting consistent testing."));
294 "Pretend that scalable vectors are supported, even if the target does "
295 "not support them. This flag should only be used for testing."));
300 "The cost of a loop that is considered 'small' by the interleaver."));
304 cl::desc(
"Enable the use of the block frequency analysis to access PGO "
305 "heuristics minimizing code growth in cold regions and being more "
306 "aggressive in hot regions."));
312 "Enable runtime interleaving until load/store ports are saturated"));
317 cl::desc(
"Max number of stores to be predicated behind an if."));
321 cl::desc(
"Count the induction variable only once when interleaving"));
325 cl::desc(
"Enable if predication of stores during vectorization."));
329 cl::desc(
"The maximum interleave count to use when interleaving a scalar "
330 "reduction in a nested loop."));
335 cl::desc(
"Prefer in-loop vector reductions, "
336 "overriding the targets preference."));
340 cl::desc(
"Enable the vectorisation of loops with in-order (strict) "
346 "Prefer predicating a reduction operation over an after loop select."));
350 cl::desc(
"Enable VPlan-native vectorization path with "
351 "support for outer loop vectorization."));
355#ifdef EXPENSIVE_CHECKS
361 cl::desc(
"Verfiy VPlans after VPlan transforms."));
370 "Build VPlan for every supported loop nest in the function and bail "
371 "out right after the build (stress test the VPlan H-CFG construction "
372 "in the VPlan-native vectorization path)."));
376 cl::desc(
"Enable loop interleaving in Loop vectorization passes"));
379 cl::desc(
"Run the Loop vectorization passes"));
382 "force-widen-divrem-via-safe-divisor",
cl::Hidden,
384 "Override cost based safe divisor widening for div/rem instructions"));
387 "vectorizer-maximize-bandwidth-for-vector-calls",
cl::init(
true),
389 cl::desc(
"Try wider VFs if they enable the use of vector variants"));
394 "Enable vectorization of early exit loops with uncountable exits."));
398 cl::desc(
"Discard VFs if their register pressure is too high."));
411 return DL.getTypeAllocSizeInBits(Ty) !=
DL.getTypeSizeInBits(Ty);
446static std::optional<ElementCount>
448 bool CanUseConstantMax =
true) {
458 if (!CanUseConstantMax)
470class GeneratedRTChecks;
503 VF(VecWidth),
UF(UnrollFactor),
Builder(
PSE.getSE()->getContext()),
506 Plan.getVectorLoopRegion()->getSinglePredecessor())) {}
616 "A high UF for the epilogue loop is likely not beneficial.");
667 EPI.MainLoopVF,
EPI.MainLoopUF) {}
709 EPI.EpilogueVF,
EPI.EpilogueUF) {
719 assert(AdditionalBypassBlock &&
720 "Trying to access AdditionalBypassBlock but it has not been set");
721 return AdditionalBypassBlock;
742 if (
I->getDebugLoc() !=
Empty)
743 return I->getDebugLoc();
745 for (
Use &
Op :
I->operands()) {
747 if (OpInst->getDebugLoc() !=
Empty)
748 return OpInst->getDebugLoc();
751 return I->getDebugLoc();
760 dbgs() <<
"LV: " << Prefix << DebugMsg;
782 if (
I &&
I->getDebugLoc())
783 DL =
I->getDebugLoc();
795 assert(Ty->isIntegerTy() &&
"Expected an integer step");
803 return B.CreateElementCount(Ty, VFxStep);
808 return B.CreateElementCount(Ty, VF);
819 <<
"loop not vectorized: " << OREMsg);
842 "Vectorizing: ", TheLoop->
isInnermost() ?
"innermost loop" :
"outer loop",
848 <<
"vectorized " << LoopType <<
"loop (vectorization width: "
850 <<
", interleaved count: " <<
ore::NV(
"InterleaveCount", IC) <<
")";
906 initializeVScaleForTuning();
987 "Profitable to scalarize relevant only for VF > 1.");
990 "cost-model should not be used for outer loops (in VPlan-native path)");
992 auto Scalars = InstsToScalarize.find(VF);
993 assert(Scalars != InstsToScalarize.end() &&
994 "VF not yet analyzed for scalarization profitability");
995 return Scalars->second.contains(
I);
1002 "cost-model should not be used for outer loops (in VPlan-native path)");
1012 auto UniformsPerVF = Uniforms.find(VF);
1013 assert(UniformsPerVF != Uniforms.end() &&
1014 "VF not yet analyzed for uniformity");
1015 return UniformsPerVF->second.count(
I);
1022 "cost-model should not be used for outer loops (in VPlan-native path)");
1026 auto ScalarsPerVF = Scalars.find(VF);
1027 assert(ScalarsPerVF != Scalars.end() &&
1028 "Scalar values are not calculated for VF");
1029 return ScalarsPerVF->second.count(
I);
1035 return VF.
isVector() && MinBWs.contains(
I) &&
1057 WideningDecisions[{
I, VF}] = {W,
Cost};
1076 for (
unsigned Idx = 0; Idx < Grp->
getFactor(); ++Idx) {
1079 WideningDecisions[{
I, VF}] = {W, InsertPosCost};
1081 WideningDecisions[{
I, VF}] = {W, OtherMemberCost};
1093 "cost-model should not be used for outer loops (in VPlan-native path)");
1095 std::pair<Instruction *, ElementCount> InstOnVF(
I, VF);
1096 auto Itr = WideningDecisions.find(InstOnVF);
1097 if (Itr == WideningDecisions.end())
1099 return Itr->second.first;
1106 std::pair<Instruction *, ElementCount> InstOnVF(
I, VF);
1107 assert(WideningDecisions.contains(InstOnVF) &&
1108 "The cost is not calculated");
1109 return WideningDecisions[InstOnVF].second;
1122 std::optional<unsigned> MaskPos,
1125 CallWideningDecisions[{CI, VF}] = {Kind, Variant, IID, MaskPos,
Cost};
1131 auto I = CallWideningDecisions.find({CI, VF});
1132 if (
I == CallWideningDecisions.end())
1155 Value *
Op = Trunc->getOperand(0);
1156 if (
Op !=
Legal->getPrimaryInduction() &&
TTI.isTruncateFree(SrcTy, DestTy))
1160 return Legal->isInductionPhi(
Op);
1176 if (VF.
isScalar() || Uniforms.contains(VF))
1179 collectLoopUniforms(VF);
1181 collectLoopScalars(VF);
1189 return Legal->isConsecutivePtr(DataType,
Ptr) &&
1197 return Legal->isConsecutivePtr(DataType,
Ptr) &&
1212 return (
LI &&
TTI.isLegalMaskedGather(Ty,
Align)) ||
1219 return (
all_of(
Legal->getReductionVars(), [&](
auto &Reduction) ->
bool {
1220 const RecurrenceDescriptor &RdxDesc = Reduction.second;
1221 return TTI.isLegalToVectorizeReduction(RdxDesc, VF);
1232 return ScalarCost < SafeDivisorCost;
1256 std::pair<InstructionCost, InstructionCost>
1284 LLVM_DEBUG(
dbgs() <<
"LV: Loop does not require scalar epilogue\n");
1291 LLVM_DEBUG(
dbgs() <<
"LV: Loop requires scalar epilogue: not exiting "
1292 "from latch block\n");
1297 "interleaved group requires scalar epilogue\n");
1300 LLVM_DEBUG(
dbgs() <<
"LV: Loop does not require scalar epilogue\n");
1312 if (!ChosenTailFoldingStyle)
1314 return IVUpdateMayOverflow ? ChosenTailFoldingStyle->first
1315 : ChosenTailFoldingStyle->second;
1323 assert(!ChosenTailFoldingStyle &&
"Tail folding must not be selected yet.");
1324 if (!
Legal->canFoldTailByMasking()) {
1330 ChosenTailFoldingStyle = {
1331 TTI.getPreferredTailFoldingStyle(
true),
1332 TTI.getPreferredTailFoldingStyle(
false)};
1342 bool EVLIsLegal = UserIC <= 1 && IsScalableVF &&
1356 dbgs() <<
"LV: Preference for VP intrinsics indicated. Will "
1357 "not try to generate VP Intrinsics "
1359 ?
"since interleave count specified is greater than 1.\n"
1360 :
"due to non-interleaving reasons.\n"));
1394 return InLoopReductions.contains(Phi);
1405 TTI.preferPredicatedReductionSelect();
1420 WideningDecisions.clear();
1421 CallWideningDecisions.clear();
1440 const unsigned IC)
const;
1450 Type *VectorTy)
const;
1460 unsigned NumPredStores = 0;
1464 std::optional<unsigned> VScaleForTuning;
1469 void initializeVScaleForTuning() {
1474 auto Max = Attr.getVScaleRangeMax();
1475 if (Max && Min == Max) {
1476 VScaleForTuning = Max;
1481 VScaleForTuning =
TTI.getVScaleForTuning();
1489 FixedScalableVFPair computeFeasibleMaxVF(
unsigned MaxTripCount,
1490 ElementCount UserVF,
1491 bool FoldTailByMasking);
1495 ElementCount clampVFByMaxTripCount(ElementCount VF,
unsigned MaxTripCount,
1496 bool FoldTailByMasking)
const;
1501 ElementCount getMaximizedVFForTarget(
unsigned MaxTripCount,
1502 unsigned SmallestType,
1503 unsigned WidestType,
1504 ElementCount MaxSafeVF,
1505 bool FoldTailByMasking);
1509 bool isScalableVectorizationAllowed();
1513 ElementCount getMaxLegalScalableVF(
unsigned MaxSafeElements);
1519 InstructionCost getMemInstScalarizationCost(Instruction *
I, ElementCount VF);
1540 ElementCount VF)
const;
1544 bool useEmulatedMaskMemRefHack(Instruction *
I, ElementCount VF);
1549 MapVector<Instruction *, uint64_t> MinBWs;
1554 using ScalarCostsTy = MapVector<Instruction *, InstructionCost>;
1558 DenseMap<ElementCount, SmallPtrSet<BasicBlock *, 4>>
1559 PredicatedBBsAfterVectorization;
1572 std::optional<std::pair<TailFoldingStyle, TailFoldingStyle>>
1573 ChosenTailFoldingStyle;
1576 std::optional<bool> IsScalableVectorizationAllowed;
1582 std::optional<unsigned> MaxSafeElements;
1588 MapVector<ElementCount, ScalarCostsTy> InstsToScalarize;
1592 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Uniforms;
1596 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Scalars;
1600 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> ForcedScalars;
1603 SmallPtrSet<PHINode *, 4> InLoopReductions;
1608 DenseMap<Instruction *, Instruction *> InLoopReductionImmediateChains;
1616 ScalarCostsTy &ScalarCosts,
1628 void collectLoopUniforms(ElementCount VF);
1637 void collectLoopScalars(ElementCount VF);
1641 using DecisionList = DenseMap<std::pair<Instruction *, ElementCount>,
1642 std::pair<InstWidening, InstructionCost>>;
1644 DecisionList WideningDecisions;
1646 using CallDecisionList =
1649 CallDecisionList CallWideningDecisions;
1653 bool needsExtract(
Value *V, ElementCount VF)
const {
1673 ElementCount VF)
const {
1675 SmallPtrSet<const Value *, 4> UniqueOperands;
1679 !needsExtract(
Op, VF))
1751class GeneratedRTChecks {
1757 Value *SCEVCheckCond =
nullptr;
1764 Value *MemRuntimeCheckCond =
nullptr;
1773 bool CostTooHigh =
false;
1775 Loop *OuterLoop =
nullptr;
1786 : DT(DT), LI(LI),
TTI(
TTI), SCEVExp(*PSE.
getSE(),
DL,
"scev.check"),
1787 MemCheckExp(*PSE.
getSE(),
DL,
"scev.check"), PSE(PSE),
1795 void create(Loop *L,
const LoopAccessInfo &LAI,
1796 const SCEVPredicate &UnionPred, ElementCount VF,
unsigned IC) {
1816 nullptr,
"vector.scevcheck");
1823 SCEVExpanderCleaner SCEVCleaner(SCEVExp);
1824 SCEVCleaner.cleanup();
1829 if (RtPtrChecking.Need) {
1830 auto *Pred = SCEVCheckBlock ? SCEVCheckBlock : Preheader;
1831 MemCheckBlock =
SplitBlock(Pred, Pred->getTerminator(), DT, LI,
nullptr,
1834 auto DiffChecks = RtPtrChecking.getDiffChecks();
1836 Value *RuntimeVF =
nullptr;
1839 [VF, &RuntimeVF](IRBuilderBase &
B,
unsigned Bits) {
1841 RuntimeVF = getRuntimeVF(B, B.getIntNTy(Bits), VF);
1847 MemCheckBlock->
getTerminator(), L, RtPtrChecking.getChecks(),
1850 assert(MemRuntimeCheckCond &&
1851 "no RT checks generated although RtPtrChecking "
1852 "claimed checks are required");
1857 if (!MemCheckBlock && !SCEVCheckBlock)
1867 if (SCEVCheckBlock) {
1870 auto *UI =
new UnreachableInst(Preheader->
getContext(), SCEVCheckBlock);
1874 if (MemCheckBlock) {
1877 auto *UI =
new UnreachableInst(Preheader->
getContext(), MemCheckBlock);
1883 if (MemCheckBlock) {
1887 if (SCEVCheckBlock) {
1893 OuterLoop =
L->getParentLoop();
1897 if (SCEVCheckBlock || MemCheckBlock)
1909 for (Instruction &
I : *SCEVCheckBlock) {
1910 if (SCEVCheckBlock->getTerminator() == &
I)
1916 if (MemCheckBlock) {
1918 for (Instruction &
I : *MemCheckBlock) {
1919 if (MemCheckBlock->getTerminator() == &
I)
1931 ScalarEvolution *SE = MemCheckExp.
getSE();
1936 const SCEV *
Cond = SE->
getSCEV(MemRuntimeCheckCond);
1941 unsigned BestTripCount = 2;
1945 PSE, OuterLoop,
false))
1946 if (EstimatedTC->isFixed())
1947 BestTripCount = EstimatedTC->getFixedValue();
1952 NewMemCheckCost = std::max(NewMemCheckCost.
getValue(),
1953 (InstructionCost::CostType)1);
1955 if (BestTripCount > 1)
1957 <<
"We expect runtime memory checks to be hoisted "
1958 <<
"out of the outer loop. Cost reduced from "
1959 << MemCheckCost <<
" to " << NewMemCheckCost <<
'\n');
1961 MemCheckCost = NewMemCheckCost;
1965 RTCheckCost += MemCheckCost;
1968 if (SCEVCheckBlock || MemCheckBlock)
1969 LLVM_DEBUG(
dbgs() <<
"Total cost of runtime checks: " << RTCheckCost
1977 ~GeneratedRTChecks() {
1978 SCEVExpanderCleaner SCEVCleaner(SCEVExp);
1979 SCEVExpanderCleaner MemCheckCleaner(MemCheckExp);
1980 bool SCEVChecksUsed = !SCEVCheckBlock || !
pred_empty(SCEVCheckBlock);
1981 bool MemChecksUsed = !MemCheckBlock || !
pred_empty(MemCheckBlock);
1983 SCEVCleaner.markResultUsed();
1985 if (MemChecksUsed) {
1986 MemCheckCleaner.markResultUsed();
1988 auto &SE = *MemCheckExp.
getSE();
1995 I.eraseFromParent();
1998 MemCheckCleaner.cleanup();
1999 SCEVCleaner.cleanup();
2001 if (!SCEVChecksUsed)
2002 SCEVCheckBlock->eraseFromParent();
2004 MemCheckBlock->eraseFromParent();
2009 std::pair<Value *, BasicBlock *> getSCEVChecks()
const {
2010 using namespace llvm::PatternMatch;
2012 return {
nullptr,
nullptr};
2014 return {SCEVCheckCond, SCEVCheckBlock};
2019 std::pair<Value *, BasicBlock *> getMemRuntimeChecks()
const {
2020 using namespace llvm::PatternMatch;
2021 if (MemRuntimeCheckCond &&
match(MemRuntimeCheckCond,
m_ZeroInt()))
2022 return {
nullptr,
nullptr};
2023 return {MemRuntimeCheckCond, MemCheckBlock};
2027 bool hasChecks()
const {
2028 return getSCEVChecks().first || getMemRuntimeChecks().first;
2071 LLVM_DEBUG(
dbgs() <<
"LV: Loop hints prevent outer loop vectorization.\n");
2077 LLVM_DEBUG(
dbgs() <<
"LV: Not vectorizing: Interleave is not supported for "
2107 for (
Loop *InnerL : L)
2130 ?
B.CreateSExtOrTrunc(Index, StepTy)
2131 :
B.CreateCast(Instruction::SIToFP, Index, StepTy);
2132 if (CastedIndex != Index) {
2134 Index = CastedIndex;
2144 assert(
X->getType() ==
Y->getType() &&
"Types don't match!");
2149 return B.CreateAdd(
X,
Y);
2155 assert(
X->getType()->getScalarType() ==
Y->getType() &&
2156 "Types don't match!");
2163 Y =
B.CreateVectorSplat(XVTy->getElementCount(),
Y);
2164 return B.CreateMul(
X,
Y);
2167 switch (InductionKind) {
2170 "Vector indices not supported for integer inductions yet");
2172 "Index type does not match StartValue type");
2174 return B.CreateSub(StartValue, Index);
2179 return B.CreatePtrAdd(StartValue,
CreateMul(Index, Step));
2182 "Vector indices not supported for FP inductions yet");
2185 (InductionBinOp->
getOpcode() == Instruction::FAdd ||
2186 InductionBinOp->
getOpcode() == Instruction::FSub) &&
2187 "Original bin op should be defined for FP induction");
2189 Value *MulExp =
B.CreateFMul(Step, Index);
2190 return B.CreateBinOp(InductionBinOp->
getOpcode(), StartValue, MulExp,
2201 if (std::optional<unsigned> MaxVScale =
TTI.getMaxVScale())
2204 if (
F.hasFnAttribute(Attribute::VScaleRange))
2205 return F.getFnAttribute(Attribute::VScaleRange).getVScaleRangeMax();
2207 return std::nullopt;
2216 ElementCount VF, std::optional<unsigned> UF = std::nullopt) {
2218 unsigned MaxUF = UF ? *UF : Cost->TTI.getMaxInterleaveFactor(VF);
2220 IntegerType *IdxTy = Cost->Legal->getWidestInductionType();
2226 if (
unsigned TC = Cost->PSE.getSmallConstantMaxTripCount()) {
2229 std::optional<unsigned> MaxVScale =
2233 MaxVF *= *MaxVScale;
2236 return (MaxUIntTripCount - TC).ugt(MaxVF * MaxUF);
2250 return TTI.enableMaskedInterleavedAccessVectorization();
2263 PreVectorPH = CheckVPIRBB;
2273 "must have incoming values for all operands");
2274 R.addOperand(R.getOperand(NumPredecessors - 2));
2300 auto CreateStep = [&]() ->
Value * {
2307 if (!
VF.isScalable())
2309 return Builder.CreateBinaryIntrinsic(
2315 Value *Step = CreateStep();
2324 CheckMinIters =
Builder.getTrue();
2326 TripCountSCEV, SE.
getSCEV(Step))) {
2329 CheckMinIters =
Builder.CreateICmp(
P,
Count, Step,
"min.iters.check");
2331 }
else if (
VF.isScalable() && !
TTI->isVScaleKnownToBeAPowerOfTwo() &&
2339 Value *MaxUIntTripCount =
2346 return CheckMinIters;
2355 VPlan *Plan =
nullptr) {
2359 auto IP = IRVPBB->
begin();
2361 R.moveBefore(*IRVPBB, IP);
2365 R.moveBefore(*IRVPBB, IRVPBB->
end());
2374 assert(VectorPH &&
"Invalid loop structure");
2376 Cost->requiresScalarEpilogue(
VF.isVector())) &&
2377 "loops not exiting via the latch without required epilogue?");
2384 Twine(Prefix) +
"scalar.ph");
2390 const SCEV2ValueTy &ExpandedSCEVs) {
2391 const SCEV *Step =
ID.getStep();
2393 return C->getValue();
2395 return U->getValue();
2396 Value *V = ExpandedSCEVs.lookup(Step);
2397 assert(V &&
"SCEV must be expanded at this point");
2407 auto *Cmp = L->getLatchCmpInst();
2409 InstsToIgnore.
insert(Cmp);
2410 for (
const auto &KV : IL) {
2419 [&](
const User *U) { return U == IV || U == Cmp; }))
2420 InstsToIgnore.
insert(IVInst);
2432struct CSEDenseMapInfo {
2443 return DenseMapInfo<Instruction *>::getTombstoneKey();
2446 static unsigned getHashValue(
const Instruction *
I) {
2447 assert(canHandle(
I) &&
"Unknown instruction!");
2452 static bool isEqual(
const Instruction *
LHS,
const Instruction *
RHS) {
2453 if (
LHS == getEmptyKey() ||
RHS == getEmptyKey() ||
2454 LHS == getTombstoneKey() ||
RHS == getTombstoneKey())
2456 return LHS->isIdenticalTo(
RHS);
2467 if (!CSEDenseMapInfo::canHandle(&In))
2473 In.replaceAllUsesWith(V);
2474 In.eraseFromParent();
2487 std::optional<unsigned> VScale) {
2491 EstimatedVF *= *VScale;
2492 assert(EstimatedVF >= 1 &&
"Estimated VF shouldn't be less than 1");
2510 for (
auto &ArgOp : CI->
args())
2521 return ScalarCallCost;
2534 assert(
ID &&
"Expected intrinsic call!");
2538 FMF = FPMO->getFastMathFlags();
2544 std::back_inserter(ParamTys),
2545 [&](
Type *Ty) { return maybeVectorizeType(Ty, VF); });
2550 return TTI.getIntrinsicInstrCost(CostAttrs,
CostKind);
2564 BasicBlock *HeaderBB = State.CFG.VPBB2IRBB[HeaderVPBB];
2579 Builder.SetInsertPoint(NewPhi);
2581 NewPhi->
addIncoming(State.get(Inc), State.CFG.VPBB2IRBB[VPBB]);
2586void LoopVectorizationCostModel::collectLoopScalars(
ElementCount VF) {
2591 "This function should not be visited twice for the same VF");
2616 "Widening decision should be ready at this moment");
2618 if (
Ptr == Store->getValueOperand())
2621 "Ptr is neither a value or pointer operand");
2627 auto IsLoopVaryingGEP = [&](
Value *
V) {
2638 if (!IsLoopVaryingGEP(
Ptr))
2650 if (IsScalarUse(MemAccess,
Ptr) &&
2654 PossibleNonScalarPtrs.
insert(
I);
2670 for (
auto *BB :
TheLoop->blocks())
2671 for (
auto &
I : *BB) {
2673 EvaluatePtrUse(Load,
Load->getPointerOperand());
2675 EvaluatePtrUse(Store,
Store->getPointerOperand());
2676 EvaluatePtrUse(Store,
Store->getValueOperand());
2679 for (
auto *
I : ScalarPtrs)
2680 if (!PossibleNonScalarPtrs.
count(
I)) {
2688 auto ForcedScalar = ForcedScalars.find(VF);
2689 if (ForcedScalar != ForcedScalars.end())
2690 for (
auto *
I : ForcedScalar->second) {
2691 LLVM_DEBUG(
dbgs() <<
"LV: Found (forced) scalar instruction: " << *
I <<
"\n");
2700 while (Idx != Worklist.
size()) {
2702 if (!IsLoopVaryingGEP(Dst->getOperand(0)))
2706 auto *J = cast<Instruction>(U);
2707 return !TheLoop->contains(J) || Worklist.count(J) ||
2708 ((isa<LoadInst>(J) || isa<StoreInst>(J)) &&
2709 IsScalarUse(J, Src));
2712 LLVM_DEBUG(
dbgs() <<
"LV: Found scalar instruction: " << *Src <<
"\n");
2718 for (
const auto &Induction :
Legal->getInductionVars()) {
2719 auto *Ind = Induction.first;
2729 auto IsDirectLoadStoreFromPtrIndvar = [&](
Instruction *Indvar,
2731 return Induction.second.getKind() ==
2739 bool ScalarInd =
all_of(Ind->users(), [&](User *U) ->
bool {
2740 auto *I = cast<Instruction>(U);
2741 return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I) ||
2742 IsDirectLoadStoreFromPtrIndvar(Ind, I);
2751 if (IndUpdatePhi &&
Legal->isFixedOrderRecurrence(IndUpdatePhi))
2756 bool ScalarIndUpdate =
all_of(IndUpdate->users(), [&](User *U) ->
bool {
2757 auto *I = cast<Instruction>(U);
2758 return I == Ind || !TheLoop->contains(I) || Worklist.count(I) ||
2759 IsDirectLoadStoreFromPtrIndvar(IndUpdate, I);
2761 if (!ScalarIndUpdate)
2766 Worklist.
insert(IndUpdate);
2767 LLVM_DEBUG(
dbgs() <<
"LV: Found scalar instruction: " << *Ind <<
"\n");
2768 LLVM_DEBUG(
dbgs() <<
"LV: Found scalar instruction: " << *IndUpdate
2772 Scalars[VF].insert_range(Worklist);
2782 switch(
I->getOpcode()) {
2785 case Instruction::Call:
2789 case Instruction::Load:
2790 case Instruction::Store: {
2799 TTI.isLegalMaskedGather(VTy, Alignment))
2801 TTI.isLegalMaskedScatter(VTy, Alignment));
2803 case Instruction::UDiv:
2804 case Instruction::SDiv:
2805 case Instruction::SRem:
2806 case Instruction::URem: {
2827 if (
Legal->blockNeedsPredication(
I->getParent()))
2839 switch(
I->getOpcode()) {
2842 "instruction should have been considered by earlier checks");
2843 case Instruction::Call:
2847 "should have returned earlier for calls not needing a mask");
2849 case Instruction::Load:
2852 case Instruction::Store: {
2860 case Instruction::UDiv:
2861 case Instruction::SDiv:
2862 case Instruction::SRem:
2863 case Instruction::URem:
2865 return !
Legal->isInvariant(
I->getOperand(1));
2869std::pair<InstructionCost, InstructionCost>
2872 assert(
I->getOpcode() == Instruction::UDiv ||
2873 I->getOpcode() == Instruction::SDiv ||
2874 I->getOpcode() == Instruction::SRem ||
2875 I->getOpcode() == Instruction::URem);
2884 ScalarizationCost = 0;
2890 ScalarizationCost +=
2894 ScalarizationCost +=
2896 TTI.getArithmeticInstrCost(
I->getOpcode(),
I->getType(),
CostKind);
2900 ScalarizationCost += getScalarizationOverhead(
I, VF);
2911 if (DivisorI && !
Legal->isInvariant(DivisorI)) {
2915 TTI.getCmpSelInstrCost(Instruction::Select, VecTy,
2921 SafeDivisorCost +=
TTI.getArithmeticInstrCost(
2923 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
2924 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
2926 return {ScalarizationCost, SafeDivisorCost};
2933 "Decision should not be set yet.");
2935 assert(Group &&
"Must have a group.");
2936 unsigned InterleaveFactor = Group->getFactor();
2940 auto &
DL =
I->getDataLayout();
2952 bool ScalarNI =
DL.isNonIntegralPointerType(ScalarTy);
2953 for (
unsigned Idx = 0; Idx < InterleaveFactor; Idx++) {
2958 bool MemberNI =
DL.isNonIntegralPointerType(MemberTy);
2960 if (MemberNI != ScalarNI)
2963 if (MemberNI && ScalarNI &&
2964 ScalarTy->getPointerAddressSpace() !=
2965 MemberTy->getPointerAddressSpace())
2974 bool PredicatedAccessRequiresMasking =
2976 Legal->isMaskRequired(
I);
2977 bool LoadAccessWithGapsRequiresEpilogMasking =
2980 bool StoreAccessWithGapsRequiresMasking =
2982 if (!PredicatedAccessRequiresMasking &&
2983 !LoadAccessWithGapsRequiresEpilogMasking &&
2984 !StoreAccessWithGapsRequiresMasking)
2991 "Masked interleave-groups for predicated accesses are not enabled.");
2993 if (Group->isReverse())
2997 bool NeedsMaskForGaps = LoadAccessWithGapsRequiresEpilogMasking ||
2998 StoreAccessWithGapsRequiresMasking;
3006 :
TTI.isLegalMaskedStore(Ty, Alignment, AS);
3018 if (!
Legal->isConsecutivePtr(ScalarTy,
Ptr))
3028 auto &
DL =
I->getDataLayout();
3035void LoopVectorizationCostModel::collectLoopUniforms(
ElementCount VF) {
3042 "This function should not be visited twice for the same VF");
3046 Uniforms[VF].
clear();
3054 auto IsOutOfScope = [&](
Value *V) ->
bool {
3066 auto AddToWorklistIfAllowed = [&](
Instruction *
I) ->
void {
3067 if (IsOutOfScope(
I)) {
3074 dbgs() <<
"LV: Found not uniform due to requiring predication: " << *
I
3078 LLVM_DEBUG(
dbgs() <<
"LV: Found uniform instruction: " << *
I <<
"\n");
3087 TheLoop->getExitingBlocks(Exiting);
3088 for (BasicBlock *
E : Exiting) {
3089 if (
Legal->hasUncountableEarlyExit() &&
TheLoop->getLoopLatch() !=
E)
3092 if (Cmp &&
TheLoop->contains(Cmp) &&
Cmp->hasOneUse())
3093 AddToWorklistIfAllowed(Cmp);
3102 if (PrevVF.isVector()) {
3103 auto Iter = Uniforms.find(PrevVF);
3104 if (Iter != Uniforms.end() && !Iter->second.contains(
I))
3107 if (!
Legal->isUniformMemOp(*
I, VF))
3117 auto IsUniformDecision = [&](
Instruction *
I, ElementCount VF) {
3120 "Widening decision should be ready at this moment");
3122 if (IsUniformMemOpUse(
I))
3125 return (WideningDecision ==
CM_Widen ||
3137 (IsUniformDecision(
I, VF) ||
Legal->isInvariant(
Ptr));
3145 SetVector<Value *> HasUniformUse;
3149 for (
auto *BB :
TheLoop->blocks())
3150 for (
auto &
I : *BB) {
3152 switch (
II->getIntrinsicID()) {
3153 case Intrinsic::sideeffect:
3154 case Intrinsic::experimental_noalias_scope_decl:
3155 case Intrinsic::assume:
3156 case Intrinsic::lifetime_start:
3157 case Intrinsic::lifetime_end:
3158 if (
TheLoop->hasLoopInvariantOperands(&
I))
3159 AddToWorklistIfAllowed(&
I);
3167 if (IsOutOfScope(EVI->getAggregateOperand())) {
3168 AddToWorklistIfAllowed(EVI);
3174 "Expected aggregate value to be call return value");
3187 if (IsUniformMemOpUse(&
I))
3188 AddToWorklistIfAllowed(&
I);
3190 if (IsVectorizedMemAccessUse(&
I,
Ptr))
3197 for (
auto *V : HasUniformUse) {
3198 if (IsOutOfScope(V))
3201 bool UsersAreMemAccesses =
all_of(
I->users(), [&](User *U) ->
bool {
3202 auto *UI = cast<Instruction>(U);
3203 return TheLoop->contains(UI) && IsVectorizedMemAccessUse(UI, V);
3205 if (UsersAreMemAccesses)
3206 AddToWorklistIfAllowed(
I);
3213 while (Idx != Worklist.
size()) {
3216 for (
auto *OV :
I->operand_values()) {
3218 if (IsOutOfScope(OV))
3223 if (
OP &&
Legal->isFixedOrderRecurrence(
OP))
3229 auto *J = cast<Instruction>(U);
3230 return Worklist.count(J) || IsVectorizedMemAccessUse(J, OI);
3232 AddToWorklistIfAllowed(OI);
3243 for (
const auto &Induction :
Legal->getInductionVars()) {
3244 auto *Ind = Induction.first;
3249 bool UniformInd =
all_of(Ind->users(), [&](User *U) ->
bool {
3250 auto *I = cast<Instruction>(U);
3251 return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I) ||
3252 IsVectorizedMemAccessUse(I, Ind);
3259 bool UniformIndUpdate =
all_of(IndUpdate->users(), [&](User *U) ->
bool {
3260 auto *I = cast<Instruction>(U);
3261 return I == Ind || Worklist.count(I) ||
3262 IsVectorizedMemAccessUse(I, IndUpdate);
3264 if (!UniformIndUpdate)
3268 AddToWorklistIfAllowed(Ind);
3269 AddToWorklistIfAllowed(IndUpdate);
3272 Uniforms[VF].insert_range(Worklist);
3278 if (
Legal->getRuntimePointerChecking()->Need) {
3280 "runtime pointer checks needed. Enable vectorization of this "
3281 "loop with '#pragma clang loop vectorize(enable)' when "
3282 "compiling with -Os/-Oz",
3283 "CantVersionLoopWithOptForSize",
ORE,
TheLoop);
3287 if (!
PSE.getPredicate().isAlwaysTrue()) {
3289 "runtime SCEV checks needed. Enable vectorization of this "
3290 "loop with '#pragma clang loop vectorize(enable)' when "
3291 "compiling with -Os/-Oz",
3292 "CantVersionLoopWithOptForSize",
ORE,
TheLoop);
3297 if (!
Legal->getLAI()->getSymbolicStrides().empty()) {
3299 "runtime stride == 1 checks needed. Enable vectorization of "
3300 "this loop without such check by compiling with -Os/-Oz",
3301 "CantVersionLoopWithOptForSize",
ORE,
TheLoop);
3308bool LoopVectorizationCostModel::isScalableVectorizationAllowed() {
3309 if (IsScalableVectorizationAllowed)
3310 return *IsScalableVectorizationAllowed;
3312 IsScalableVectorizationAllowed =
false;
3318 "ScalableVectorizationDisabled",
ORE,
TheLoop);
3322 LLVM_DEBUG(
dbgs() <<
"LV: Scalable vectorization is available\n");
3325 std::numeric_limits<ElementCount::ScalarTy>::max());
3336 "Scalable vectorization not supported for the reduction "
3337 "operations found in this loop.",
3346 !this->
TTI.isElementTypeLegalForScalableVector(Ty);
3349 "for all element types found in this loop.",
3356 "for safe distance analysis.",
3361 IsScalableVectorizationAllowed =
true;
3366LoopVectorizationCostModel::getMaxLegalScalableVF(
unsigned MaxSafeElements) {
3367 if (!isScalableVectorizationAllowed())
3371 std::numeric_limits<ElementCount::ScalarTy>::max());
3372 if (
Legal->isSafeForAnyVectorWidth())
3373 return MaxScalableVF;
3381 "Max legal vector width too small, scalable vectorization "
3385 return MaxScalableVF;
3389 unsigned MaxTripCount,
ElementCount UserVF,
bool FoldTailByMasking) {
3391 unsigned SmallestType, WidestType;
3398 unsigned MaxSafeElementsPowerOf2 =
3400 if (!
Legal->isSafeForAnyStoreLoadForwardDistances()) {
3401 unsigned SLDist =
Legal->getMaxStoreLoadForwardSafeDistanceInBits();
3402 MaxSafeElementsPowerOf2 =
3403 std::min(MaxSafeElementsPowerOf2, SLDist / WidestType);
3406 auto MaxSafeScalableVF = getMaxLegalScalableVF(MaxSafeElementsPowerOf2);
3408 if (!
Legal->isSafeForAnyVectorWidth())
3409 this->MaxSafeElements = MaxSafeElementsPowerOf2;
3411 LLVM_DEBUG(
dbgs() <<
"LV: The max safe fixed VF is: " << MaxSafeFixedVF
3413 LLVM_DEBUG(
dbgs() <<
"LV: The max safe scalable VF is: " << MaxSafeScalableVF
3418 auto MaxSafeUserVF =
3419 UserVF.
isScalable() ? MaxSafeScalableVF : MaxSafeFixedVF;
3424 return FixedScalableVFPair(
3436 <<
" is unsafe, clamping to max safe VF="
3437 << MaxSafeFixedVF <<
".\n");
3439 return OptimizationRemarkAnalysis(
DEBUG_TYPE,
"VectorizationFactor",
3442 <<
"User-specified vectorization factor "
3443 <<
ore::NV(
"UserVectorizationFactor", UserVF)
3444 <<
" is unsafe, clamping to maximum safe vectorization factor "
3445 <<
ore::NV(
"VectorizationFactor", MaxSafeFixedVF);
3447 return MaxSafeFixedVF;
3452 <<
" is ignored because scalable vectors are not "
3455 return OptimizationRemarkAnalysis(
DEBUG_TYPE,
"VectorizationFactor",
3458 <<
"User-specified vectorization factor "
3459 <<
ore::NV(
"UserVectorizationFactor", UserVF)
3460 <<
" is ignored because the target does not support scalable "
3461 "vectors. The compiler will pick a more suitable value.";
3465 <<
" is unsafe. Ignoring scalable UserVF.\n");
3467 return OptimizationRemarkAnalysis(
DEBUG_TYPE,
"VectorizationFactor",
3470 <<
"User-specified vectorization factor "
3471 <<
ore::NV(
"UserVectorizationFactor", UserVF)
3472 <<
" is unsafe. Ignoring the hint to let the compiler pick a "
3473 "more suitable value.";
3478 LLVM_DEBUG(
dbgs() <<
"LV: The Smallest and Widest types: " << SmallestType
3479 <<
" / " << WidestType <<
" bits.\n");
3484 getMaximizedVFForTarget(MaxTripCount, SmallestType, WidestType,
3485 MaxSafeFixedVF, FoldTailByMasking))
3489 getMaximizedVFForTarget(MaxTripCount, SmallestType, WidestType,
3490 MaxSafeScalableVF, FoldTailByMasking))
3491 if (MaxVF.isScalable()) {
3492 Result.ScalableVF = MaxVF;
3493 LLVM_DEBUG(
dbgs() <<
"LV: Found feasible scalable VF = " << MaxVF
3502 if (
Legal->getRuntimePointerChecking()->Need &&
TTI.hasBranchDivergence()) {
3506 "Not inserting runtime ptr check for divergent target",
3507 "runtime pointer checks needed. Not enabled for divergent target",
3508 "CantVersionLoopWithDivergentTarget",
ORE,
TheLoop);
3514 unsigned MaxTC =
PSE.getSmallConstantMaxTripCount();
3517 LLVM_DEBUG(
dbgs() <<
"LV: Found maximum trip count: " << MaxTC <<
'\n');
3520 "loop trip count is one, irrelevant for vectorization",
3531 Legal->getWidestInductionType()->getScalarSizeInBits() &&
3535 "Trip count computation wrapped",
3536 "backedge-taken count is -1, loop trip count wrapped to 0",
3541 switch (ScalarEpilogueStatus) {
3543 return computeFeasibleMaxVF(MaxTC, UserVF,
false);
3548 dbgs() <<
"LV: vector predicate hint/switch found.\n"
3549 <<
"LV: Not allowing scalar epilogue, creating predicated "
3550 <<
"vector loop.\n");
3557 dbgs() <<
"LV: Not allowing scalar epilogue due to -Os/-Oz.\n");
3559 LLVM_DEBUG(
dbgs() <<
"LV: Not allowing scalar epilogue due to low trip "
3575 assert(WideningDecisions.empty() && Uniforms.empty() && Scalars.empty() &&
3576 "No decisions should have been taken at this point");
3586 std::optional<unsigned> MaxPowerOf2RuntimeVF =
3590 if (MaxVScale &&
TTI.isVScaleKnownToBeAPowerOfTwo()) {
3591 MaxPowerOf2RuntimeVF = std::max<unsigned>(
3592 *MaxPowerOf2RuntimeVF,
3595 MaxPowerOf2RuntimeVF = std::nullopt;
3598 auto NoScalarEpilogueNeeded = [
this, &UserIC](
unsigned MaxVF) {
3602 !
Legal->hasUncountableEarlyExit())
3604 unsigned MaxVFtimesIC = UserIC ? MaxVF * UserIC : MaxVF;
3609 const SCEV *BackedgeTakenCount =
PSE.getSymbolicMaxBackedgeTakenCount();
3611 BackedgeTakenCount ==
PSE.getBackedgeTakenCount()) &&
3612 "Invalid loop count");
3614 BackedgeTakenCount, SE->
getOne(BackedgeTakenCount->
getType()));
3621 if (MaxPowerOf2RuntimeVF > 0u) {
3623 "MaxFixedVF must be a power of 2");
3624 if (NoScalarEpilogueNeeded(*MaxPowerOf2RuntimeVF)) {
3626 LLVM_DEBUG(
dbgs() <<
"LV: No tail will remain for any chosen VF.\n");
3632 if (ExpectedTC && ExpectedTC->isFixed() &&
3633 ExpectedTC->getFixedValue() <=
3634 TTI.getMinTripCountTailFoldingThreshold()) {
3635 if (MaxPowerOf2RuntimeVF > 0u) {
3641 LLVM_DEBUG(
dbgs() <<
"LV: Picking a fixed-width so that no tail will "
3642 "remain for any chosen VF.\n");
3649 "The trip count is below the minial threshold value.",
3650 "loop trip count is too low, avoiding vectorization",
"LowTripCount",
3665 <<
"LV: tail is folded with EVL, forcing unroll factor to be 1. Will "
3666 "try to generate VP Intrinsics with scalable vector "
3671 assert(ContainsScalableVF &&
"Expected scalable vector factor.");
3681 LLVM_DEBUG(
dbgs() <<
"LV: Cannot fold tail by masking: vectorize with a "
3682 "scalar epilogue instead.\n");
3688 LLVM_DEBUG(
dbgs() <<
"LV: Can't fold tail by masking: don't vectorize\n");
3694 "unable to calculate the loop count due to complex control flow",
3700 "Cannot optimize for size and vectorize at the same time.",
3701 "cannot optimize for size and vectorize at the same time. "
3702 "Enable vectorization of this loop with '#pragma clang loop "
3703 "vectorize(enable)' when compiling with -Os/-Oz",
3715 if (
TTI.shouldConsiderVectorizationRegPressure())
3731 (
TTI.shouldMaximizeVectorBandwidth(RegKind) ||
3733 Legal->hasVectorCallVariants())));
3736ElementCount LoopVectorizationCostModel::clampVFByMaxTripCount(
3737 ElementCount VF,
unsigned MaxTripCount,
bool FoldTailByMasking)
const {
3751 if (MaxTripCount && MaxTripCount <= EstimatedVF &&
3759 LLVM_DEBUG(
dbgs() <<
"LV: Clamping the MaxVF to maximum power of two not "
3760 "exceeding the constant trip count: "
3761 << ClampedUpperTripCount <<
"\n");
3763 FoldTailByMasking ? VF.
isScalable() :
false);
3768ElementCount LoopVectorizationCostModel::getMaximizedVFForTarget(
3769 unsigned MaxTripCount,
unsigned SmallestType,
unsigned WidestType,
3771 bool ComputeScalableMaxVF = MaxSafeVF.
isScalable();
3772 const TypeSize WidestRegister =
TTI.getRegisterBitWidth(
3777 auto MinVF = [](
const ElementCount &
LHS,
const ElementCount &
RHS) {
3779 "Scalable flags must match");
3787 ComputeScalableMaxVF);
3788 MaxVectorElementCount = MinVF(MaxVectorElementCount, MaxSafeVF);
3790 << (MaxVectorElementCount * WidestType) <<
" bits.\n");
3792 if (!MaxVectorElementCount) {
3794 << (ComputeScalableMaxVF ?
"scalable" :
"fixed")
3795 <<
" vector registers.\n");
3799 ElementCount MaxVF = clampVFByMaxTripCount(MaxVectorElementCount,
3800 MaxTripCount, FoldTailByMasking);
3803 if (MaxVF != MaxVectorElementCount)
3818 ComputeScalableMaxVF);
3819 MaxVF = MinVF(MaxVectorElementCountMaxBW, MaxSafeVF);
3821 if (ElementCount MinVF =
3822 TTI.getMinimumVF(SmallestType, ComputeScalableMaxVF)) {
3825 <<
") with target's minimum: " << MinVF <<
'\n');
3830 MaxVF = clampVFByMaxTripCount(MaxVF, MaxTripCount, FoldTailByMasking);
3832 if (MaxVectorElementCount != MaxVF) {
3844 const unsigned MaxTripCount,
3846 bool IsEpilogue)
const {
3852 unsigned EstimatedWidthB =
B.Width.getKnownMinValue();
3853 if (std::optional<unsigned> VScale = CM.getVScaleForTuning()) {
3854 if (
A.Width.isScalable())
3855 EstimatedWidthA *= *VScale;
3856 if (
B.Width.isScalable())
3857 EstimatedWidthB *= *VScale;
3864 return CostA < CostB ||
3865 (CostA == CostB && EstimatedWidthA > EstimatedWidthB);
3870 bool PreferScalable = !TTI.preferFixedOverScalableIfEqualCost(IsEpilogue) &&
3871 A.Width.isScalable() && !
B.Width.isScalable();
3882 return CmpFn(CostA * EstimatedWidthB, CostB * EstimatedWidthA);
3884 auto GetCostForTC = [MaxTripCount, HasTail](
unsigned VF,
3896 return VectorCost * (MaxTripCount / VF) +
3897 ScalarCost * (MaxTripCount % VF);
3898 return VectorCost *
divideCeil(MaxTripCount, VF);
3901 auto RTCostA = GetCostForTC(EstimatedWidthA, CostA,
A.ScalarCost);
3902 auto RTCostB = GetCostForTC(EstimatedWidthB, CostB,
B.ScalarCost);
3903 return CmpFn(RTCostA, RTCostB);
3909 bool IsEpilogue)
const {
3910 const unsigned MaxTripCount = PSE.getSmallConstantMaxTripCount();
3911 return LoopVectorizationPlanner::isMoreProfitable(
A,
B, MaxTripCount, HasTail,
3917 using RecipeVFPair = std::pair<VPRecipeBase *, ElementCount>;
3919 for (
const auto &Plan : VPlans) {
3928 VPCostContext CostCtx(CM.TTI, *CM.TLI, *Plan, CM, CM.CostKind);
3929 precomputeCosts(*Plan, VF, CostCtx);
3932 for (
auto &R : *VPBB) {
3933 if (!R.cost(VF, CostCtx).isValid())
3939 if (InvalidCosts.
empty())
3947 for (
auto &Pair : InvalidCosts)
3952 sort(InvalidCosts, [&Numbering](RecipeVFPair &
A, RecipeVFPair &
B) {
3953 unsigned NA = Numbering[
A.first];
3954 unsigned NB = Numbering[
B.first];
3969 Subset =
Tail.take_front(1);
3976 [](
const auto *R) {
return Instruction::PHI; })
3977 .Case<VPWidenSelectRecipe>(
3978 [](
const auto *R) {
return Instruction::Select; })
3979 .Case<VPWidenStoreRecipe>(
3980 [](
const auto *R) {
return Instruction::Store; })
3981 .Case<VPWidenLoadRecipe>(
3982 [](
const auto *R) {
return Instruction::Load; })
3983 .Case<VPWidenCallRecipe, VPWidenIntrinsicRecipe>(
3984 [](
const auto *R) {
return Instruction::Call; })
3987 [](
const auto *R) {
return R->getOpcode(); })
3989 return R->getStoredValues().empty() ? Instruction::Load
3990 : Instruction::Store;
3998 if (Subset ==
Tail ||
Tail[Subset.size()].first != R) {
3999 std::string OutString;
4001 assert(!Subset.empty() &&
"Unexpected empty range");
4002 OS <<
"Recipe with invalid costs prevented vectorization at VF=(";
4003 for (
const auto &Pair : Subset)
4004 OS << (Pair.second == Subset.front().second ?
"" :
", ") << Pair.second;
4006 if (Opcode == Instruction::Call) {
4009 Name =
Int->getIntrinsicName();
4013 WidenCall ? WidenCall->getCalledScalarFunction()
4015 ->getLiveInIRValue());
4018 OS <<
" call to " << Name;
4023 Tail =
Tail.drop_front(Subset.size());
4027 Subset =
Tail.take_front(Subset.size() + 1);
4028 }
while (!
Tail.empty());
4050 switch (R.getVPDefID()) {
4051 case VPDef::VPDerivedIVSC:
4052 case VPDef::VPScalarIVStepsSC:
4053 case VPDef::VPReplicateSC:
4054 case VPDef::VPInstructionSC:
4055 case VPDef::VPCanonicalIVPHISC:
4056 case VPDef::VPVectorPointerSC:
4057 case VPDef::VPVectorEndPointerSC:
4058 case VPDef::VPExpandSCEVSC:
4059 case VPDef::VPEVLBasedIVPHISC:
4060 case VPDef::VPPredInstPHISC:
4061 case VPDef::VPBranchOnMaskSC:
4063 case VPDef::VPReductionSC:
4064 case VPDef::VPActiveLaneMaskPHISC:
4065 case VPDef::VPWidenCallSC:
4066 case VPDef::VPWidenCanonicalIVSC:
4067 case VPDef::VPWidenCastSC:
4068 case VPDef::VPWidenGEPSC:
4069 case VPDef::VPWidenIntrinsicSC:
4070 case VPDef::VPWidenSC:
4071 case VPDef::VPWidenSelectSC:
4072 case VPDef::VPBlendSC:
4073 case VPDef::VPFirstOrderRecurrencePHISC:
4074 case VPDef::VPHistogramSC:
4075 case VPDef::VPWidenPHISC:
4076 case VPDef::VPWidenIntOrFpInductionSC:
4077 case VPDef::VPWidenPointerInductionSC:
4078 case VPDef::VPReductionPHISC:
4079 case VPDef::VPInterleaveEVLSC:
4080 case VPDef::VPInterleaveSC:
4081 case VPDef::VPWidenLoadEVLSC:
4082 case VPDef::VPWidenLoadSC:
4083 case VPDef::VPWidenStoreEVLSC:
4084 case VPDef::VPWidenStoreSC:
4090 auto WillGenerateTargetVectors = [&
TTI, VF](
Type *VectorTy) {
4091 unsigned NumLegalParts =
TTI.getNumberOfParts(VectorTy);
4107 if (R.getNumDefinedValues() == 0 &&
4116 R.getNumDefinedValues() >= 1 ? R.getVPValue(0) : R.getOperand(1);
4118 if (!Visited.
insert({ScalarTy}).second)
4132 [](
auto *VPRB) { return VPRB->isReplicator(); });
4138 LLVM_DEBUG(
dbgs() <<
"LV: Scalar loop costs: " << ExpectedCost <<
".\n");
4139 assert(ExpectedCost.
isValid() &&
"Unexpected invalid cost for scalar loop");
4142 [](std::unique_ptr<VPlan> &
P) {
return P->hasScalarVFOnly(); }) &&
4143 "Expected Scalar VF to be a candidate");
4150 if (ForceVectorization &&
4151 (VPlans.size() > 1 || !VPlans[0]->hasScalarVFOnly())) {
4158 for (
auto &
P : VPlans) {
4160 P->vectorFactors().end());
4163 if (
any_of(VFs, [
this](ElementCount VF) {
4164 return CM.shouldConsiderRegPressureForVF(VF);
4168 for (
unsigned I = 0;
I < VFs.size();
I++) {
4169 ElementCount VF = VFs[
I];
4177 if (CM.shouldConsiderRegPressureForVF(VF) &&
4185 VPCostContext CostCtx(CM.TTI, *CM.TLI, *
P, CM, CM.CostKind);
4186 VPRegionBlock *VectorRegion =
P->getVectorLoopRegion();
4187 assert(VectorRegion &&
"Expected to have a vector region!");
4190 for (VPRecipeBase &R : *VPBB) {
4194 switch (VPI->getOpcode()) {
4197 case Instruction::Select: {
4198 VPValue *VPV = VPI->getVPSingleValue();
4201 switch (WR->getOpcode()) {
4202 case Instruction::UDiv:
4203 case Instruction::SDiv:
4204 case Instruction::URem:
4205 case Instruction::SRem:
4212 C += VPI->cost(VF, CostCtx);
4216 unsigned Multiplier =
4219 C += VPI->cost(VF * Multiplier, CostCtx);
4223 C += VPI->cost(VF, CostCtx);
4235 <<
" costs: " << (Candidate.Cost / Width));
4238 << CM.getVScaleForTuning().value_or(1) <<
")");
4244 <<
"LV: Not considering vector loop of width " << VF
4245 <<
" because it will not generate any vector instructions.\n");
4252 <<
"LV: Not considering vector loop of width " << VF
4253 <<
" because it would cause replicated blocks to be generated,"
4254 <<
" which isn't allowed when optimizing for size.\n");
4258 if (isMoreProfitable(Candidate, ChosenFactor,
P->hasScalarTail()))
4259 ChosenFactor = Candidate;
4265 "There are conditional stores.",
4266 "store that is conditionally executed prevents vectorization",
4267 "ConditionalStore", ORE, OrigLoop);
4268 ChosenFactor = ScalarCost;
4272 !isMoreProfitable(ChosenFactor, ScalarCost,
4273 !CM.foldTailByMasking()))
dbgs()
4274 <<
"LV: Vectorization seems to be not beneficial, "
4275 <<
"but was forced by a user.\n");
4276 return ChosenFactor;
4280bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
4284 if (
any_of(OrigLoop->getHeader()->phis(), [&](PHINode &Phi) {
4285 if (!Legal->isReductionVariable(&Phi))
4286 return Legal->isFixedOrderRecurrence(&Phi);
4287 RecurKind RK = Legal->getRecurrenceDescriptor(&Phi).getRecurrenceKind();
4288 return RK == RecurKind::FMinNum || RK == RecurKind::FMaxNum;
4294 for (
const auto &Entry : Legal->getInductionVars()) {
4297 Entry.first->getIncomingValueForBlock(OrigLoop->getLoopLatch());
4298 for (User *U :
PostInc->users())
4302 for (User *U :
Entry.first->users())
4311 if (OrigLoop->getExitingBlock() != OrigLoop->getLoopLatch())
4325 if (!
TTI.preferEpilogueVectorization())
4330 if (
TTI.getMaxInterleaveFactor(VF) <= 1)
4335 :
TTI.getEpilogueVectorizationMinVF();
4343 LLVM_DEBUG(
dbgs() <<
"LEV: Epilogue vectorization is disabled.\n");
4347 if (!CM.isScalarEpilogueAllowed()) {
4348 LLVM_DEBUG(
dbgs() <<
"LEV: Unable to vectorize epilogue because no "
4349 "epilogue is allowed.\n");
4355 if (!isCandidateForEpilogueVectorization(MainLoopVF)) {
4356 LLVM_DEBUG(
dbgs() <<
"LEV: Unable to vectorize epilogue because the loop "
4357 "is not a supported candidate.\n");
4362 LLVM_DEBUG(
dbgs() <<
"LEV: Epilogue vectorization factor is forced.\n");
4365 return {ForcedEC, 0, 0};
4367 LLVM_DEBUG(
dbgs() <<
"LEV: Epilogue vectorization forced factor is not "
4372 if (OrigLoop->getHeader()->getParent()->hasOptSize()) {
4374 dbgs() <<
"LEV: Epilogue vectorization skipped due to opt for size.\n");
4378 if (!CM.isEpilogueVectorizationProfitable(MainLoopVF, IC)) {
4379 LLVM_DEBUG(
dbgs() <<
"LEV: Epilogue vectorization is not profitable for "
4391 Type *TCType = Legal->getWidestInductionType();
4392 const SCEV *RemainingIterations =
nullptr;
4393 unsigned MaxTripCount = 0;
4397 RemainingIterations =
4401 if (RemainingIterations->
isZero())
4411 << MaxTripCount <<
"\n");
4414 for (
auto &NextVF : ProfitableVFs) {
4421 if ((!NextVF.Width.isScalable() && MainLoopVF.
isScalable() &&
4423 (NextVF.Width.isScalable() &&
4425 (!NextVF.Width.isScalable() && !MainLoopVF.
isScalable() &&
4431 if (RemainingIterations && !NextVF.Width.isScalable()) {
4434 SE.
getConstant(TCType, NextVF.Width.getFixedValue()),
4435 RemainingIterations))
4439 if (Result.Width.isScalar() ||
4440 isMoreProfitable(NextVF, Result, MaxTripCount, !CM.foldTailByMasking(),
4447 << Result.Width <<
"\n");
4451std::pair<unsigned, unsigned>
4453 unsigned MinWidth = -1U;
4454 unsigned MaxWidth = 8;
4460 for (
const auto &PhiDescriptorPair :
Legal->getReductionVars()) {
4464 MinWidth = std::min(
4468 MaxWidth = std::max(MaxWidth,
4473 MinWidth = std::min<unsigned>(
4474 MinWidth,
DL.getTypeSizeInBits(
T->getScalarType()).getFixedValue());
4475 MaxWidth = std::max<unsigned>(
4476 MaxWidth,
DL.getTypeSizeInBits(
T->getScalarType()).getFixedValue());
4479 return {MinWidth, MaxWidth};
4487 for (
Instruction &
I : BB->instructionsWithoutDebug()) {
4501 if (!
Legal->isReductionVariable(PN))
4504 Legal->getRecurrenceDescriptor(PN);
4514 T = ST->getValueOperand()->getType();
4517 "Expected the load/store/recurrence type to be sized");
4541 if (!CM.isScalarEpilogueAllowed())
4546 LLVM_DEBUG(
dbgs() <<
"LV: Preference for VP intrinsics indicated. "
4547 "Unroll factor forced to be 1.\n");
4552 if (!Legal->isSafeForAnyVectorWidth())
4561 const bool HasReductions =
4567 if (LoopCost == 0) {
4569 LoopCost = CM.expectedCost(VF);
4571 LoopCost = cost(Plan, VF);
4572 assert(LoopCost.
isValid() &&
"Expected to have chosen a VF with valid cost");
4583 for (
auto &Pair : R.MaxLocalUsers) {
4584 Pair.second = std::max(Pair.second, 1U);
4598 unsigned IC = UINT_MAX;
4600 for (
const auto &Pair : R.MaxLocalUsers) {
4601 unsigned TargetNumRegisters = TTI.getNumberOfRegisters(Pair.first);
4604 << TTI.getRegisterClassName(Pair.first)
4605 <<
" register class\n");
4613 unsigned MaxLocalUsers = Pair.second;
4614 unsigned LoopInvariantRegs = 0;
4615 if (R.LoopInvariantRegs.contains(Pair.first))
4616 LoopInvariantRegs = R.LoopInvariantRegs[Pair.first];
4618 unsigned TmpIC =
llvm::bit_floor((TargetNumRegisters - LoopInvariantRegs) /
4622 TmpIC =
llvm::bit_floor((TargetNumRegisters - LoopInvariantRegs - 1) /
4623 std::max(1U, (MaxLocalUsers - 1)));
4626 IC = std::min(IC, TmpIC);
4630 unsigned MaxInterleaveCount = TTI.getMaxInterleaveFactor(VF);
4646 if (BestKnownTC && (BestKnownTC->isFixed() || VF.
isScalable())) {
4648 unsigned AvailableTC =
4654 if (CM.requiresScalarEpilogue(VF.
isVector()))
4657 unsigned InterleaveCountLB =
bit_floor(std::max(
4658 1u, std::min(AvailableTC / (EstimatedVF * 2), MaxInterleaveCount)));
4672 unsigned InterleaveCountUB =
bit_floor(std::max(
4673 1u, std::min(AvailableTC / EstimatedVF, MaxInterleaveCount)));
4674 MaxInterleaveCount = InterleaveCountLB;
4676 if (InterleaveCountUB != InterleaveCountLB) {
4677 unsigned TailTripCountUB =
4678 (AvailableTC % (EstimatedVF * InterleaveCountUB));
4679 unsigned TailTripCountLB =
4680 (AvailableTC % (EstimatedVF * InterleaveCountLB));
4683 if (TailTripCountUB == TailTripCountLB)
4684 MaxInterleaveCount = InterleaveCountUB;
4692 MaxInterleaveCount = InterleaveCountLB;
4696 assert(MaxInterleaveCount > 0 &&
4697 "Maximum interleave count must be greater than 0");
4701 if (IC > MaxInterleaveCount)
4702 IC = MaxInterleaveCount;
4705 IC = std::max(1u, IC);
4707 assert(IC > 0 &&
"Interleave count must be greater than 0.");
4711 if (VF.
isVector() && HasReductions) {
4712 LLVM_DEBUG(
dbgs() <<
"LV: Interleaving because of reductions.\n");
4720 bool ScalarInterleavingRequiresPredication =
4722 return Legal->blockNeedsPredication(BB);
4724 bool ScalarInterleavingRequiresRuntimePointerCheck =
4725 (VF.
isScalar() && Legal->getRuntimePointerChecking()->Need);
4730 <<
"LV: IC is " << IC <<
'\n'
4731 <<
"LV: VF is " << VF <<
'\n');
4732 const bool AggressivelyInterleaveReductions =
4733 TTI.enableAggressiveInterleaving(HasReductions);
4734 if (!ScalarInterleavingRequiresRuntimePointerCheck &&
4735 !ScalarInterleavingRequiresPredication && LoopCost <
SmallLoopCost) {
4744 unsigned NumStores = 0;
4745 unsigned NumLoads = 0;
4759 if (
unsigned StoreOps = InterleaveR->getNumStoreOperands())
4760 NumStores += StoreOps;
4762 NumLoads += InterleaveR->getNumDefinedValues();
4777 unsigned StoresIC = IC / (NumStores ? NumStores : 1);
4778 unsigned LoadsIC = IC / (NumLoads ? NumLoads : 1);
4784 bool HasSelectCmpReductions =
4788 auto *RedR = dyn_cast<VPReductionPHIRecipe>(&R);
4789 return RedR && (RecurrenceDescriptor::isAnyOfRecurrenceKind(
4790 RedR->getRecurrenceKind()) ||
4791 RecurrenceDescriptor::isFindIVRecurrenceKind(
4792 RedR->getRecurrenceKind()));
4794 if (HasSelectCmpReductions) {
4795 LLVM_DEBUG(
dbgs() <<
"LV: Not interleaving select-cmp reductions.\n");
4804 if (HasReductions && OrigLoop->getLoopDepth() > 1) {
4805 bool HasOrderedReductions =
4808 auto *RedR = dyn_cast<VPReductionPHIRecipe>(&R);
4810 return RedR && RedR->isOrdered();
4812 if (HasOrderedReductions) {
4814 dbgs() <<
"LV: Not interleaving scalar ordered reductions.\n");
4819 SmallIC = std::min(SmallIC,
F);
4820 StoresIC = std::min(StoresIC,
F);
4821 LoadsIC = std::min(LoadsIC,
F);
4825 std::max(StoresIC, LoadsIC) > SmallIC) {
4827 dbgs() <<
"LV: Interleaving to saturate store or load ports.\n");
4828 return std::max(StoresIC, LoadsIC);
4833 if (VF.
isScalar() && AggressivelyInterleaveReductions) {
4837 return std::max(IC / 2, SmallIC);
4840 LLVM_DEBUG(
dbgs() <<
"LV: Interleaving to reduce branch cost.\n");
4846 if (AggressivelyInterleaveReductions) {
4855bool LoopVectorizationCostModel::useEmulatedMaskMemRefHack(
Instruction *
I,
4866 "Expecting a scalar emulated instruction");
4879 if (InstsToScalarize.contains(VF) ||
4880 PredicatedBBsAfterVectorization.contains(VF))
4886 ScalarCostsTy &ScalarCostsVF = InstsToScalarize[VF];
4896 ScalarCostsTy ScalarCosts;
4903 !useEmulatedMaskMemRefHack(&
I, VF) &&
4904 computePredInstDiscount(&
I, ScalarCosts, VF) >= 0) {
4905 for (
const auto &[
I, IC] : ScalarCosts)
4906 ScalarCostsVF.
insert({
I, IC});
4909 for (
const auto &[
I,
Cost] : ScalarCosts) {
4911 if (!CI || !CallWideningDecisions.contains({CI, VF}))
4914 CallWideningDecisions[{CI, VF}].Cost =
Cost;
4918 PredicatedBBsAfterVectorization[VF].insert(BB);
4920 if (Pred->getSingleSuccessor() == BB)
4921 PredicatedBBsAfterVectorization[VF].insert(Pred);
4930 "Instruction marked uniform-after-vectorization will be predicated");
4948 if (!
I->hasOneUse() || PredInst->
getParent() !=
I->getParent() ||
4967 for (
Use &U :
I->operands())
4980 while (!Worklist.
empty()) {
4984 if (ScalarCosts.contains(
I))
5007 ScalarCost +=
TTI.getScalarizationOverhead(
5020 for (Use &U :
I->operands())
5023 "Instruction has non-scalar type");
5024 if (CanBeScalarized(J))
5026 else if (needsExtract(J, VF)) {
5029 ScalarCost +=
TTI.getScalarizationOverhead(
5042 Discount += VectorCost - ScalarCost;
5043 ScalarCosts[
I] = ScalarCost;
5059 ValuesToIgnoreForVF);
5066 for (
Instruction &
I : BB->instructionsWithoutDebug()) {
5079 LLVM_DEBUG(
dbgs() <<
"LV: Found an estimated cost of " <<
C <<
" for VF "
5080 << VF <<
" For instruction: " <<
I <<
'\n');
5108 const Loop *TheLoop) {
5116 auto *SE = PSE.
getSE();
5117 unsigned NumOperands = Gep->getNumOperands();
5118 for (
unsigned Idx = 1; Idx < NumOperands; ++Idx) {
5119 Value *Opd = Gep->getOperand(Idx);
5121 !
Legal->isInductionVariable(Opd))
5130LoopVectorizationCostModel::getMemInstScalarizationCost(
Instruction *
I,
5133 "Scalarization cost of instruction implies vectorization.");
5138 auto *SE =
PSE.getSE();
5164 Cost += getScalarizationOverhead(
I, VF);
5175 Cost +=
TTI.getScalarizationOverhead(
5180 if (useEmulatedMaskMemRefHack(
I, VF))
5190LoopVectorizationCostModel::getConsecutiveMemOpCost(
Instruction *
I,
5196 int ConsecutiveStride =
Legal->isConsecutivePtr(ValTy,
Ptr);
5198 assert((ConsecutiveStride == 1 || ConsecutiveStride == -1) &&
5199 "Stride should be 1 or -1 for consecutive memory access");
5202 if (
Legal->isMaskRequired(
I)) {
5203 Cost +=
TTI.getMaskedMemoryOpCost(
I->getOpcode(), VectorTy, Alignment, AS,
5207 Cost +=
TTI.getMemoryOpCost(
I->getOpcode(), VectorTy, Alignment, AS,
5211 bool Reverse = ConsecutiveStride < 0;
5219LoopVectorizationCostModel::getUniformMemOpCost(
Instruction *
I,
5229 return TTI.getAddressComputationCost(PtrTy,
nullptr,
nullptr,
CostKind) +
5230 TTI.getMemoryOpCost(Instruction::Load, ValTy, Alignment, AS,
5237 bool IsLoopInvariantStoreValue =
Legal->isInvariant(
SI->getValueOperand());
5243 TTI.getAddressComputationCost(PtrTy,
nullptr,
nullptr,
CostKind) +
5244 TTI.getMemoryOpCost(Instruction::Store, ValTy, Alignment, AS,
CostKind);
5245 if (!IsLoopInvariantStoreValue)
5246 Cost +=
TTI.getIndexedVectorInstrCostFromEnd(Instruction::ExtractElement,
5252LoopVectorizationCostModel::getGatherScatterCost(
Instruction *
I,
5263 return TTI.getAddressComputationCost(PtrTy,
nullptr,
nullptr,
CostKind) +
5264 TTI.getGatherScatterOpCost(
I->getOpcode(), VectorTy,
Ptr,
5265 Legal->isMaskRequired(
I), Alignment,
5270LoopVectorizationCostModel::getInterleaveGroupCost(
Instruction *
I,
5273 assert(Group &&
"Fail to get an interleaved access group.");
5280 unsigned InterleaveFactor = Group->getFactor();
5284 SmallVector<unsigned, 4> Indices;
5285 for (
unsigned IF = 0;
IF < InterleaveFactor;
IF++)
5286 if (Group->getMember(IF))
5290 bool UseMaskForGaps =
5294 InsertPos->
getOpcode(), WideVecTy, Group->getFactor(), Indices,
5298 if (Group->isReverse()) {
5301 "Reverse masked interleaved access not supported.");
5302 Cost += Group->getNumMembers() *
5309std::optional<InstructionCost>
5316 return std::nullopt;
5334 return std::nullopt;
5345 Instruction *LastChain = InLoopReductionImmediateChains.lookup(RetI);
5347 return std::nullopt;
5353 ReductionPhi = InLoopReductionImmediateChains.at(ReductionPhi);
5362 BaseCost =
TTI.getMinMaxReductionCost(MinMaxID, VectorTy,
5365 BaseCost =
TTI.getArithmeticReductionCost(
5373 TTI.getArithmeticInstrCost(Instruction::FMul, VectorTy,
CostKind);
5390 if (RedOp && RdxDesc.
getOpcode() == Instruction::Add &&
5396 !
TheLoop->isLoopInvariant(Op0) && !
TheLoop->isLoopInvariant(Op1) &&
5408 TTI.getCastInstrCost(Op0->
getOpcode(), MulType, ExtType,
5411 TTI.getArithmeticInstrCost(Instruction::Mul, MulType,
CostKind);
5413 TTI.getCastInstrCost(RedOp->
getOpcode(), VectorTy, MulType,
5421 RedCost < ExtCost * 2 + MulCost + Ext2Cost + BaseCost)
5422 return I == RetI ? RedCost : 0;
5424 !
TheLoop->isLoopInvariant(RedOp)) {
5433 TTI.getCastInstrCost(RedOp->
getOpcode(), VectorTy, ExtType,
5435 if (RedCost.
isValid() && RedCost < BaseCost + ExtCost)
5436 return I == RetI ? RedCost : 0;
5437 }
else if (RedOp && RdxDesc.
getOpcode() == Instruction::Add &&
5441 !
TheLoop->isLoopInvariant(Op0) && !
TheLoop->isLoopInvariant(Op1)) {
5460 TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy,
CostKind);
5466 if (Op0Ty != LargestOpTy || Op1Ty != LargestOpTy) {
5467 Instruction *ExtraExtOp = (Op0Ty != LargestOpTy) ? Op0 : Op1;
5468 ExtraExtCost =
TTI.getCastInstrCost(
5475 (RedCost + ExtraExtCost) < (ExtCost0 + ExtCost1 + MulCost + BaseCost))
5476 return I == RetI ? RedCost : 0;
5480 TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy,
CostKind);
5486 if (RedCost.
isValid() && RedCost < MulCost + BaseCost)
5487 return I == RetI ? RedCost : 0;
5491 return I == RetI ? std::optional<InstructionCost>(BaseCost) : std::nullopt;
5495LoopVectorizationCostModel::getMemoryInstructionCost(
Instruction *
I,
5506 return TTI.getAddressComputationCost(PtrTy,
nullptr,
nullptr,
CostKind) +
5507 TTI.getMemoryOpCost(
I->getOpcode(), ValTy, Alignment, AS,
CostKind,
5514LoopVectorizationCostModel::getScalarizationOverhead(
Instruction *
I,
5531 Cost +=
TTI.getScalarizationOverhead(
5553 for (
auto *V : filterExtractingOperands(
Ops, VF))
5576 if (
Legal->isUniformMemOp(
I, VF)) {
5577 auto IsLegalToScalarize = [&]() {
5597 return TheLoop->isLoopInvariant(
SI.getValueOperand());
5609 IsLegalToScalarize() ? getUniformMemOpCost(&
I, VF)
5615 if (GatherScatterCost < ScalarizationCost)
5625 int ConsecutiveStride =
Legal->isConsecutivePtr(
5627 assert((ConsecutiveStride == 1 || ConsecutiveStride == -1) &&
5628 "Expected consecutive stride.");
5637 unsigned NumAccesses = 1;
5640 assert(Group &&
"Fail to get an interleaved access group.");
5646 NumAccesses = Group->getNumMembers();
5648 InterleaveCost = getInterleaveGroupCost(&
I, VF);
5653 ? getGatherScatterCost(&
I, VF) * NumAccesses
5657 getMemInstScalarizationCost(&
I, VF) * NumAccesses;
5663 if (InterleaveCost <= GatherScatterCost &&
5664 InterleaveCost < ScalarizationCost) {
5666 Cost = InterleaveCost;
5667 }
else if (GatherScatterCost < ScalarizationCost) {
5669 Cost = GatherScatterCost;
5672 Cost = ScalarizationCost;
5679 for (
unsigned Idx = 0; Idx < Group->getFactor(); ++Idx) {
5680 if (
auto *
I = Group->getMember(Idx)) {
5682 getMemInstScalarizationCost(
I, VF));
5698 if (
TTI.prefersVectorizedAddressing())
5707 if (PtrDef &&
TheLoop->contains(PtrDef) &&
5715 while (!Worklist.
empty()) {
5717 for (
auto &
Op :
I->operands())
5719 if ((InstOp->getParent() ==
I->getParent()) && !
isa<PHINode>(InstOp) &&
5720 AddrDefs.
insert(InstOp).second)
5724 for (
auto *
I : AddrDefs) {
5742 for (
unsigned I = 0;
I < Group->getFactor(); ++
I) {
5757 ForcedScalars[VF].insert(
I);
5764 "Trying to set a vectorization decision for a scalar VF");
5766 auto ForcedScalar = ForcedScalars.find(VF);
5781 for (
auto &ArgOp : CI->
args())
5790 TTI.getCallInstrCost(ScalarFunc, ScalarRetTy, ScalarTys,
CostKind);
5800 "Unexpected valid cost for scalarizing scalable vectors");
5807 if (VF.
isVector() && ((ForcedScalar != ForcedScalars.end() &&
5808 ForcedScalar->second.contains(CI)) ||
5816 bool MaskRequired =
Legal->isMaskRequired(CI);
5819 for (
Type *ScalarTy : ScalarTys)
5828 std::nullopt, *RedCost);
5839 if (Info.Shape.VF != VF)
5843 if (MaskRequired && !Info.isMasked())
5847 bool ParamsOk =
true;
5849 switch (Param.ParamKind) {
5855 if (!
PSE.getSE()->isLoopInvariant(
PSE.getSCEV(ScalarParam),
5892 VectorCost =
TTI.getCallInstrCost(
nullptr, RetTy, Tys,
CostKind);
5903 if (VectorCost <=
Cost) {
5925 return !OpI || !
TheLoop->contains(OpI) ||
5929 [
this](
Value *
Op) { return shouldConsiderInvariant(Op); }));
5941 return InstsToScalarize[VF][
I];
5944 auto ForcedScalar = ForcedScalars.find(VF);
5945 if (VF.
isVector() && ForcedScalar != ForcedScalars.end()) {
5946 auto InstSet = ForcedScalar->second;
5947 if (InstSet.count(
I))
5952 Type *RetTy =
I->getType();
5955 auto *SE =
PSE.getSE();
5959 [[maybe_unused]]
auto HasSingleCopyAfterVectorization =
5964 auto Scalarized = InstsToScalarize.find(VF);
5965 assert(Scalarized != InstsToScalarize.end() &&
5966 "VF not yet analyzed for scalarization profitability");
5967 return !Scalarized->second.count(
I) &&
5969 auto *UI = cast<Instruction>(U);
5970 return !Scalarized->second.count(UI);
5979 assert(
I->getOpcode() == Instruction::GetElementPtr ||
5980 I->getOpcode() == Instruction::PHI ||
5981 (
I->getOpcode() == Instruction::BitCast &&
5982 I->getType()->isPointerTy()) ||
5983 HasSingleCopyAfterVectorization(
I, VF));
5989 !
TTI.getNumberOfParts(VectorTy))
5993 switch (
I->getOpcode()) {
5994 case Instruction::GetElementPtr:
6000 case Instruction::Br: {
6007 bool ScalarPredicatedBB =
false;
6010 (PredicatedBBsAfterVectorization[VF].count(BI->
getSuccessor(0)) ||
6011 PredicatedBBsAfterVectorization[VF].count(BI->
getSuccessor(1))) &&
6013 ScalarPredicatedBB =
true;
6015 if (ScalarPredicatedBB) {
6023 TTI.getScalarizationOverhead(
6031 return TTI.getCFInstrCost(Instruction::Br,
CostKind);
6039 case Instruction::Switch: {
6041 return TTI.getCFInstrCost(Instruction::Switch,
CostKind);
6043 return Switch->getNumCases() *
6044 TTI.getCmpSelInstrCost(
6046 toVectorTy(Switch->getCondition()->getType(), VF),
6050 case Instruction::PHI: {
6067 Type *ResultTy = Phi->getType();
6073 auto *Phi = dyn_cast<PHINode>(U);
6074 if (Phi && Phi->getParent() == TheLoop->getHeader())
6079 auto &ReductionVars =
Legal->getReductionVars();
6080 auto Iter = ReductionVars.find(HeaderUser);
6081 if (Iter != ReductionVars.end() &&
6083 Iter->second.getRecurrenceKind()))
6086 return (Phi->getNumIncomingValues() - 1) *
6087 TTI.getCmpSelInstrCost(
6088 Instruction::Select,
toVectorTy(ResultTy, VF),
6098 Intrinsic::vp_merge,
toVectorTy(Phi->getType(), VF),
6099 {toVectorTy(Type::getInt1Ty(Phi->getContext()), VF)});
6103 return TTI.getCFInstrCost(Instruction::PHI,
CostKind);
6105 case Instruction::UDiv:
6106 case Instruction::SDiv:
6107 case Instruction::URem:
6108 case Instruction::SRem:
6112 ScalarCost : SafeDivisorCost;
6116 case Instruction::Add:
6117 case Instruction::Sub: {
6118 auto Info =
Legal->getHistogramInfo(
I);
6125 if (!RHS || RHS->getZExtValue() != 1)
6127 TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy,
CostKind);
6131 Type *ScalarTy =
I->getType();
6135 {PtrTy, ScalarTy, MaskTy});
6138 return TTI.getIntrinsicInstrCost(ICA,
CostKind) + MulCost +
6139 TTI.getArithmeticInstrCost(
I->getOpcode(), VectorTy,
CostKind);
6143 case Instruction::FAdd:
6144 case Instruction::FSub:
6145 case Instruction::Mul:
6146 case Instruction::FMul:
6147 case Instruction::FDiv:
6148 case Instruction::FRem:
6149 case Instruction::Shl:
6150 case Instruction::LShr:
6151 case Instruction::AShr:
6152 case Instruction::And:
6153 case Instruction::Or:
6154 case Instruction::Xor: {
6158 if (
I->getOpcode() == Instruction::Mul &&
6159 ((
TheLoop->isLoopInvariant(
I->getOperand(0)) &&
6160 PSE.getSCEV(
I->getOperand(0))->isOne()) ||
6161 (
TheLoop->isLoopInvariant(
I->getOperand(1)) &&
6162 PSE.getSCEV(
I->getOperand(1))->isOne())))
6171 Value *Op2 =
I->getOperand(1);
6177 auto Op2Info =
TTI.getOperandInfo(Op2);
6183 return TTI.getArithmeticInstrCost(
6185 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
6188 case Instruction::FNeg: {
6189 return TTI.getArithmeticInstrCost(
6191 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
6192 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
6193 I->getOperand(0),
I);
6195 case Instruction::Select: {
6200 const Value *Op0, *Op1;
6211 return TTI.getArithmeticInstrCost(
6213 VectorTy,
CostKind, {Op1VK, Op1VP}, {Op2VK, Op2VP}, {Op0, Op1},
I);
6216 Type *CondTy =
SI->getCondition()->getType();
6222 Pred = Cmp->getPredicate();
6223 return TTI.getCmpSelInstrCost(
I->getOpcode(), VectorTy, CondTy, Pred,
6224 CostKind, {TTI::OK_AnyValue, TTI::OP_None},
6225 {TTI::OK_AnyValue, TTI::OP_None},
I);
6227 case Instruction::ICmp:
6228 case Instruction::FCmp: {
6229 Type *ValTy =
I->getOperand(0)->getType();
6235 MinBWs[
I] == MinBWs[Op0AsInstruction]) &&
6236 "if both the operand and the compare are marked for "
6237 "truncation, they must have the same bitwidth");
6242 return TTI.getCmpSelInstrCost(
6245 {TTI::OK_AnyValue, TTI::OP_None}, {TTI::OK_AnyValue, TTI::OP_None},
I);
6247 case Instruction::Store:
6248 case Instruction::Load: {
6253 "CM decision should be taken at this point");
6260 return getMemoryInstructionCost(
I, VF);
6262 case Instruction::BitCast:
6263 if (
I->getType()->isPointerTy())
6266 case Instruction::ZExt:
6267 case Instruction::SExt:
6268 case Instruction::FPToUI:
6269 case Instruction::FPToSI:
6270 case Instruction::FPExt:
6271 case Instruction::PtrToInt:
6272 case Instruction::IntToPtr:
6273 case Instruction::SIToFP:
6274 case Instruction::UIToFP:
6275 case Instruction::Trunc:
6276 case Instruction::FPTrunc: {
6280 "Expected a load or a store!");
6306 unsigned Opcode =
I->getOpcode();
6309 if (Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) {
6312 CCH = ComputeCCH(Store);
6315 else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt ||
6316 Opcode == Instruction::FPExt) {
6318 CCH = ComputeCCH(Load);
6326 return TTI.getCastInstrCost(Instruction::Trunc, Trunc->getDestTy(),
6327 Trunc->getSrcTy(), CCH,
CostKind, Trunc);
6334 Type *SrcScalarTy =
I->getOperand(0)->getType();
6346 (
I->getOpcode() == Instruction::ZExt ||
6347 I->getOpcode() == Instruction::SExt))
6351 return TTI.getCastInstrCost(Opcode, VectorTy, SrcVecTy, CCH,
CostKind,
I);
6353 case Instruction::Call:
6355 case Instruction::ExtractValue:
6357 case Instruction::Alloca:
6365 return TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy,
CostKind);
6380 auto IsLiveOutDead = [
this, RequiresScalarEpilogue](
User *U) {
6381 return RequiresScalarEpilogue &&
6394 Legal->isInvariantAddressOfReduction(
SI->getPointerOperand())) {
6396 DeadInvariantStoreOps[
SI->getPointerOperand()].push_back(
6397 SI->getValueOperand());
6406 all_of(
I.users(), [
this, IsLiveOutDead](
User *U) {
6407 return VecValuesToIgnore.contains(U) ||
6408 ValuesToIgnore.contains(U) || IsLiveOutDead(U);
6417 if (Group->getInsertPos() == &
I)
6420 DeadInterleavePointerOps.
push_back(PointerOp);
6426 if (Br->isConditional())
6433 for (
unsigned I = 0;
I != DeadInterleavePointerOps.
size(); ++
I) {
6436 Instruction *UI = cast<Instruction>(U);
6437 return !VecValuesToIgnore.contains(U) &&
6438 (!isAccessInterleaved(UI) ||
6439 getInterleavedAccessGroup(UI)->getInsertPos() == UI);
6446 for (
const auto &[
_,
Ops] : DeadInvariantStoreOps)
6462 for (
unsigned I = 0;
I != DeadOps.
size(); ++
I) {
6474 if ((ThenEmpty && ElseEmpty) ||
6476 ElseBB->
phis().empty()) ||
6478 ThenBB->
phis().empty())) {
6490 return !VecValuesToIgnore.contains(U) &&
6491 !ValuesToIgnore.contains(U) && !IsLiveOutDead(U);
6499 [
this](
User *U) { return ValuesToIgnore.contains(U); }))
6508 for (
const auto &Reduction :
Legal->getReductionVars()) {
6515 for (
const auto &Induction :
Legal->getInductionVars()) {
6524 if (!InLoopReductions.empty())
6527 for (
const auto &Reduction :
Legal->getReductionVars()) {
6528 PHINode *Phi = Reduction.first;
6539 !
TTI.preferInLoopReduction(Kind, Phi->getType()))
6547 bool InLoop = !ReductionOperations.
empty();
6550 InLoopReductions.insert(Phi);
6553 for (
auto *
I : ReductionOperations) {
6554 InLoopReductionImmediateChains[
I] = LastChain;
6558 LLVM_DEBUG(
dbgs() <<
"LV: Using " << (InLoop ?
"inloop" :
"out of loop")
6559 <<
" reduction for phi: " << *Phi <<
"\n");
6572 unsigned WidestType;
6576 TTI.enableScalableVectorization()
6581 unsigned N =
RegSize.getKnownMinValue() / WidestType;
6592 if (!OrigLoop->isInnermost()) {
6602 <<
"overriding computed VF.\n");
6605 }
else if (UserVF.
isScalable() && !TTI.supportsScalableVectors() &&
6607 LLVM_DEBUG(
dbgs() <<
"LV: Not vectorizing. Scalable VF requested, but "
6608 <<
"not supported by the target.\n");
6610 "Scalable vectorization requested but not supported by the target",
6611 "the scalable user-specified vectorization width for outer-loop "
6612 "vectorization cannot be used because the target does not support "
6613 "scalable vectors.",
6614 "ScalableVFUnfeasible", ORE, OrigLoop);
6619 "VF needs to be a power of two");
6621 <<
"VF " << VF <<
" to build VPlans.\n");
6631 return {VF, 0 , 0 };
6635 dbgs() <<
"LV: Not vectorizing. Inner loops aren't supported in the "
6636 "VPlan-native path.\n");
6641 assert(OrigLoop->isInnermost() &&
"Inner loop expected.");
6642 CM.collectValuesToIgnore();
6643 CM.collectElementTypesForWidening();
6650 if (CM.blockNeedsPredicationForAnyReason(OrigLoop->getHeader()) &&
6654 <<
"LV: Invalidate all interleaved groups due to fold-tail by masking "
6655 "which requires masked-interleaved support.\n");
6656 if (CM.InterleaveInfo.invalidateGroups())
6660 CM.invalidateCostModelingDecisions();
6663 if (CM.foldTailByMasking())
6664 Legal->prepareToFoldTailByMasking();
6671 "UserVF ignored because it may be larger than the maximal safe VF",
6672 "InvalidUserVF", ORE, OrigLoop);
6675 "VF needs to be a power of two");
6678 CM.collectInLoopReductions();
6679 if (CM.selectUserVectorizationFactor(UserVF)) {
6681 buildVPlansWithVPRecipes(UserVF, UserVF);
6686 "InvalidCost", ORE, OrigLoop);
6699 CM.collectInLoopReductions();
6700 for (
const auto &VF : VFCandidates) {
6702 CM.collectNonVectorizedAndSetWideningDecisions(VF);
6721 return CM.isUniformAfterVectorization(
I, VF);
6725 return CM.ValuesToIgnore.contains(UI) ||
6726 (IsVector &&
CM.VecValuesToIgnore.contains(UI)) ||
6746 for (
const auto &[
IV, IndDesc] :
Legal->getInductionVars()) {
6750 for (
unsigned I = 0;
I != IVInsts.
size();
I++) {
6751 for (
Value *
Op : IVInsts[
I]->operands()) {
6753 if (
Op ==
IV || !OpI || !OrigLoop->
contains(OpI) || !
Op->hasOneUse())
6759 for (User *U :
IV->users()) {
6772 if (TC == VF && !CM.foldTailByMasking())
6776 for (Instruction *IVInst : IVInsts) {
6781 dbgs() <<
"Cost of " << InductionCost <<
" for VF " << VF
6782 <<
": induction instruction " << *IVInst <<
"\n";
6784 Cost += InductionCost;
6794 CM.TheLoop->getExitingBlocks(Exiting);
6795 SetVector<Instruction *> ExitInstrs;
6797 for (BasicBlock *EB : Exiting) {
6802 ExitInstrs.
insert(CondI);
6806 for (
unsigned I = 0;
I != ExitInstrs.
size(); ++
I) {
6808 if (!OrigLoop->contains(CondI) ||
6813 dbgs() <<
"Cost of " << CondICost <<
" for VF " << VF
6814 <<
": exit condition instruction " << *CondI <<
"\n";
6820 any_of(OpI->users(), [&ExitInstrs,
this](User *U) {
6821 return OrigLoop->contains(cast<Instruction>(U)->getParent()) &&
6822 !ExitInstrs.contains(cast<Instruction>(U));
6834 for (BasicBlock *BB : OrigLoop->blocks()) {
6838 if (BB == OrigLoop->getLoopLatch())
6840 auto BranchCost = CostCtx.
getLegacyCost(BB->getTerminator(), VF);
6847 for (Instruction *ForcedScalar : CM.ForcedScalars[VF]) {
6853 dbgs() <<
"Cost of " << ForcedCost <<
" for VF " << VF
6854 <<
": forced scalar " << *ForcedScalar <<
"\n";
6858 for (
const auto &[Scalarized, ScalarCost] : CM.InstsToScalarize[VF]) {
6863 dbgs() <<
"Cost of " << ScalarCost <<
" for VF " << VF
6864 <<
": profitable to scalarize " << *Scalarized <<
"\n";
6874 VPCostContext CostCtx(CM.TTI, *CM.TLI, Plan, CM, CM.CostKind);
6882 <<
" (Estimated cost per lane: ");
6884 double CostPerLane = double(
Cost.
getValue()) / EstimatedWidth;
6907 return &WidenMem->getIngredient();
6916 auto *IG =
IR->getInterleaveGroup();
6917 unsigned NumMembers = IG->getNumMembers();
6918 for (
unsigned I = 0;
I != NumMembers; ++
I) {
6952 if (RepR->isSingleScalar() &&
6954 RepR->getUnderlyingInstr(), VF))
6957 if (
Instruction *UI = GetInstructionForCost(&R)) {
6962 if (
match(&R,
m_Cmp(Pred, m_VPValue(), m_VPValue())) &&
6974 return any_of(TheLoop->
blocks(), [&SeenInstrs, &CostCtx,
6976 return any_of(*BB, [&SeenInstrs, &CostCtx, TheLoop, BB](Instruction &I) {
6979 if (isa<PHINode>(&I) && BB == TheLoop->getHeader() &&
6980 CostCtx.CM.Legal->isInductionPhi(cast<PHINode>(&I)))
6982 return !SeenInstrs.contains(&I) && !CostCtx.skipCostComputation(&I, true);
6992 VPlan &FirstPlan = *VPlans[0];
6998 ?
"Reciprocal Throughput\n"
7000 ?
"Instruction Latency\n"
7003 ?
"Code Size and Latency\n"
7008 "More than a single plan/VF w/o any plan having scalar VF");
7012 LLVM_DEBUG(
dbgs() <<
"LV: Scalar loop costs: " << ScalarCost <<
".\n");
7017 if (ForceVectorization) {
7024 for (
auto &
P : VPlans) {
7026 P->vectorFactors().end());
7030 return CM.shouldConsiderRegPressureForVF(VF);
7034 for (
unsigned I = 0;
I < VFs.
size();
I++) {
7041 <<
"LV: Not considering vector loop of width " << VF
7042 <<
" because it will not generate any vector instructions.\n");
7048 <<
"LV: Not considering vector loop of width " << VF
7049 <<
" because it would cause replicated blocks to be generated,"
7050 <<
" which isn't allowed when optimizing for size.\n");
7057 if (CM.shouldConsiderRegPressureForVF(VF) &&
7059 LLVM_DEBUG(
dbgs() <<
"LV(REG): Not considering vector loop of width "
7060 << VF <<
" because it uses too many registers\n");
7064 if (isMoreProfitable(CurrentFactor, BestFactor,
P->hasScalarTail()))
7065 BestFactor = CurrentFactor;
7068 if (isMoreProfitable(CurrentFactor, ScalarFactor,
P->hasScalarTail()))
7069 ProfitableVFs.push_back(CurrentFactor);
7085 VPCostContext CostCtx(CM.TTI, *CM.TLI, BestPlan, CM, CM.CostKind);
7086 precomputeCosts(BestPlan, BestFactor.
Width, CostCtx);
7093 BestFactor.
Width) ||
7096 " VPlan cost model and legacy cost model disagreed");
7098 "when vectorizing, the scalar cost must be computed.");
7108 "RdxResult must be ComputeFindIVResult");
7126 if (!EpiRedResult ||
7132 auto *EpiRedHeaderPhi =
7134 RecurKind Kind = EpiRedHeaderPhi->getRecurrenceKind();
7135 Value *MainResumeValue;
7139 "unexpected start recipe");
7140 MainResumeValue = VPI->getOperand(0)->getUnderlyingValue();
7142 MainResumeValue = EpiRedHeaderPhi->getStartValue()->getUnderlyingValue();
7144 [[maybe_unused]]
Value *StartV =
7145 EpiRedResult->getOperand(1)->getLiveInIRValue();
7148 "AnyOf expected to start with ICMP_NE");
7149 assert(Cmp->getOperand(1) == StartV &&
7150 "AnyOf expected to start by comparing main resume value to original "
7152 MainResumeValue = Cmp->getOperand(0);
7155 Value *SentinelV = EpiRedResult->getOperand(2)->getLiveInIRValue();
7157 Value *Cmp, *OrigResumeV, *CmpOp;
7158 [[maybe_unused]]
bool IsExpectedPattern =
7159 match(MainResumeValue,
7165 assert(IsExpectedPattern &&
"Unexpected reduction resume pattern");
7166 MainResumeValue = OrigResumeV;
7181 "Trying to execute plan with unsupported VF");
7183 "Trying to execute plan with unsupported UF");
7185 ++LoopsEarlyExitVectorized;
7192 bool HasBranchWeights =
7194 if (HasBranchWeights) {
7195 std::optional<unsigned> VScale = CM.getVScaleForTuning();
7197 BestVPlan, BestVF, VScale);
7202 attachRuntimeChecks(BestVPlan, ILV.
RTChecks, HasBranchWeights);
7215 OrigLoop->getStartLoc(),
7216 OrigLoop->getHeader())
7217 <<
"Created vector loop never executes due to insufficient trip "
7237 BestVPlan, VectorPH, CM.foldTailByMasking(),
7238 CM.requiresScalarEpilogue(BestVF.
isVector()));
7249 assert(VectorizingEpilogue &&
"should only re-use the existing trip "
7250 "count during epilogue vectorization");
7254 OrigLoop->getParentLoop(),
7255 Legal->getWidestInductionType());
7257#ifdef EXPENSIVE_CHECKS
7258 assert(DT->verify(DominatorTree::VerificationLevel::Fast));
7269 "final VPlan is invalid");
7276 if (!Exit->hasPredecessors())
7298 MDNode *LID = OrigLoop->getLoopID();
7299 unsigned OrigLoopInvocationWeight = 0;
7300 std::optional<unsigned> OrigAverageTripCount =
7312 bool DisableRuntimeUnroll = !ILV.
RTChecks.hasChecks() && !BestVF.
isScalar();
7314 HeaderVPBB ? LI->getLoopFor(State.CFG.VPBB2IRBB.lookup(HeaderVPBB))
7316 HeaderVPBB, BestVPlan, VectorizingEpilogue, LID, OrigAverageTripCount,
7317 OrigLoopInvocationWeight,
7319 DisableRuntimeUnroll);
7327 return ExpandedSCEVs;
7342 EPI.EpilogueIterationCountCheck =
7344 EPI.EpilogueIterationCountCheck->setName(
"iter.check");
7354 EPI.MainLoopIterationCountCheck =
7363 dbgs() <<
"Create Skeleton for epilogue vectorized loop (first pass)\n"
7364 <<
"Main Loop VF:" <<
EPI.MainLoopVF
7365 <<
", Main Loop UF:" <<
EPI.MainLoopUF
7366 <<
", Epilogue Loop VF:" <<
EPI.EpilogueVF
7367 <<
", Epilogue Loop UF:" <<
EPI.EpilogueUF <<
"\n";
7373 dbgs() <<
"intermediate fn:\n"
7374 << *
OrigLoop->getHeader()->getParent() <<
"\n";
7380 assert(Bypass &&
"Expected valid bypass basic block.");
7384 VectorPH, ForEpilogue ?
EPI.EpilogueVF :
EPI.MainLoopVF,
7385 ForEpilogue ?
EPI.EpilogueUF :
EPI.MainLoopUF);
7389 TCCheckBlock->
setName(
"vector.main.loop.iter.check");
7415 return TCCheckBlock;
7429 VectorPH->
setName(
"vec.epilog.ph");
7432 "vec.epilog.iter.check",
true);
7436 VecEpilogueIterationCountCheck);
7437 AdditionalBypassBlock = VecEpilogueIterationCountCheck;
7441 assert(
EPI.MainLoopIterationCountCheck &&
EPI.EpilogueIterationCountCheck &&
7442 "expected this to be saved from the previous pass.");
7443 EPI.MainLoopIterationCountCheck->getTerminator()->replaceUsesOfWith(
7444 VecEpilogueIterationCountCheck, VectorPH);
7446 EPI.EpilogueIterationCountCheck->getTerminator()->replaceUsesOfWith(
7447 VecEpilogueIterationCountCheck, ScalarPH);
7454 VecEpilogueIterationCountCheck, ScalarPH);
7457 VecEpilogueIterationCountCheck, ScalarPH);
7459 DT->changeImmediateDominator(ScalarPH,
EPI.EpilogueIterationCountCheck);
7467 for (
PHINode *Phi : PhisInBlock) {
7469 Phi->replaceIncomingBlockWith(
7471 VecEpilogueIterationCountCheck);
7478 return EPI.EpilogueIterationCountCheck == IncB;
7481 Phi->removeIncomingValue(
EPI.EpilogueIterationCountCheck);
7483 Phi->removeIncomingValue(SCEVCheckBlock);
7485 Phi->removeIncomingValue(MemCheckBlock);
7496 "Expected trip count to have been saved in the first pass.");
7503 auto P =
Cost->requiresScalarEpilogue(
EPI.EpilogueVF.isVector())
7507 Value *CheckMinIters =
7510 EPI.EpilogueVF,
EPI.EpilogueUF),
7511 "min.epilog.iters.check");
7514 auto VScale =
Cost->getVScaleForTuning();
7515 unsigned MainLoopStep =
7517 unsigned EpilogueLoopStep =
7525 unsigned EstimatedSkipCount = std::min(MainLoopStep, EpilogueLoopStep);
7526 const uint32_t Weights[] = {EstimatedSkipCount,
7527 MainLoopStep - EstimatedSkipCount};
7536 Plan.setEntry(NewEntry);
7544 dbgs() <<
"Create Skeleton for epilogue vectorized loop (second pass)\n"
7545 <<
"Epilogue Loop VF:" <<
EPI.EpilogueVF
7546 <<
", Epilogue Loop UF:" <<
EPI.EpilogueUF <<
"\n";
7552 dbgs() <<
"final fn:\n" << *
OrigLoop->getHeader()->getParent() <<
"\n";
7560 "Must be called with either a load or store");
7566 "CM decision should be taken at this point.");
7579 if (
Legal->isMaskRequired(
I))
7593 Ptr->getUnderlyingValue()->stripPointerCasts());
7605 -1, Flags,
I->getDebugLoc());
7608 GEP ?
GEP->getNoWrapFlags()
7612 Builder.insert(VectorPtr);
7616 return new VPWidenLoadRecipe(*Load,
Ptr, Mask, Consecutive,
Reverse,
7617 VPIRMetadata(*Load, LVer),
I->getDebugLoc());
7620 return new VPWidenStoreRecipe(*Store,
Ptr,
Operands[0], Mask, Consecutive,
7621 Reverse, VPIRMetadata(*Store, LVer),
7634 "step must be loop invariant");
7641 TruncI->getDebugLoc());
7645 IndDesc, Phi->getDebugLoc());
7653 if (
auto *
II = Legal->getIntOrFpInductionDescriptor(Phi))
7655 *PSE.getSE(), *OrigLoop);
7658 if (
auto *
II = Legal->getPointerInductionDescriptor(Phi)) {
7660 return new VPWidenPointerInductionRecipe(
7661 Phi,
Operands[0], Step, &Plan.getVFxUF(), *
II,
7663 [&](ElementCount VF) {
7664 return CM.isScalarAfterVectorization(Phi, VF);
7667 Phi->getDebugLoc());
7681 auto IsOptimizableIVTruncate =
7682 [&](
Instruction *
K) -> std::function<
bool(ElementCount)> {
7683 return [=](ElementCount VF) ->
bool {
7684 return CM.isOptimizableIVTruncate(K, VF);
7689 IsOptimizableIVTruncate(
I),
Range)) {
7692 const InductionDescriptor &
II = *Legal->getIntOrFpInductionDescriptor(Phi);
7693 VPValue *
Start = Plan.getOrAddLiveIn(
II.getStartValue());
7704 [
this, CI](ElementCount VF) {
7705 return CM.isScalarWithPredication(CI, VF);
7713 if (
ID && (
ID == Intrinsic::assume ||
ID == Intrinsic::lifetime_end ||
7714 ID == Intrinsic::lifetime_start ||
ID == Intrinsic::sideeffect ||
7715 ID == Intrinsic::pseudoprobe ||
7716 ID == Intrinsic::experimental_noalias_scope_decl))
7722 bool ShouldUseVectorIntrinsic =
7724 [&](ElementCount VF) ->
bool {
7725 return CM.getCallWideningDecision(CI, VF).Kind ==
7729 if (ShouldUseVectorIntrinsic)
7730 return new VPWidenIntrinsicRecipe(*CI,
ID,
Ops, CI->
getType(),
7734 std::optional<unsigned> MaskPos;
7738 [&](ElementCount VF) ->
bool {
7753 LoopVectorizationCostModel::CallWideningDecision Decision =
7754 CM.getCallWideningDecision(CI, VF);
7764 if (ShouldUseVectorCall) {
7765 if (MaskPos.has_value()) {
7773 VPValue *
Mask =
nullptr;
7774 if (Legal->isMaskRequired(CI))
7777 Mask = Plan.getOrAddLiveIn(
7780 Ops.insert(
Ops.begin() + *MaskPos, Mask);
7784 return new VPWidenCallRecipe(CI, Variant,
Ops, CI->
getDebugLoc());
7792 !
isa<StoreInst>(
I) &&
"Instruction should have been handled earlier");
7795 auto WillScalarize = [
this,
I](ElementCount VF) ->
bool {
7796 return CM.isScalarAfterVectorization(
I, VF) ||
7797 CM.isProfitableToScalarize(
I, VF) ||
7798 CM.isScalarWithPredication(
I, VF);
7806 switch (
I->getOpcode()) {
7809 case Instruction::SDiv:
7810 case Instruction::UDiv:
7811 case Instruction::SRem:
7812 case Instruction::URem: {
7815 if (CM.isPredicatedInst(
I)) {
7819 Plan.getOrAddLiveIn(ConstantInt::get(
I->getType(), 1u,
false));
7820 auto *SafeRHS = Builder.createSelect(Mask,
Ops[1], One,
I->getDebugLoc());
7822 return new VPWidenRecipe(*
I,
Ops);
7826 case Instruction::Add:
7827 case Instruction::And:
7828 case Instruction::AShr:
7829 case Instruction::FAdd:
7830 case Instruction::FCmp:
7831 case Instruction::FDiv:
7832 case Instruction::FMul:
7833 case Instruction::FNeg:
7834 case Instruction::FRem:
7835 case Instruction::FSub:
7836 case Instruction::ICmp:
7837 case Instruction::LShr:
7838 case Instruction::Mul:
7839 case Instruction::Or:
7840 case Instruction::Select:
7841 case Instruction::Shl:
7842 case Instruction::Sub:
7843 case Instruction::Xor:
7844 case Instruction::Freeze: {
7850 ScalarEvolution &SE = *PSE.getSE();
7851 auto GetConstantViaSCEV = [
this, &SE](VPValue *
Op) {
7852 if (!
Op->isLiveIn())
7854 Value *
V =
Op->getUnderlyingValue();
7860 return Plan.getOrAddLiveIn(
C->getValue());
7863 if (
I->getOpcode() == Instruction::Mul)
7864 NewOps[0] = GetConstantViaSCEV(NewOps[0]);
7866 NewOps[1] = GetConstantViaSCEV(NewOps[1]);
7868 return new VPWidenRecipe(*
I, NewOps);
7870 case Instruction::ExtractValue: {
7874 assert(EVI->getNumIndices() == 1 &&
"Expected one extractvalue index");
7875 unsigned Idx = EVI->getIndices()[0];
7876 NewOps.push_back(Plan.getOrAddLiveIn(ConstantInt::get(I32Ty, Idx,
false)));
7877 return new VPWidenRecipe(*
I, NewOps);
7883VPRecipeBuilder::tryToWidenHistogram(
const HistogramInfo *HI,
7886 unsigned Opcode =
HI->Update->getOpcode();
7887 assert((Opcode == Instruction::Add || Opcode == Instruction::Sub) &&
7888 "Histogram update operation must be an Add or Sub");
7898 if (Legal->isMaskRequired(
HI->Store))
7901 return new VPHistogramRecipe(Opcode, HGramOps,
HI->Store->getDebugLoc());
7908 [&](
ElementCount VF) {
return CM.isUniformAfterVectorization(
I, VF); },
7911 bool IsPredicated = CM.isPredicatedInst(
I);
7919 case Intrinsic::assume:
7920 case Intrinsic::lifetime_start:
7921 case Intrinsic::lifetime_end:
7943 VPValue *BlockInMask =
nullptr;
7944 if (!IsPredicated) {
7948 LLVM_DEBUG(
dbgs() <<
"LV: Scalarizing and predicating:" << *
I <<
"\n");
7959 assert((
Range.Start.isScalar() || !IsUniform || !IsPredicated ||
7961 "Should not predicate a uniform recipe");
7972 PartialReductionChains;
7973 for (
const auto &[Phi, RdxDesc] : Legal->getReductionVars()) {
7974 getScaledReductions(Phi, RdxDesc.getLoopExitInstr(),
Range,
7975 PartialReductionChains);
7984 for (
const auto &[PartialRdx,
_] : PartialReductionChains)
7985 PartialReductionOps.
insert(PartialRdx.ExtendUser);
7987 auto ExtendIsOnlyUsedByPartialReductions =
7989 return all_of(Extend->users(), [&](
const User *U) {
7990 return PartialReductionOps.contains(U);
7996 for (
auto Pair : PartialReductionChains) {
7998 if (ExtendIsOnlyUsedByPartialReductions(Chain.
ExtendA) &&
7999 (!Chain.
ExtendB || ExtendIsOnlyUsedByPartialReductions(Chain.
ExtendB)))
8000 ScaledReductionMap.try_emplace(Chain.
Reduction, Pair.second);
8004bool VPRecipeBuilder::getScaledReductions(
8006 SmallVectorImpl<std::pair<PartialReductionChain, unsigned>> &Chains) {
8014 Value *
Op = Update->getOperand(0);
8015 Value *PhiOp = Update->getOperand(1);
8023 if (getScaledReductions(
PHI, OpInst,
Range, Chains)) {
8024 PHI = Chains.rbegin()->first.Reduction;
8026 Op = Update->getOperand(0);
8027 PhiOp = Update->getOperand(1);
8035 using namespace llvm::PatternMatch;
8042 std::optional<unsigned> BinOpc;
8043 Type *ExtOpTypes[2] = {
nullptr};
8045 auto CollectExtInfo = [
this, &Exts,
8046 &ExtOpTypes](SmallVectorImpl<Value *> &
Ops) ->
bool {
8055 if (!CM.TheLoop->contains(Exts[
I]))
8073 if (!CollectExtInfo(
Ops))
8076 BinOpc = std::make_optional(ExtendUser->
getOpcode());
8080 if (!CollectExtInfo(
Ops))
8083 ExtendUser = Update;
8084 BinOpc = std::nullopt;
8092 PartialReductionChain Chain(RdxExitInstr, Exts[0], Exts[1], ExtendUser);
8094 TypeSize PHISize =
PHI->getType()->getPrimitiveSizeInBits();
8101 [&](ElementCount VF) {
8103 Update->getOpcode(), ExtOpTypes[0], ExtOpTypes[1],
8104 PHI->getType(), VF, OpAExtend, OpBExtend, BinOpc, CM.CostKind);
8108 Chains.emplace_back(Chain, TargetScaleFactor);
8127 "Non-header phis should have been handled during predication");
8129 assert(
Operands.size() == 2 &&
"Must have 2 operands for header phis");
8130 if ((Recipe = tryToOptimizeInductionPHI(Phi,
Operands,
Range)))
8134 assert((Legal->isReductionVariable(Phi) ||
8135 Legal->isFixedOrderRecurrence(Phi)) &&
8136 "can only widen reductions and fixed-order recurrences here");
8138 if (Legal->isReductionVariable(Phi)) {
8141 Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader()));
8144 unsigned ScaleFactor =
8148 CM.useOrderedReductions(RdxDesc), ScaleFactor);
8160 assert(!R->isPhi() &&
"only VPPhi nodes expected at this point");
8162 if (
isa<TruncInst>(Instr) && (Recipe = tryToOptimizeInductionTruncate(
8175 if (
auto HistInfo = Legal->getHistogramInfo(
SI))
8176 return tryToWidenHistogram(*HistInfo,
Operands);
8184 if (!shouldWiden(Instr,
Range))
8199 return tryToWiden(Instr,
Operands);
8205 unsigned ScaleFactor) {
8207 "Unexpected number of operands for partial reduction");
8216 unsigned ReductionOpcode = Reduction->getOpcode();
8217 if (ReductionOpcode == Instruction::Sub) {
8218 auto *
const Zero = ConstantInt::get(Reduction->getType(), 0);
8220 Ops.push_back(Plan.getOrAddLiveIn(Zero));
8221 Ops.push_back(BinOp);
8224 ReductionOpcode = Instruction::Add;
8228 if (CM.blockNeedsPredicationForAnyReason(Reduction->getParent())) {
8229 assert((ReductionOpcode == Instruction::Add ||
8230 ReductionOpcode == Instruction::Sub) &&
8231 "Expected an ADD or SUB operation for predicated partial "
8232 "reductions (because the neutral element in the mask is zero)!");
8235 Plan.getOrAddLiveIn(ConstantInt::get(Reduction->getType(), 0));
8236 BinOp = Builder.createSelect(
Cond, BinOp, Zero, Reduction->getDebugLoc());
8239 ScaleFactor, Reduction);
8242void LoopVectorizationPlanner::buildVPlansWithVPRecipes(
ElementCount MinVF,
8251 OrigLoop, LI, DT, PSE.
getSE());
8256 LVer.prepareNoAliasMetadata();
8262 OrigLoop, *LI,
Legal->getWidestInductionType(),
8265 auto MaxVFTimes2 = MaxVF * 2;
8267 VFRange SubRange = {VF, MaxVFTimes2};
8268 if (
auto Plan = tryToBuildVPlanWithVPRecipes(
8269 std::unique_ptr<VPlan>(VPlan0->duplicate()), SubRange, &LVer)) {
8274 *Plan, CM.getMinimalBitwidths());
8277 if (CM.foldTailWithEVL() && !HasScalarVF)
8279 *Plan, CM.getMaxSafeElements());
8281 VPlans.push_back(std::move(Plan));
8296 if (WideIntOrFp && WideIntOrFp->getTruncInst())
8303 if (!WideIntOrFp || !WideIntOrFp->isCanonical()) {
8306 Start, VectorTC, Step);
8319 {EndValue, Start}, WideIV->
getDebugLoc(),
"bc.resume.val");
8320 return ResumePhiRecipe;
8335 VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
8346 WideIVR, VectorPHBuilder, ScalarPHBuilder, TypeInfo,
8349 IVEndValues[WideIVR] = ResumePhi->getOperand(0);
8350 ScalarPhiIRI->addOperand(ResumePhi);
8357 "should only skip truncated wide inductions");
8365 auto *ResumeFromVectorLoop = VectorPhiR->getBackedgeValue();
8367 "Cannot handle loops with uncountable early exits");
8371 "vector.recur.extract");
8372 StringRef Name = IsFOR ?
"scalar.recur.init" :
"bc.merge.rdx";
8374 {ResumeFromVectorLoop, VectorPhiR->getStartValue()}, {}, Name);
8387 VPBuilder ScalarPHBuilder(ScalarPHVPBB);
8388 VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
8400 "Cannot handle loops with uncountable early exits");
8472 for (
VPUser *U : FOR->users()) {
8486 {},
"vector.recur.extract.for.phi");
8492VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
8495 using namespace llvm::VPlanPatternMatch;
8496 SmallPtrSet<const InterleaveGroup<Instruction> *, 1> InterleaveGroups;
8503 bool RequiresScalarEpilogueCheck =
8505 [
this](ElementCount VF) {
8506 return !CM.requiresScalarEpilogue(VF.
isVector());
8511 CM.foldTailByMasking());
8519 bool IVUpdateMayOverflow =
false;
8520 for (ElementCount VF :
Range)
8530 auto *IVInc = Plan->getVectorLoopRegion()
8531 ->getExitingBasicBlock()
8534 assert(
match(IVInc, m_VPInstruction<Instruction::Add>(
8536 "Did not find the canonical IV increment");
8549 for (InterleaveGroup<Instruction> *IG : IAI.getInterleaveGroups()) {
8550 auto ApplyIG = [IG,
this](ElementCount VF) ->
bool {
8552 CM.getWideningDecision(IG->getInsertPos(), VF) ==
8557 "Unsupported interleave factor for scalable vectors");
8562 InterleaveGroups.
insert(IG);
8569 *Plan, CM.foldTailByMasking());
8575 VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
8576 Builder, BlockMaskCache, LVer);
8577 RecipeBuilder.collectScaledReductions(
Range);
8581 VPRegionBlock *LoopRegion = Plan->getVectorLoopRegion();
8583 ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
8586 auto *MiddleVPBB = Plan->getMiddleBlock();
8590 DenseMap<VPValue *, VPValue *> Old2New;
8595 auto *UnderlyingValue = SingleDef->getUnderlyingValue();
8609 UnderlyingValue &&
"unsupported recipe");
8614 Builder.setInsertPoint(SingleDef);
8621 Legal->isInvariantAddressOfReduction(
SI->getPointerOperand())) {
8623 if (Legal->isInvariantStoreOfReduction(SI)) {
8625 new VPReplicateRecipe(SI,
R.operands(),
true ,
8626 nullptr , VPIRMetadata(*SI, LVer));
8627 Recipe->insertBefore(*MiddleVPBB, MBIP);
8629 R.eraseFromParent();
8633 VPRecipeBase *Recipe =
8634 RecipeBuilder.tryToCreateWidenRecipe(SingleDef,
Range);
8636 Recipe = RecipeBuilder.handleReplication(Instr,
R.operands(),
Range);
8638 RecipeBuilder.setRecipe(Instr, Recipe);
8644 Builder.insert(Recipe);
8651 "Unexpected multidef recipe");
8652 R.eraseFromParent();
8661 RecipeBuilder.updateBlockMaskCache(Old2New);
8662 for (VPValue *Old : Old2New.
keys())
8663 Old->getDefiningRecipe()->eraseFromParent();
8666 !Plan->getVectorLoopRegion()->getEntryBasicBlock()->empty() &&
8667 "entry block must be set to a VPRegionBlock having a non-empty entry "
8673 for (
const auto &[Phi,
ID] : Legal->getInductionVars()) {
8675 Phi->getIncomingValueForBlock(OrigLoop->getLoopLatch()));
8678 VPWidenInductionRecipe *WideIV =
8680 VPRecipeBase *
R = RecipeBuilder.getRecipe(IVInc);
8685 DenseMap<VPValue *, VPValue *> IVEndValues;
8694 adjustRecipesForReductions(Plan, RecipeBuilder,
Range.Start);
8706 if (!CM.foldTailWithEVL()) {
8707 VPCostContext CostCtx(CM.TTI, *CM.TLI, *Plan, CM, CM.CostKind);
8712 for (ElementCount VF :
Range)
8714 Plan->setName(
"Initial VPlan");
8720 InterleaveGroups, RecipeBuilder,
8721 CM.isScalarEpilogueAllowed());
8725 Legal->getLAI()->getSymbolicStrides());
8727 auto BlockNeedsPredication = [
this](
BasicBlock *BB) {
8728 return Legal->blockNeedsPredication(BB);
8731 BlockNeedsPredication);
8743 bool WithoutRuntimeCheck =
8746 WithoutRuntimeCheck);
8759 assert(!OrigLoop->isInnermost());
8763 OrigLoop, *LI, Legal->getWidestInductionType(),
8772 for (ElementCount VF :
Range)
8777 [
this](PHINode *
P) {
8778 return Legal->getIntOrFpInductionDescriptor(
P);
8785 DenseMap<VPBasicBlock *, VPValue *> BlockMaskCache;
8786 VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
8787 Builder, BlockMaskCache,
nullptr );
8788 for (
auto &R : Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
8792 RecipeBuilder.setRecipe(HeaderR->getUnderlyingInstr(), HeaderR);
8794 DenseMap<VPValue *, VPValue *> IVEndValues;
8816void LoopVectorizationPlanner::adjustRecipesForReductions(
8818 using namespace VPlanPatternMatch;
8819 VPRegionBlock *VectorLoopRegion = Plan->getVectorLoopRegion();
8821 VPBasicBlock *MiddleVPBB = Plan->getMiddleBlock();
8824 for (VPRecipeBase &R : Header->phis()) {
8826 if (!PhiR || !PhiR->isInLoop() || (MinVF.
isScalar() && !PhiR->isOrdered()))
8833 "AnyOf and FindIV reductions are not allowed for in-loop reductions");
8836 SetVector<VPSingleDefRecipe *> Worklist;
8838 for (
unsigned I = 0;
I != Worklist.
size(); ++
I) {
8839 VPSingleDefRecipe *Cur = Worklist[
I];
8840 for (VPUser *U : Cur->
users()) {
8842 if (!UserRecipe->getParent()->getEnclosingLoopRegion()) {
8843 assert((UserRecipe->getParent() == MiddleVPBB ||
8844 UserRecipe->getParent() == Plan->getScalarPreheader()) &&
8845 "U must be either in the loop region, the middle block or the "
8846 "scalar preheader.");
8849 Worklist.
insert(UserRecipe);
8860 VPSingleDefRecipe *PreviousLink = PhiR;
8861 for (VPSingleDefRecipe *CurrentLink :
drop_begin(Worklist)) {
8863 assert(Blend->getNumIncomingValues() == 2 &&
8864 "Blend must have 2 incoming values");
8865 if (Blend->getIncomingValue(0) == PhiR) {
8866 Blend->replaceAllUsesWith(Blend->getIncomingValue(1));
8868 assert(Blend->getIncomingValue(1) == PhiR &&
8869 "PhiR must be an operand of the blend");
8870 Blend->replaceAllUsesWith(Blend->getIncomingValue(0));
8875 Instruction *CurrentLinkI = CurrentLink->getUnderlyingInstr();
8878 unsigned IndexOfFirstOperand;
8882 VPBasicBlock *LinkVPBB = CurrentLink->getParent();
8886 "Expected instruction to be a call to the llvm.fmuladd intrinsic");
8889 CurrentLink->getOperand(2) == PreviousLink &&
8890 "expected a call where the previous link is the added operand");
8896 VPInstruction *FMulRecipe =
new VPInstruction(
8898 {CurrentLink->getOperand(0), CurrentLink->getOperand(1)},
8900 LinkVPBB->
insert(FMulRecipe, CurrentLink->getIterator());
8903 CurrentLinkI->
getOpcode() == Instruction::Sub) {
8904 Type *PhiTy = PhiR->getUnderlyingValue()->getType();
8905 auto *
Zero = Plan->getOrAddLiveIn(ConstantInt::get(PhiTy, 0));
8906 VPWidenRecipe *
Sub =
new VPWidenRecipe(
8907 Instruction::Sub, {
Zero, CurrentLink->getOperand(1)}, {},
8909 Sub->setUnderlyingValue(CurrentLinkI);
8910 LinkVPBB->
insert(
Sub, CurrentLink->getIterator());
8916 "need to have the compare of the select");
8920 "must be a select recipe");
8921 IndexOfFirstOperand = 1;
8924 "Expected to replace a VPWidenSC");
8925 IndexOfFirstOperand = 0;
8930 CurrentLink->getOperand(IndexOfFirstOperand) == PreviousLink
8931 ? IndexOfFirstOperand + 1
8932 : IndexOfFirstOperand;
8933 VecOp = CurrentLink->getOperand(VecOpId);
8934 assert(VecOp != PreviousLink &&
8935 CurrentLink->getOperand(CurrentLink->getNumOperands() - 1 -
8936 (VecOpId - IndexOfFirstOperand)) ==
8938 "PreviousLink must be the operand other than VecOp");
8941 VPValue *CondOp =
nullptr;
8942 if (CM.blockNeedsPredicationForAnyReason(CurrentLinkI->
getParent()))
8946 RecurrenceDescriptor RdxDesc = Legal->getRecurrenceDescriptor(
8952 auto *RedRecipe =
new VPReductionRecipe(
8953 Kind, FMFs, CurrentLinkI, PreviousLink, VecOp, CondOp,
8960 RedRecipe->insertBefore(&*std::prev(std::prev(LinkVPBB->
end())));
8964 CurrentLink->replaceAllUsesWith(RedRecipe);
8966 PreviousLink = RedRecipe;
8970 Builder.setInsertPoint(&*std::prev(std::prev(LatchVPBB->
end())));
8972 for (VPRecipeBase &R :
8973 Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
8978 const RecurrenceDescriptor &RdxDesc = Legal->getRecurrenceDescriptor(
8989 if (!PhiR->
isInLoop() && CM.foldTailByMasking() &&
8992 std::optional<FastMathFlags> FMFs =
8997 Builder.createSelect(
Cond, OrigExitingVPV, PhiR, {},
"", FMFs);
8998 OrigExitingVPV->replaceUsesWithIf(NewExitingVPV, [](VPUser &U,
unsigned) {
9007 if (CM.usePredicatedReductionSelect())
9018 DebugLoc ExitDL = OrigLoop->getLoopLatch()->getTerminator()->getDebugLoc();
9024 VPInstruction *FinalReductionResult;
9025 VPBuilder::InsertPointGuard Guard(Builder);
9026 Builder.setInsertPoint(MiddleVPBB, IP);
9031 FinalReductionResult =
9036 FinalReductionResult =
9038 {PhiR,
Start, NewExitingVPV}, ExitDL);
9044 FinalReductionResult =
9046 {PhiR, NewExitingVPV},
Flags, ExitDL);
9053 assert(!PhiR->
isInLoop() &&
"Unexpected truncated inloop reduction!");
9055 "Unexpected truncated min-max recurrence!");
9058 new VPWidenCastRecipe(Instruction::Trunc, NewExitingVPV, RdxTy);
9060 RdxDesc.
isSigned() ? Instruction::SExt : Instruction::ZExt;
9061 auto *Extnd =
new VPWidenCastRecipe(ExtendOpc, Trunc, PhiTy);
9062 Trunc->insertAfter(NewExitingVPV->getDefiningRecipe());
9063 Extnd->insertAfter(Trunc);
9065 PhiR->
setOperand(1, Extnd->getVPSingleValue());
9070 FinalReductionResult =
9071 Builder.createScalarCast(ExtendOpc, FinalReductionResult, PhiTy, {});
9076 for (
auto *U :
to_vector(OrigExitingVPV->users())) {
9078 if (FinalReductionResult == U || Parent->getParent())
9080 U->replaceUsesOfWith(OrigExitingVPV, FinalReductionResult);
9091 return isa<VPWidenSelectRecipe>(U) ||
9092 (isa<VPReplicateRecipe>(U) &&
9093 cast<VPReplicateRecipe>(U)->getUnderlyingInstr()->getOpcode() ==
9094 Instruction::Select);
9099 if (VPRecipeBase *CmpR =
Cmp->getDefiningRecipe())
9101 Builder.setInsertPoint(
Select);
9105 if (
Select->getOperand(1) == PhiR)
9106 Cmp = Builder.createNot(Cmp);
9107 VPValue *
Or = Builder.createOr(PhiR, Cmp);
9108 Select->getVPSingleValue()->replaceAllUsesWith(
Or);
9114 OrigLoop->getHeader()->getContext())));
9129 VPBuilder PHBuilder(Plan->getVectorPreheader());
9130 VPValue *Iden = Plan->getOrAddLiveIn(
9133 unsigned ScaleFactor =
9137 auto *ScaleFactorVPV =
9138 Plan->getOrAddLiveIn(ConstantInt::get(I32Ty, ScaleFactor));
9139 VPValue *StartV = PHBuilder.createNaryOp(
9147 for (VPRecipeBase *R : ToDelete)
9148 R->eraseFromParent();
9153void LoopVectorizationPlanner::attachRuntimeChecks(
9154 VPlan &Plan, GeneratedRTChecks &RTChecks,
bool HasBranchWeights)
const {
9155 const auto &[SCEVCheckCond, SCEVCheckBlock] = RTChecks.getSCEVChecks();
9156 if (SCEVCheckBlock && SCEVCheckBlock->hasNPredecessors(0)) {
9157 assert((!CM.OptForSize ||
9159 "Cannot SCEV check stride or overflow when optimizing for size");
9163 const auto &[MemCheckCond, MemCheckBlock] = RTChecks.getMemRuntimeChecks();
9164 if (MemCheckBlock && MemCheckBlock->hasNPredecessors(0)) {
9168 "Runtime checks are not supported for outer loops yet");
9170 if (CM.OptForSize) {
9173 "Cannot emit memory checks when optimizing for size, unless forced "
9176 return OptimizationRemarkAnalysis(
DEBUG_TYPE,
"VectorizationCodeSize",
9177 OrigLoop->getStartLoc(),
9178 OrigLoop->getHeader())
9179 <<
"Code-size may be reduced by not forcing "
9180 "vectorization, or by source-code modifications "
9181 "eliminating the need for runtime checks "
9182 "(e.g., adding 'restrict').";
9196 bool IsIndvarOverflowCheckNeededForVF =
9197 VF.
isScalable() && !TTI.isVScaleKnownToBeAPowerOfTwo() &&
9199 CM.getTailFoldingStyle() !=
9206 Plan, VF, UF, MinProfitableTripCount,
9207 CM.requiresScalarEpilogue(VF.
isVector()), CM.foldTailByMasking(),
9208 IsIndvarOverflowCheckNeededForVF, OrigLoop, BranchWeigths,
9209 OrigLoop->getLoopPredecessor()->getTerminator()->getDebugLoc(),
9214 assert(!State.Lane &&
"VPDerivedIVRecipe being replicated.");
9219 State.Builder.setFastMathFlags(FPBinOp->getFastMathFlags());
9227 State.set(
this, DerivedIV,
VPLane(0));
9273 if (
TTI->preferPredicateOverEpilogue(&TFI))
9292 LLVM_DEBUG(
dbgs() <<
"LV: cannot compute the outer-loop trip count\n");
9296 Function *
F = L->getHeader()->getParent();
9302 LoopVectorizationCostModel CM(
SEL, L, PSE, LI, LVL, *
TTI, TLI, DB, AC, ORE,
F,
9303 &Hints, IAI, PSI, BFI);
9307 LoopVectorizationPlanner LVP(L, LI, DT, TLI, *
TTI, LVL, CM, IAI, PSE, Hints,
9327 GeneratedRTChecks Checks(PSE, DT, LI,
TTI,
F->getDataLayout(), CM.
CostKind);
9329 BFI, PSI, Checks, BestPlan);
9331 << L->getHeader()->getParent()->getName() <<
"\"\n");
9353 if (S->getValueOperand()->getType()->isFloatTy())
9363 while (!Worklist.
empty()) {
9365 if (!L->contains(
I))
9367 if (!Visited.
insert(
I).second)
9377 I->getDebugLoc(), L->getHeader())
9378 <<
"floating point conversion changes vector width. "
9379 <<
"Mixed floating point precision requires an up/down "
9380 <<
"cast that will negatively impact performance.";
9383 for (
Use &
Op :
I->operands())
9399 for (
auto *PredVPBB : ExitVPBB->getPredecessors()) {
9405 << PredVPBB->getName() <<
":\n");
9406 Cost += PredVPBB->cost(VF, CostCtx);
9425 std::optional<unsigned> VScale) {
9441 <<
"LV: Interleaving only is not profitable due to runtime checks\n");
9500 uint64_t MinTC = std::max(MinTC1, MinTC2);
9502 MinTC =
alignTo(MinTC, IntVF);
9506 dbgs() <<
"LV: Minimum required TC for runtime checks to be profitable:"
9513 LLVM_DEBUG(
dbgs() <<
"LV: Vectorization is not beneficial: expected "
9514 "trip count < minimum profitable VF ("
9525 : InterleaveOnlyWhenForced(Opts.InterleaveOnlyWhenForced ||
9527 VectorizeOnlyWhenForced(Opts.VectorizeOnlyWhenForced ||
9548 if (EpiWidenedPhis.
contains(&VPIRInst->getIRPhi()))
9567 auto AddFreezeForFindLastIVReductions = [](
VPlan &Plan,
9568 bool UpdateResumePhis) {
9574 VPValue *OrigStart = VPI->getOperand(1);
9578 Builder.createNaryOp(Instruction::Freeze, {OrigStart}, {},
"fr");
9580 if (UpdateResumePhis)
9586 AddFreezeForFindLastIVReductions(MainPlan,
true);
9587 AddFreezeForFindLastIVReductions(EpiPlan,
false);
9594 auto ResumePhiIter =
9596 return match(&R, m_VPInstruction<Instruction::PHI>(m_Specific(VectorTC),
9599 VPPhi *ResumePhi =
nullptr;
9600 if (ResumePhiIter == MainScalarPH->
phis().
end()) {
9604 "vec.epilog.resume.val");
9607 if (MainScalarPH->
begin() == MainScalarPH->
end())
9609 else if (&*MainScalarPH->
begin() != ResumePhi)
9621 const SCEV2ValueTy &ExpandedSCEVs,
9625 Header->
setName(
"vec.epilog.vector.body");
9639 PHINode *EPResumeVal = &*L->getLoopPreheader()->phis().begin();
9644 "Must only have a single non-zero incoming value");
9656 [](
Value *Inc) { return match(Inc, m_SpecificInt(0)); }) &&
9657 "all incoming values must be 0");
9663 return isa<VPScalarIVStepsRecipe>(U) ||
9664 isa<VPDerivedIVRecipe>(U) ||
9665 cast<VPRecipeBase>(U)->isScalarCast() ||
9666 cast<VPInstruction>(U)->getOpcode() ==
9669 "the canonical IV should only be used by its increment or "
9670 "ScalarIVSteps when resetting the start value");
9671 IV->setOperand(0, VPV);
9675 Value *ResumeV =
nullptr;
9680 auto *VPI = dyn_cast<VPInstruction>(U);
9682 (VPI->getOpcode() == VPInstruction::ComputeAnyOfResult ||
9683 VPI->getOpcode() == VPInstruction::ComputeReductionResult ||
9684 VPI->getOpcode() == VPInstruction::ComputeFindIVResult);
9687 ->getIncomingValueForBlock(L->getLoopPreheader());
9688 RecurKind RK = ReductionPhi->getRecurrenceKind();
9696 ResumeV = Builder.CreateICmpNE(ResumeV, StartV);
9699 ToFrozen[StartV] =
cast<PHINode>(ResumeV)->getIncomingValueForBlock(
9710 Value *Cmp = Builder.CreateICmpEQ(ResumeV, ToFrozen[StartV]);
9711 Value *
Sentinel = RdxResult->getOperand(2)->getLiveInIRValue();
9712 ResumeV = Builder.CreateSelect(Cmp,
Sentinel, ResumeV);
9718 "unexpected start value");
9719 VPI->setOperand(0, StartVal);
9731 assert(ResumeV &&
"Must have a resume value");
9745 if (VPI && VPI->getOpcode() == Instruction::Freeze) {
9747 ToFrozen.
lookup(VPI->getOperand(0)->getLiveInIRValue())));
9762 ExpandR->eraseFromParent();
9772 const SCEV2ValueTy &ExpandedSCEVs,
Value *MainVectorTripCount,
9777 Value *EndValueFromAdditionalBypass = MainVectorTripCount;
9778 if (OrigPhi != OldInduction) {
9779 auto *BinOp =
II.getInductionBinOp();
9785 EndValueFromAdditionalBypass =
9787 II.getStartValue(), Step,
II.getKind(), BinOp);
9788 EndValueFromAdditionalBypass->
setName(
"ind.end");
9790 return EndValueFromAdditionalBypass;
9796 const SCEV2ValueTy &ExpandedSCEVs,
9797 Value *MainVectorTripCount) {
9802 if (Phi.getBasicBlockIndex(Pred) != -1)
9804 Phi.addIncoming(Phi.getIncomingValueForBlock(BypassBlock), Pred);
9808 if (ScalarPH->hasPredecessors()) {
9811 for (
const auto &[R, IRPhi] :
9812 zip(ScalarPH->phis(), ScalarPH->getIRBasicBlock()->phis())) {
9821 auto *Inc =
cast<PHINode>(IVPhi->getIncomingValueForBlock(PH));
9823 IVPhi,
II, BypassBuilder, ExpandedSCEVs, MainVectorTripCount,
9826 Inc->setIncomingValueForBlock(BypassBlock, V);
9832 "VPlan-native path is not enabled. Only process inner loops.");
9835 << L->getHeader()->getParent()->getName() <<
"' from "
9836 << L->getLocStr() <<
"\n");
9841 dbgs() <<
"LV: Loop hints:"
9852 Function *
F = L->getHeader()->getParent();
9874 LLVM_DEBUG(
dbgs() <<
"LV: Not vectorizing: Cannot prove legality.\n");
9881 "early exit is not enabled",
9882 "UncountableEarlyExitLoopsDisabled",
ORE, L);
9888 "faulting load is not supported",
9889 "PotentiallyFaultingLoadsNotSupported",
ORE, L);
9898 if (!L->isInnermost())
9902 assert(L->isInnermost() &&
"Inner loop expected.");
9905 bool UseInterleaved =
TTI->enableInterleavedAccessVectorization();
9919 [LoopLatch](
BasicBlock *BB) { return BB != LoopLatch; })) {
9921 "requiring a scalar epilogue is unsupported",
9922 "UncountableEarlyExitUnsupported",
ORE, L);
9935 if (ExpectedTC && ExpectedTC->isFixed() &&
9937 LLVM_DEBUG(
dbgs() <<
"LV: Found a loop with a very small trip count. "
9938 <<
"This loop is worth vectorizing only if no scalar "
9939 <<
"iteration overheads are incurred.");
9941 LLVM_DEBUG(
dbgs() <<
" But vectorizing was explicitly forced.\n");
9957 if (
F->hasFnAttribute(Attribute::NoImplicitFloat)) {
9959 "Can't vectorize when the NoImplicitFloat attribute is used",
9960 "loop not vectorized due to NoImplicitFloat attribute",
9961 "NoImplicitFloat",
ORE, L);
9971 TTI->isFPVectorizationPotentiallyUnsafe()) {
9973 "Potentially unsafe FP op prevents vectorization",
9974 "loop not vectorized due to unsafe FP support.",
9975 "UnsafeFP",
ORE, L);
9980 bool AllowOrderedReductions;
9985 AllowOrderedReductions =
TTI->enableOrderedReductions();
9990 ExactFPMathInst->getDebugLoc(),
9991 ExactFPMathInst->getParent())
9992 <<
"loop not vectorized: cannot prove it is safe to reorder "
9993 "floating-point operations";
9995 LLVM_DEBUG(
dbgs() <<
"LV: loop not vectorized: cannot prove it is safe to "
9996 "reorder floating-point operations\n");
10002 LoopVectorizationCostModel CM(
SEL, L, PSE,
LI, &LVL, *
TTI,
TLI,
DB,
AC,
ORE,
10005 LoopVectorizationPlanner LVP(L,
LI,
DT,
TLI, *
TTI, &LVL, CM, IAI, PSE, Hints,
10013 LVP.
plan(UserVF, UserIC);
10020 GeneratedRTChecks Checks(PSE,
DT,
LI,
TTI,
F->getDataLayout(), CM.
CostKind);
10025 unsigned SelectedIC = std::max(IC, UserIC);
10034 if (Checks.getSCEVChecks().first &&
10035 match(Checks.getSCEVChecks().first,
m_One()))
10037 if (Checks.getMemRuntimeChecks().first &&
10038 match(Checks.getMemRuntimeChecks().first,
m_One()))
10043 bool ForceVectorization =
10047 if (!ForceVectorization &&
10053 DEBUG_TYPE,
"CantReorderMemOps", L->getStartLoc(),
10055 <<
"loop not vectorized: cannot prove it is safe to reorder "
10056 "memory operations";
10065 std::pair<StringRef, std::string> VecDiagMsg, IntDiagMsg;
10066 bool VectorizeLoop =
true, InterleaveLoop =
true;
10068 LLVM_DEBUG(
dbgs() <<
"LV: Vectorization is possible but not beneficial.\n");
10070 "VectorizationNotBeneficial",
10071 "the cost-model indicates that vectorization is not beneficial"};
10072 VectorizeLoop =
false;
10078 LLVM_DEBUG(
dbgs() <<
"LV: Ignoring UserIC, because vectorization and "
10079 "interleaving should be avoided up front\n");
10080 IntDiagMsg = {
"InterleavingAvoided",
10081 "Ignoring UserIC, because interleaving was avoided up front"};
10082 InterleaveLoop =
false;
10083 }
else if (IC == 1 && UserIC <= 1) {
10087 "InterleavingNotBeneficial",
10088 "the cost-model indicates that interleaving is not beneficial"};
10089 InterleaveLoop =
false;
10091 IntDiagMsg.first =
"InterleavingNotBeneficialAndDisabled";
10092 IntDiagMsg.second +=
10093 " and is explicitly disabled or interleave count is set to 1";
10095 }
else if (IC > 1 && UserIC == 1) {
10097 LLVM_DEBUG(
dbgs() <<
"LV: Interleaving is beneficial but is explicitly "
10099 IntDiagMsg = {
"InterleavingBeneficialButDisabled",
10100 "the cost-model indicates that interleaving is beneficial "
10101 "but is explicitly disabled or interleave count is set to 1"};
10102 InterleaveLoop =
false;
10108 if (!VectorizeLoop && InterleaveLoop && LVL.
hasHistograms()) {
10109 LLVM_DEBUG(
dbgs() <<
"LV: Not interleaving without vectorization due "
10110 <<
"to histogram operations.\n");
10112 "HistogramPreventsScalarInterleaving",
10113 "Unable to interleave without vectorization due to constraints on "
10114 "the order of histogram operations"};
10115 InterleaveLoop =
false;
10119 IC = UserIC > 0 ? UserIC : IC;
10123 if (!VectorizeLoop && !InterleaveLoop) {
10127 L->getStartLoc(), L->getHeader())
10128 << VecDiagMsg.second;
10132 L->getStartLoc(), L->getHeader())
10133 << IntDiagMsg.second;
10138 if (!VectorizeLoop && InterleaveLoop) {
10142 L->getStartLoc(), L->getHeader())
10143 << VecDiagMsg.second;
10145 }
else if (VectorizeLoop && !InterleaveLoop) {
10147 <<
") in " << L->getLocStr() <<
'\n');
10150 L->getStartLoc(), L->getHeader())
10151 << IntDiagMsg.second;
10153 }
else if (VectorizeLoop && InterleaveLoop) {
10155 <<
") in " << L->getLocStr() <<
'\n');
10161 using namespace ore;
10166 <<
"interleaved loop (interleaved count: "
10167 << NV(
"InterleaveCount", IC) <<
")";
10184 std::unique_ptr<VPlan> BestMainPlan(BestPlan.
duplicate());
10195 PSI, Checks, *BestMainPlan);
10197 *BestMainPlan, MainILV,
DT,
false);
10203 BFI,
PSI, Checks, BestEpiPlan);
10211 BestEpiPlan, LVL, ExpandedSCEVs,
10213 ++LoopsEpilogueVectorized;
10215 InnerLoopVectorizer LB(L, PSE,
LI,
DT,
TTI,
AC, VF.
Width, IC, &CM,
BFI,
PSI,
10229 assert(
DT->verify(DominatorTree::VerificationLevel::Fast) &&
10230 "DT not preserved correctly");
10245 if (!
TTI->getNumberOfRegisters(
TTI->getRegisterClassForType(
true)) &&
10249 bool Changed =
false, CFGChanged =
false;
10256 for (
const auto &L : *
LI)
10268 LoopsAnalyzed += Worklist.
size();
10271 while (!Worklist.
empty()) {
10314 if (
PSI &&
PSI->hasProfileSummary())
10317 if (!Result.MadeAnyChange)
10331 if (Result.MadeCFGChange) {
10347 OS, MapClassName2PassName);
10350 OS << (InterleaveOnlyWhenForced ?
"" :
"no-") <<
"interleave-forced-only;";
10351 OS << (VectorizeOnlyWhenForced ?
"" :
"no-") <<
"vectorize-forced-only;";
static unsigned getIntrinsicID(const SDNode *N)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPU Lower Kernel Arguments
AMDGPU Register Bank Select
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static bool isEqual(const Function &Caller, const Function &Callee)
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
This is the interface for LLVM's primary stateless and local alias analysis.
static bool IsEmptyBlock(MachineBasicBlock *MBB)
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static cl::opt< OutputCostKind > CostKind("cost-kind", cl::desc("Target cost kind"), cl::init(OutputCostKind::RecipThroughput), cl::values(clEnumValN(OutputCostKind::RecipThroughput, "throughput", "Reciprocal throughput"), clEnumValN(OutputCostKind::Latency, "latency", "Instruction latency"), clEnumValN(OutputCostKind::CodeSize, "code-size", "Code size"), clEnumValN(OutputCostKind::SizeAndLatency, "size-latency", "Code size and latency"), clEnumValN(OutputCostKind::All, "all", "Print all cost kinds")))
static cl::opt< IntrinsicCostStrategy > IntrinsicCost("intrinsic-cost-strategy", cl::desc("Costing strategy for intrinsic instructions"), cl::init(IntrinsicCostStrategy::InstructionCost), cl::values(clEnumValN(IntrinsicCostStrategy::InstructionCost, "instruction-cost", "Use TargetTransformInfo::getInstructionCost"), clEnumValN(IntrinsicCostStrategy::IntrinsicCost, "intrinsic-cost", "Use TargetTransformInfo::getIntrinsicInstrCost"), clEnumValN(IntrinsicCostStrategy::TypeBasedIntrinsicCost, "type-based-intrinsic-cost", "Calculate the intrinsic cost based only on argument types")))
static InstructionCost getCost(Instruction &Inst, TTI::TargetCostKind CostKind, TargetTransformInfo &TTI, TargetLibraryInfo &TLI)
This file defines DenseMapInfo traits for DenseMap.
This file defines the DenseMap class.
This is the interface for a simple mod/ref and alias analysis over globals.
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
Module.h This file contains the declarations for the Module class.
This defines the Use class.
static bool hasNoUnsignedWrap(BinaryOperator &I)
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
static std::pair< Value *, APInt > getMask(Value *WideMask, unsigned Factor, ElementCount LeafValueEC)
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Legalize the Machine IR a function s Machine IR
static cl::opt< unsigned, true > VectorizationFactor("force-vector-width", cl::Hidden, cl::desc("Sets the SIMD width. Zero is autoselect."), cl::location(VectorizerParams::VectorizationFactor))
This header provides classes for managing per-loop analyses.
static const char * VerboseDebug
This file defines the LoopVectorizationLegality class.
This file provides a LoopVectorizationPlanner class.
static void collectSupportedLoops(Loop &L, LoopInfo *LI, OptimizationRemarkEmitter *ORE, SmallVectorImpl< Loop * > &V)
static cl::opt< unsigned > EpilogueVectorizationMinVF("epilogue-vectorization-minimum-VF", cl::Hidden, cl::desc("Only loops with vectorization factor equal to or larger than " "the specified value are considered for epilogue vectorization."))
static cl::opt< unsigned > EpilogueVectorizationForceVF("epilogue-vectorization-force-VF", cl::init(1), cl::Hidden, cl::desc("When epilogue vectorization is enabled, and a value greater than " "1 is specified, forces the given VF for all applicable epilogue " "loops."))
static void addScalarResumePhis(VPRecipeBuilder &Builder, VPlan &Plan, DenseMap< VPValue *, VPValue * > &IVEndValues)
Create resume phis in the scalar preheader for first-order recurrences, reductions and inductions,...
static Type * maybeVectorizeType(Type *Ty, ElementCount VF)
static ElementCount determineVPlanVF(const TargetTransformInfo &TTI, LoopVectorizationCostModel &CM)
static ElementCount getSmallConstantTripCount(ScalarEvolution *SE, const Loop *L)
A version of ScalarEvolution::getSmallConstantTripCount that returns an ElementCount to include loops...
static cl::opt< unsigned > VectorizeMemoryCheckThreshold("vectorize-memory-check-threshold", cl::init(128), cl::Hidden, cl::desc("The maximum allowed number of runtime memory checks"))
static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan)
Prepare MainPlan for vectorizing the main vector loop during epilogue vectorization.
static cl::opt< unsigned > TinyTripCountVectorThreshold("vectorizer-min-trip-count", cl::init(16), cl::Hidden, cl::desc("Loops with a constant trip count that is smaller than this " "value are vectorized only if no scalar iteration overheads " "are incurred."))
Loops with a known constant trip count below this number are vectorized only if no scalar iteration o...
static void debugVectorizationMessage(const StringRef Prefix, const StringRef DebugMsg, Instruction *I)
Write a DebugMsg about vectorization to the debug output stream.
static cl::opt< bool > EnableCondStoresVectorization("enable-cond-stores-vec", cl::init(true), cl::Hidden, cl::desc("Enable if predication of stores during vectorization."))
static VPIRBasicBlock * replaceVPBBWithIRVPBB(VPBasicBlock *VPBB, BasicBlock *IRBB, VPlan *Plan=nullptr)
Replace VPBB with a VPIRBasicBlock wrapping IRBB.
static VPInstruction * addResumePhiRecipeForInduction(VPWidenInductionRecipe *WideIV, VPBuilder &VectorPHBuilder, VPBuilder &ScalarPHBuilder, VPTypeAnalysis &TypeInfo, VPValue *VectorTC)
Create and return a ResumePhi for WideIV, unless it is truncated.
static Value * emitTransformedIndex(IRBuilderBase &B, Value *Index, Value *StartValue, Value *Step, InductionDescriptor::InductionKind InductionKind, const BinaryOperator *InductionBinOp)
Compute the transformed value of Index at offset StartValue using step StepValue.
static DebugLoc getDebugLocFromInstOrOperands(Instruction *I)
Look for a meaningful debug location on the instruction or its operands.
static Value * createInductionAdditionalBypassValues(PHINode *OrigPhi, const InductionDescriptor &II, IRBuilder<> &BypassBuilder, const SCEV2ValueTy &ExpandedSCEVs, Value *MainVectorTripCount, Instruction *OldInduction)
static void fixReductionScalarResumeWhenVectorizingEpilog(VPPhi *EpiResumePhiR, PHINode &EpiResumePhi, BasicBlock *BypassBlock)
static Value * getStartValueFromReductionResult(VPInstruction *RdxResult)
static cl::opt< bool > ForceTargetSupportsScalableVectors("force-target-supports-scalable-vectors", cl::init(false), cl::Hidden, cl::desc("Pretend that scalable vectors are supported, even if the target does " "not support them. This flag should only be used for testing."))
static bool useActiveLaneMaskForControlFlow(TailFoldingStyle Style)
static cl::opt< bool > EnableEarlyExitVectorization("enable-early-exit-vectorization", cl::init(true), cl::Hidden, cl::desc("Enable vectorization of early exit loops with uncountable exits."))
static cl::opt< bool > ConsiderRegPressure("vectorizer-consider-reg-pressure", cl::init(false), cl::Hidden, cl::desc("Discard VFs if their register pressure is too high."))
static unsigned estimateElementCount(ElementCount VF, std::optional< unsigned > VScale)
This function attempts to return a value that represents the ElementCount at runtime.
static constexpr uint32_t MinItersBypassWeights[]
static cl::opt< unsigned > ForceTargetNumScalarRegs("force-target-num-scalar-regs", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's number of scalar registers."))
static cl::opt< bool > UseWiderVFIfCallVariantsPresent("vectorizer-maximize-bandwidth-for-vector-calls", cl::init(true), cl::Hidden, cl::desc("Try wider VFs if they enable the use of vector variants"))
static std::optional< unsigned > getMaxVScale(const Function &F, const TargetTransformInfo &TTI)
static cl::opt< unsigned > SmallLoopCost("small-loop-cost", cl::init(20), cl::Hidden, cl::desc("The cost of a loop that is considered 'small' by the interleaver."))
static bool planContainsAdditionalSimplifications(VPlan &Plan, VPCostContext &CostCtx, Loop *TheLoop, ElementCount VF)
Return true if the original loop \ TheLoop contains any instructions that do not have corresponding r...
static cl::opt< unsigned > ForceTargetNumVectorRegs("force-target-num-vector-regs", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's number of vector registers."))
static bool isExplicitVecOuterLoop(Loop *OuterLp, OptimizationRemarkEmitter *ORE)
static cl::opt< bool > EnableIndVarRegisterHeur("enable-ind-var-reg-heur", cl::init(true), cl::Hidden, cl::desc("Count the induction variable only once when interleaving"))
static cl::opt< TailFoldingStyle > ForceTailFoldingStyle("force-tail-folding-style", cl::desc("Force the tail folding style"), cl::init(TailFoldingStyle::None), cl::values(clEnumValN(TailFoldingStyle::None, "none", "Disable tail folding"), clEnumValN(TailFoldingStyle::Data, "data", "Create lane mask for data only, using active.lane.mask intrinsic"), clEnumValN(TailFoldingStyle::DataWithoutLaneMask, "data-without-lane-mask", "Create lane mask with compare/stepvector"), clEnumValN(TailFoldingStyle::DataAndControlFlow, "data-and-control", "Create lane mask using active.lane.mask intrinsic, and use " "it for both data and control flow"), clEnumValN(TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck, "data-and-control-without-rt-check", "Similar to data-and-control, but remove the runtime check"), clEnumValN(TailFoldingStyle::DataWithEVL, "data-with-evl", "Use predicated EVL instructions for tail folding. If EVL " "is unsupported, fallback to data-without-lane-mask.")))
static cl::opt< bool > EnableEpilogueVectorization("enable-epilogue-vectorization", cl::init(true), cl::Hidden, cl::desc("Enable vectorization of epilogue loops."))
static ScalarEpilogueLowering getScalarEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI, TargetTransformInfo *TTI, TargetLibraryInfo *TLI, LoopVectorizationLegality &LVL, InterleavedAccessInfo *IAI)
static void preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L, const SCEV2ValueTy &ExpandedSCEVs, EpilogueLoopVectorizationInfo &EPI)
Prepare Plan for vectorizing the epilogue loop.
static cl::opt< bool > PreferPredicatedReductionSelect("prefer-predicated-reduction-select", cl::init(false), cl::Hidden, cl::desc("Prefer predicating a reduction operation over an after loop select."))
static VPWidenIntOrFpInductionRecipe * createWidenInductionRecipes(PHINode *Phi, Instruction *PhiOrTrunc, VPValue *Start, const InductionDescriptor &IndDesc, VPlan &Plan, ScalarEvolution &SE, Loop &OrigLoop)
Creates a VPWidenIntOrFpInductionRecpipe for Phi.
static cl::opt< bool > PreferInLoopReductions("prefer-inloop-reductions", cl::init(false), cl::Hidden, cl::desc("Prefer in-loop vector reductions, " "overriding the targets preference."))
static cl::opt< bool > EnableLoadStoreRuntimeInterleave("enable-loadstore-runtime-interleave", cl::init(true), cl::Hidden, cl::desc("Enable runtime interleaving until load/store ports are saturated"))
static cl::opt< bool > VPlanBuildStressTest("vplan-build-stress-test", cl::init(false), cl::Hidden, cl::desc("Build VPlan for every supported loop nest in the function and bail " "out right after the build (stress test the VPlan H-CFG construction " "in the VPlan-native vectorization path)."))
static bool hasIrregularType(Type *Ty, const DataLayout &DL)
A helper function that returns true if the given type is irregular.
static cl::opt< bool > LoopVectorizeWithBlockFrequency("loop-vectorize-with-block-frequency", cl::init(true), cl::Hidden, cl::desc("Enable the use of the block frequency analysis to access PGO " "heuristics minimizing code growth in cold regions and being more " "aggressive in hot regions."))
static std::optional< ElementCount > getSmallBestKnownTC(PredicatedScalarEvolution &PSE, Loop *L, bool CanUseConstantMax=true)
Returns "best known" trip count, which is either a valid positive trip count or std::nullopt when an ...
static Value * getExpandedStep(const InductionDescriptor &ID, const SCEV2ValueTy &ExpandedSCEVs)
Return the expanded step for ID using ExpandedSCEVs to look up SCEV expansion results.
static bool useActiveLaneMask(TailFoldingStyle Style)
static bool hasReplicatorRegion(VPlan &Plan)
static bool isIndvarOverflowCheckKnownFalse(const LoopVectorizationCostModel *Cost, ElementCount VF, std::optional< unsigned > UF=std::nullopt)
For the given VF and UF and maximum trip count computed for the loop, return whether the induction va...
static void addFullyUnrolledInstructionsToIgnore(Loop *L, const LoopVectorizationLegality::InductionList &IL, SmallPtrSetImpl< Instruction * > &InstsToIgnore)
Knowing that loop L executes a single vector iteration, add instructions that will get simplified and...
static cl::opt< PreferPredicateTy::Option > PreferPredicateOverEpilogue("prefer-predicate-over-epilogue", cl::init(PreferPredicateTy::ScalarEpilogue), cl::Hidden, cl::desc("Tail-folding and predication preferences over creating a scalar " "epilogue loop."), cl::values(clEnumValN(PreferPredicateTy::ScalarEpilogue, "scalar-epilogue", "Don't tail-predicate loops, create scalar epilogue"), clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue, "predicate-else-scalar-epilogue", "prefer tail-folding, create scalar epilogue if tail " "folding fails."), clEnumValN(PreferPredicateTy::PredicateOrDontVectorize, "predicate-dont-vectorize", "prefers tail-folding, don't attempt vectorization if " "tail-folding fails.")))
static cl::opt< bool > EnableInterleavedMemAccesses("enable-interleaved-mem-accesses", cl::init(false), cl::Hidden, cl::desc("Enable vectorization on interleaved memory accesses in a loop"))
static cl::opt< bool > EnableMaskedInterleavedMemAccesses("enable-masked-interleaved-mem-accesses", cl::init(false), cl::Hidden, cl::desc("Enable vectorization on masked interleaved memory accesses in a loop"))
An interleave-group may need masking if it resides in a block that needs predication,...
static cl::opt< bool > ForceOrderedReductions("force-ordered-reductions", cl::init(false), cl::Hidden, cl::desc("Enable the vectorisation of loops with in-order (strict) " "FP reductions"))
static void cse(BasicBlock *BB)
Perform cse of induction variable instructions.
static const SCEV * getAddressAccessSCEV(Value *Ptr, LoopVectorizationLegality *Legal, PredicatedScalarEvolution &PSE, const Loop *TheLoop)
Gets Address Access SCEV after verifying that the access pattern is loop invariant except the inducti...
static cl::opt< cl::boolOrDefault > ForceSafeDivisor("force-widen-divrem-via-safe-divisor", cl::Hidden, cl::desc("Override cost based safe divisor widening for div/rem instructions"))
static InstructionCost calculateEarlyExitCost(VPCostContext &CostCtx, VPlan &Plan, ElementCount VF)
For loops with uncountable early exits, find the cost of doing work when exiting the loop early,...
static cl::opt< unsigned > ForceTargetMaxVectorInterleaveFactor("force-target-max-vector-interleave", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's max interleave factor for " "vectorized loops."))
static bool processLoopInVPlanNativePath(Loop *L, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, LoopVectorizationLegality *LVL, TargetTransformInfo *TTI, TargetLibraryInfo *TLI, DemandedBits *DB, AssumptionCache *AC, OptimizationRemarkEmitter *ORE, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, LoopVectorizeHints &Hints, LoopVectorizationRequirements &Requirements)
static bool useMaskedInterleavedAccesses(const TargetTransformInfo &TTI)
static cl::opt< unsigned > NumberOfStoresToPredicate("vectorize-num-stores-pred", cl::init(1), cl::Hidden, cl::desc("Max number of stores to be predicated behind an if."))
The number of stores in a loop that are allowed to need predication.
static cl::opt< unsigned > MaxNestedScalarReductionIC("max-nested-scalar-reduction-interleave", cl::init(2), cl::Hidden, cl::desc("The maximum interleave count to use when interleaving a scalar " "reduction in a nested loop."))
static cl::opt< unsigned > ForceTargetMaxScalarInterleaveFactor("force-target-max-scalar-interleave", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's max interleave factor for " "scalar loops."))
static void checkMixedPrecision(Loop *L, OptimizationRemarkEmitter *ORE)
static bool willGenerateVectors(VPlan &Plan, ElementCount VF, const TargetTransformInfo &TTI)
Check if any recipe of Plan will generate a vector value, which will be assigned a vector register.
static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks, VectorizationFactor &VF, Loop *L, PredicatedScalarEvolution &PSE, VPCostContext &CostCtx, VPlan &Plan, ScalarEpilogueLowering SEL, std::optional< unsigned > VScale)
This function determines whether or not it's still profitable to vectorize the loop given the extra w...
static void addExitUsersForFirstOrderRecurrences(VPlan &Plan, VFRange &Range)
Handle users in the exit block for first order reductions in the original exit block.
static void fixScalarResumeValuesFromBypass(BasicBlock *BypassBlock, Loop *L, VPlan &BestEpiPlan, LoopVectorizationLegality &LVL, const SCEV2ValueTy &ExpandedSCEVs, Value *MainVectorTripCount)
static cl::opt< bool > MaximizeBandwidth("vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden, cl::desc("Maximize bandwidth when selecting vectorization factor which " "will be determined by the smallest type in loop."))
static OptimizationRemarkAnalysis createLVAnalysis(const char *PassName, StringRef RemarkName, Loop *TheLoop, Instruction *I, DebugLoc DL={})
Create an analysis remark that explains why vectorization failed.
mir Rename Register Operands
This file implements a map that provides insertion order iteration.
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
This file contains the declarations for profiling metadata utility functions.
const SmallVectorImpl< MachineOperand > & Cond
static BinaryOperator * CreateMul(Value *S1, Value *S2, const Twine &Name, BasicBlock::iterator InsertBefore, Value *FlagsOp)
static BinaryOperator * CreateAdd(Value *S1, Value *S2, const Twine &Name, BasicBlock::iterator InsertBefore, Value *FlagsOp)
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
#define DEBUG_WITH_TYPE(TYPE,...)
DEBUG_WITH_TYPE macro - This macro should be used by passes to emit debug information.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")
LocallyHashedType DenseMapInfo< LocallyHashedType >::Empty
This file implements the TypeSwitch template, which mimics a switch() statement whose cases are type ...
This file contains the declarations of different VPlan-related auxiliary helpers.
This file declares the class VPlanVerifier, which contains utility functions to check the consistency...
This file contains the declarations of the Vectorization Plan base classes:
static const char PassName[]
static const uint32_t IV[8]
A manager for alias analyses.
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
uint64_t getZExtValue() const
Get zero extended value.
unsigned getActiveBits() const
Compute the number of active bits in the value.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
LLVM_ABI unsigned getVScaleRangeMin() const
Returns the minimum value for the vscale_range attribute.
LLVM Basic Block Representation.
iterator begin()
Instruction iterator methods.
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
LLVM_ABI const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const Function * getParent() const
Return the enclosing method, or null if none.
LLVM_ABI InstListType::const_iterator getFirstNonPHIIt() const
Returns an iterator to the first instruction in this block that is not a PHINode instruction.
LLVM_ABI const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
LLVM_ABI const BasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this basic block belongs to.
LLVM_ABI LLVMContext & getContext() const
Get the context in which this basic block lives.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
BinaryOps getOpcode() const
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Conditional or Unconditional Branch instruction.
bool isConditional() const
static BranchInst * Create(BasicBlock *IfTrue, InsertPosition InsertBefore=nullptr)
BasicBlock * getSuccessor(unsigned i) const
Represents analyses that only rely on functions' control flow.
bool isNoBuiltin() const
Return true if the call should not be treated as a call to a builtin.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Value * getArgOperand(unsigned i) const
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
unsigned arg_size() const
This class represents a function call, abstracting a target machine's calling convention.
static Type * makeCmpResultType(Type *opnd_type)
Create a result type for fcmp/icmp.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_UGT
unsigned greater than
@ ICMP_ULT
unsigned less than
@ ICMP_ULE
unsigned less or equal
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
An abstraction over a floating-point predicate, and a pack of an integer predicate with samesign info...
This is the shared class of boolean and integer constants.
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
A parsed version of the target data layout string in and methods for querying it.
static DebugLoc getTemporary()
static DebugLoc getUnknown()
An analysis that produces DemandedBits for a function.
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
void insert_range(Range &&R)
Inserts range of 'std::pair<KeyT, ValueT>' values into the map.
Implements a dense probed hash-table based set.
Analysis pass which computes a DominatorTree.
void changeImmediateDominator(DomTreeNodeBase< NodeT > *N, DomTreeNodeBase< NodeT > *NewIDom)
changeImmediateDominator - This method is used to update the dominator tree information when a node's...
void eraseNode(NodeT *BB)
eraseNode - Removes a node from the dominator tree.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
constexpr bool isVector() const
One or more elements.
static constexpr ElementCount getScalable(ScalarTy MinVal)
static constexpr ElementCount getFixed(ScalarTy MinVal)
static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)
constexpr bool isScalar() const
Exactly one element.
void printDebugTracesAtEnd() override
BasicBlock * getAdditionalBypassBlock() const
Return the additional bypass block which targets the scalar loop by skipping the epilogue loop after ...
BasicBlock * createVectorizedLoopSkeleton() final
Implements the interface for creating a vectorized skeleton using the epilogue loop strategy (i....
EpilogueVectorizerEpilogueLoop(Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, const TargetTransformInfo *TTI, AssumptionCache *AC, EpilogueLoopVectorizationInfo &EPI, LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, GeneratedRTChecks &Checks, VPlan &Plan)
void printDebugTracesAtStart() override
Allow subclasses to override and print debug traces before/after vplan execution, when trace informat...
BasicBlock * emitMinimumVectorEpilogueIterCountCheck(BasicBlock *VectorPH, BasicBlock *Bypass, BasicBlock *Insert)
Emits an iteration count bypass check after the main vector loop has finished to see if there are any...
A specialized derived class of inner loop vectorizer that performs vectorization of main loops in the...
void introduceCheckBlockInVPlan(BasicBlock *CheckIRBB)
Introduces a new VPIRBasicBlock for CheckIRBB to Plan between the vector preheader and its predecesso...
BasicBlock * emitIterationCountCheck(BasicBlock *VectorPH, BasicBlock *Bypass, bool ForEpilogue)
Emits an iteration count bypass check once for the main loop (when ForEpilogue is false) and once for...
EpilogueVectorizerMainLoop(Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, const TargetTransformInfo *TTI, AssumptionCache *AC, EpilogueLoopVectorizationInfo &EPI, LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, GeneratedRTChecks &Check, VPlan &Plan)
void printDebugTracesAtEnd() override
Value * createIterationCountCheck(BasicBlock *VectorPH, ElementCount VF, unsigned UF) const
void printDebugTracesAtStart() override
Allow subclasses to override and print debug traces before/after vplan execution, when trace informat...
BasicBlock * createVectorizedLoopSkeleton() final
Implements the interface for creating a vectorized skeleton using the main loop strategy (i....
Convenience struct for specifying and reasoning about fast-math flags.
Class to represent function types.
param_iterator param_begin() const
param_iterator param_end() const
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Represents flags for the getelementptr instruction/expression.
static GEPNoWrapFlags none()
Common base class shared among various IRBuilders.
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
A struct for saving information about induction variables.
const SCEV * getStep() const
InductionKind
This enum represents the kinds of inductions that we support.
@ IK_NoInduction
Not an induction variable.
@ IK_FpInduction
Floating point induction variable.
@ IK_PtrInduction
Pointer induction var. Step = C.
@ IK_IntInduction
Integer induction variable. Step = C.
const SmallVectorImpl< Instruction * > & getCastInsts() const
Returns a reference to the type cast instructions in the induction update chain, that are redundant w...
Value * getStartValue() const
ElementCount MinProfitableTripCount
InnerLoopAndEpilogueVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, const TargetTransformInfo *TTI, AssumptionCache *AC, EpilogueLoopVectorizationInfo &EPI, LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, GeneratedRTChecks &Checks, VPlan &Plan, ElementCount VecWidth, ElementCount MinProfitableTripCount, unsigned UnrollFactor)
EpilogueLoopVectorizationInfo & EPI
Holds and updates state information required to vectorize the main loop and its epilogue in two separ...
InnerLoopVectorizer vectorizes loops which contain only one basic block to a specified vectorization ...
virtual void printDebugTracesAtStart()
Allow subclasses to override and print debug traces before/after vplan execution, when trace informat...
Value * TripCount
Trip count of the original loop.
const TargetTransformInfo * TTI
Target Transform Info.
LoopVectorizationCostModel * Cost
The profitablity analysis.
BlockFrequencyInfo * BFI
BFI and PSI are used to check for profile guided size optimizations.
Value * getTripCount() const
Returns the original loop trip count.
friend class LoopVectorizationPlanner
PredicatedScalarEvolution & PSE
A wrapper around ScalarEvolution used to add runtime SCEV checks.
DominatorTree * DT
Dominator Tree.
void setTripCount(Value *TC)
Used to set the trip count after ILV's construction and after the preheader block has been executed.
void fixVectorizedLoop(VPTransformState &State)
Fix the vectorized code, taking care of header phi's, and more.
virtual BasicBlock * createVectorizedLoopSkeleton()
Creates a basic block for the scalar preheader.
virtual void printDebugTracesAtEnd()
AssumptionCache * AC
Assumption Cache.
InnerLoopVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, const TargetTransformInfo *TTI, AssumptionCache *AC, ElementCount VecWidth, unsigned UnrollFactor, LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, GeneratedRTChecks &RTChecks, VPlan &Plan)
IRBuilder Builder
The builder that we use.
void fixNonInductionPHIs(VPTransformState &State)
Fix the non-induction PHIs in Plan.
VPBasicBlock * VectorPHVPBB
The vector preheader block of Plan, used as target for check blocks introduced during skeleton creati...
unsigned UF
The vectorization unroll factor to use.
GeneratedRTChecks & RTChecks
Structure to hold information about generated runtime checks, responsible for cleaning the checks,...
virtual ~InnerLoopVectorizer()=default
ElementCount VF
The vectorization SIMD factor to use.
Loop * OrigLoop
The original loop.
BasicBlock * createScalarPreheader(StringRef Prefix)
Create and return a new IR basic block for the scalar preheader whose name is prefixed with Prefix.
InstSimplifyFolder - Use InstructionSimplify to fold operations to existing values.
static InstructionCost getInvalid(CostType Val=0)
static InstructionCost getMax()
CostType getValue() const
This function is intended to be used as sparingly as possible, since the class provides the full rang...
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
LLVM_ABI void moveBefore(InstListType::iterator InsertPos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Instruction * user_back()
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
const char * getOpcodeName() const
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Class to represent integer types.
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
LLVM_ABI APInt getMask() const
For example, this is 0xFF for an 8 bit integer, 0xFFFF for i16, etc.
The group of interleaved loads/stores sharing the same stride and close to each other.
uint32_t getFactor() const
InstTy * getMember(uint32_t Index) const
Get the member with the given index Index.
InstTy * getInsertPos() const
uint32_t getNumMembers() const
Drive the analysis of interleaved memory accesses in the loop.
bool requiresScalarEpilogue() const
Returns true if an interleaved group that may access memory out-of-bounds requires a scalar epilogue ...
LLVM_ABI void analyzeInterleaving(bool EnableMaskedInterleavedGroup)
Analyze the interleaved accesses and collect them in interleave groups.
An instruction for reading from memory.
Type * getPointerOperandType() const
This analysis provides dependence information for the memory accesses of a loop.
Drive the analysis of memory accesses in the loop.
const RuntimePointerChecking * getRuntimePointerChecking() const
unsigned getNumRuntimePointerChecks() const
Number of memchecks required to prove independence of otherwise may-alias pointers.
Analysis pass that exposes the LoopInfo for a function.
bool contains(const LoopT *L) const
Return true if the specified loop is contained within in this loop.
BlockT * getLoopLatch() const
If there is a single latch block for this loop, return it.
bool isInnermost() const
Return true if the loop does not contain any (natural) loops.
BlockT * getHeader() const
iterator_range< block_iterator > blocks() const
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
Store the result of a depth first search within basic blocks contained by a single loop.
RPOIterator beginRPO() const
Reverse iterate over the cached postorder blocks.
void perform(const LoopInfo *LI)
Traverse the loop blocks and store the DFS result.
RPOIterator endRPO() const
Wrapper class to LoopBlocksDFS that provides a standard begin()/end() interface for the DFS reverse p...
void perform(const LoopInfo *LI)
Traverse the loop blocks and store the DFS result.
void removeBlock(BlockT *BB)
This method completely removes BB from all data structures, including all of the Loop objects it is n...
LoopVectorizationCostModel - estimates the expected speedups due to vectorization.
SmallPtrSet< Type *, 16 > ElementTypesInLoop
All element types found in the loop.
bool isLegalMaskedLoad(Type *DataType, Value *Ptr, Align Alignment, unsigned AddressSpace) const
Returns true if the target machine supports masked load operation for the given DataType and kind of ...
LoopVectorizationCostModel(ScalarEpilogueLowering SEL, Loop *L, PredicatedScalarEvolution &PSE, LoopInfo *LI, LoopVectorizationLegality *Legal, const TargetTransformInfo &TTI, const TargetLibraryInfo *TLI, DemandedBits *DB, AssumptionCache *AC, OptimizationRemarkEmitter *ORE, const Function *F, const LoopVectorizeHints *Hints, InterleavedAccessInfo &IAI, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI)
void collectElementTypesForWidening()
Collect all element types in the loop for which widening is needed.
bool canVectorizeReductions(ElementCount VF) const
Returns true if the target machine supports all of the reduction variables found for the given VF.
bool isLegalMaskedStore(Type *DataType, Value *Ptr, Align Alignment, unsigned AddressSpace) const
Returns true if the target machine supports masked store operation for the given DataType and kind of...
bool isEpilogueVectorizationProfitable(const ElementCount VF, const unsigned IC) const
Returns true if epilogue vectorization is considered profitable, and false otherwise.
bool isPredicatedInst(Instruction *I) const
Returns true if I is an instruction that needs to be predicated at runtime.
bool hasPredStores() const
void collectValuesToIgnore()
Collect values we want to ignore in the cost model.
void collectInLoopReductions()
Split reductions into those that happen in the loop, and those that happen outside.
std::pair< unsigned, unsigned > getSmallestAndWidestTypes()
bool isUniformAfterVectorization(Instruction *I, ElementCount VF) const
Returns true if I is known to be uniform after vectorization.
void collectNonVectorizedAndSetWideningDecisions(ElementCount VF)
Collect values that will not be widened, including Uniforms, Scalars, and Instructions to Scalarize f...
PredicatedScalarEvolution & PSE
Predicated scalar evolution analysis.
const LoopVectorizeHints * Hints
Loop Vectorize Hint.
std::optional< unsigned > getMaxSafeElements() const
Return maximum safe number of elements to be processed per vector iteration, which do not prevent sto...
const TargetTransformInfo & TTI
Vector target information.
friend class LoopVectorizationPlanner
const Function * TheFunction
LoopVectorizationLegality * Legal
Vectorization legality.
std::optional< InstructionCost > getReductionPatternCost(Instruction *I, ElementCount VF, Type *VectorTy) const
Return the cost of instructions in an inloop reduction pattern, if I is part of that pattern.
InstructionCost getInstructionCost(Instruction *I, ElementCount VF)
Returns the execution time cost of an instruction for a given vector width.
DemandedBits * DB
Demanded bits analysis.
bool interleavedAccessCanBeWidened(Instruction *I, ElementCount VF) const
Returns true if I is a memory instruction in an interleaved-group of memory accesses that can be vect...
const TargetLibraryInfo * TLI
Target Library Info.
bool memoryInstructionCanBeWidened(Instruction *I, ElementCount VF)
Returns true if I is a memory instruction with consecutive memory access that can be widened.
const InterleaveGroup< Instruction > * getInterleavedAccessGroup(Instruction *Instr) const
Get the interleaved access group that Instr belongs to.
InstructionCost getVectorIntrinsicCost(CallInst *CI, ElementCount VF) const
Estimate cost of an intrinsic call instruction CI if it were vectorized with factor VF.
bool OptForSize
Whether this loop should be optimized for size based on function attribute or profile information.
bool useMaxBandwidth(TargetTransformInfo::RegisterKind RegKind)
bool isScalarAfterVectorization(Instruction *I, ElementCount VF) const
Returns true if I is known to be scalar after vectorization.
bool isOptimizableIVTruncate(Instruction *I, ElementCount VF)
Return True if instruction I is an optimizable truncate whose operand is an induction variable.
FixedScalableVFPair computeMaxVF(ElementCount UserVF, unsigned UserIC)
bool shouldConsiderRegPressureForVF(ElementCount VF)
Loop * TheLoop
The loop that we evaluate.
TTI::TargetCostKind CostKind
The kind of cost that we are calculating.
TailFoldingStyle getTailFoldingStyle(bool IVUpdateMayOverflow=true) const
Returns the TailFoldingStyle that is best for the current loop.
InterleavedAccessInfo & InterleaveInfo
The interleave access information contains groups of interleaved accesses with the same stride and cl...
SmallPtrSet< const Value *, 16 > ValuesToIgnore
Values to ignore in the cost model.
void setVectorizedCallDecision(ElementCount VF)
A call may be vectorized in different ways depending on whether we have vectorized variants available...
void invalidateCostModelingDecisions()
Invalidates decisions already taken by the cost model.
bool isAccessInterleaved(Instruction *Instr) const
Check if Instr belongs to any interleaved access group.
bool selectUserVectorizationFactor(ElementCount UserVF)
Setup cost-based decisions for user vectorization factor.
std::optional< unsigned > getVScaleForTuning() const
Return the value of vscale used for tuning the cost model.
OptimizationRemarkEmitter * ORE
Interface to emit optimization remarks.
LoopInfo * LI
Loop Info analysis.
bool requiresScalarEpilogue(bool IsVectorizing) const
Returns true if we're required to use a scalar epilogue for at least the final iteration of the origi...
SmallPtrSet< const Value *, 16 > VecValuesToIgnore
Values to ignore in the cost model when VF > 1.
bool isInLoopReduction(PHINode *Phi) const
Returns true if the Phi is part of an inloop reduction.
bool isProfitableToScalarize(Instruction *I, ElementCount VF) const
void setWideningDecision(const InterleaveGroup< Instruction > *Grp, ElementCount VF, InstWidening W, InstructionCost Cost)
Save vectorization decision W and Cost taken by the cost model for interleaving group Grp and vector ...
const MapVector< Instruction *, uint64_t > & getMinimalBitwidths() const
CallWideningDecision getCallWideningDecision(CallInst *CI, ElementCount VF) const
bool isLegalGatherOrScatter(Value *V, ElementCount VF)
Returns true if the target machine can represent V as a masked gather or scatter operation.
bool canTruncateToMinimalBitwidth(Instruction *I, ElementCount VF) const
bool runtimeChecksRequired()
bool shouldConsiderInvariant(Value *Op)
Returns true if Op should be considered invariant and if it is trivially hoistable.
bool foldTailByMasking() const
Returns true if all loop blocks should be masked to fold tail loop.
bool foldTailWithEVL() const
Returns true if VP intrinsics with explicit vector length support should be generated in the tail fol...
bool usePredicatedReductionSelect() const
Returns true if the predicated reduction select should be used to set the incoming value for the redu...
bool blockNeedsPredicationForAnyReason(BasicBlock *BB) const
Returns true if the instructions in this block requires predication for any reason,...
void setCallWideningDecision(CallInst *CI, ElementCount VF, InstWidening Kind, Function *Variant, Intrinsic::ID IID, std::optional< unsigned > MaskPos, InstructionCost Cost)
void setTailFoldingStyles(bool IsScalableVF, unsigned UserIC)
Selects and saves TailFoldingStyle for 2 options - if IV update may overflow or not.
AssumptionCache * AC
Assumption cache.
void setWideningDecision(Instruction *I, ElementCount VF, InstWidening W, InstructionCost Cost)
Save vectorization decision W and Cost taken by the cost model for instruction I and vector width VF.
InstWidening
Decision that was taken during cost calculation for memory instruction.
bool isScalarWithPredication(Instruction *I, ElementCount VF) const
Returns true if I is an instruction which requires predication and for which our chosen predication s...
InstructionCost getVectorCallCost(CallInst *CI, ElementCount VF) const
Estimate cost of a call instruction CI if it were vectorized with factor VF.
bool useOrderedReductions(const RecurrenceDescriptor &RdxDesc) const
Returns true if we should use strict in-order reductions for the given RdxDesc.
std::pair< InstructionCost, InstructionCost > getDivRemSpeculationCost(Instruction *I, ElementCount VF) const
Return the costs for our two available strategies for lowering a div/rem operation which requires spe...
bool isDivRemScalarWithPredication(InstructionCost ScalarCost, InstructionCost SafeDivisorCost) const
Given costs for both strategies, return true if the scalar predication lowering should be used for di...
InstructionCost expectedCost(ElementCount VF)
Returns the expected execution cost.
void setCostBasedWideningDecision(ElementCount VF)
Memory access instruction may be vectorized in more than one way.
InstWidening getWideningDecision(Instruction *I, ElementCount VF) const
Return the cost model decision for the given instruction I and vector width VF.
FixedScalableVFPair MaxPermissibleVFWithoutMaxBW
The highest VF possible for this loop, without using MaxBandwidth.
bool isScalarEpilogueAllowed() const
Returns true if a scalar epilogue is not allowed due to optsize or a loop hint annotation.
InstructionCost getWideningCost(Instruction *I, ElementCount VF)
Return the vectorization cost for the given instruction I and vector width VF.
void collectInstsToScalarize(ElementCount VF)
Collects the instructions to scalarize for each predicated instruction in the loop.
LoopVectorizationLegality checks if it is legal to vectorize a loop, and to what vectorization factor...
MapVector< PHINode *, InductionDescriptor > InductionList
InductionList saves induction variables and maps them to the induction descriptor.
const SmallPtrSetImpl< const Instruction * > & getPotentiallyFaultingLoads() const
Returns potentially faulting loads.
bool canVectorize(bool UseVPlanNativePath)
Returns true if it is legal to vectorize this loop.
bool canVectorizeFPMath(bool EnableStrictReductions)
Returns true if it is legal to vectorize the FP math operations in this loop.
PHINode * getPrimaryInduction()
Returns the primary induction variable.
const SmallVector< BasicBlock *, 4 > & getCountableExitingBlocks() const
Returns all exiting blocks with a countable exit, i.e.
const InductionList & getInductionVars() const
Returns the induction variables found in the loop.
bool hasUncountableEarlyExit() const
Returns true if the loop has exactly one uncountable early exit, i.e.
bool hasHistograms() const
Returns a list of all known histogram operations in the loop.
const LoopAccessInfo * getLAI() const
Planner drives the vectorization process after having passed Legality checks.
VectorizationFactor selectEpilogueVectorizationFactor(const ElementCount MaxVF, unsigned IC)
VPlan & getPlanFor(ElementCount VF) const
Return the VPlan for VF.
VectorizationFactor planInVPlanNativePath(ElementCount UserVF)
Use the VPlan-native path to plan how to best vectorize, return the best VF and its cost.
void updateLoopMetadataAndProfileInfo(Loop *VectorLoop, VPBasicBlock *HeaderVPBB, const VPlan &Plan, bool VectorizingEpilogue, MDNode *OrigLoopID, std::optional< unsigned > OrigAverageTripCount, unsigned OrigLoopInvocationWeight, unsigned EstimatedVFxUF, bool DisableRuntimeUnroll)
Update loop metadata and profile info for both the scalar remainder loop and VectorLoop,...
void buildVPlans(ElementCount MinVF, ElementCount MaxVF)
Build VPlans for power-of-2 VF's between MinVF and MaxVF inclusive, according to the information gath...
VectorizationFactor computeBestVF()
Compute and return the most profitable vectorization factor.
DenseMap< const SCEV *, Value * > executePlan(ElementCount VF, unsigned UF, VPlan &BestPlan, InnerLoopVectorizer &LB, DominatorTree *DT, bool VectorizingEpilogue)
Generate the IR code for the vectorized loop captured in VPlan BestPlan according to the best selecte...
unsigned selectInterleaveCount(VPlan &Plan, ElementCount VF, InstructionCost LoopCost)
void emitInvalidCostRemarks(OptimizationRemarkEmitter *ORE)
Emit remarks for recipes with invalid costs in the available VPlans.
static bool getDecisionAndClampRange(const std::function< bool(ElementCount)> &Predicate, VFRange &Range)
Test a Predicate on a Range of VF's.
void printPlans(raw_ostream &O)
void plan(ElementCount UserVF, unsigned UserIC)
Build VPlans for the specified UserVF and UserIC if they are non-zero or all applicable candidate VFs...
void addMinimumIterationCheck(VPlan &Plan, ElementCount VF, unsigned UF, ElementCount MinProfitableTripCount) const
Create a check to Plan to see if the vector loop should be executed based on its trip count.
bool hasPlanWithVF(ElementCount VF) const
Look through the existing plans and return true if we have one with vectorization factor VF.
This holds vectorization requirements that must be verified late in the process.
Instruction * getExactFPInst()
Utility class for getting and setting loop vectorizer hints in the form of loop metadata.
bool isScalableVectorizationDisabled() const
enum ForceKind getForce() const
bool allowVectorization(Function *F, Loop *L, bool VectorizeOnlyWhenForced) const
void emitRemarkWithHints() const
Dumps all the hint information.
bool isPotentiallyUnsafe() const
ElementCount getWidth() const
@ FK_Enabled
Forcing enabled.
@ FK_Undefined
Not selected.
@ FK_Disabled
Forcing disabled.
unsigned getPredicate() const
const char * vectorizeAnalysisPassName() const
If hints are provided that force vectorization, use the AlwaysPrint pass name to force the frontend t...
unsigned getInterleave() const
This class emits a version of the loop where run-time checks ensure that may-alias pointers can't ove...
Represents a single loop in the control flow graph.
DebugLoc getStartLoc() const
Return the debug location of the start of this loop.
This class implements a map that also provides access to all stored values in a deterministic order.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Function * getFunction(StringRef Name) const
Look up the specified function in the module symbol table.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
op_range incoming_values()
void setIncomingValueForBlock(const BasicBlock *BB, Value *V)
Set every incoming value(s) for block BB to V.
Value * getIncomingValueForBlock(const BasicBlock *BB) const
unsigned getNumIncomingValues() const
Return the number of incoming edges.
An interface layer with SCEV used to manage how we see SCEV expressions for values in the context of ...
ScalarEvolution * getSE() const
Returns the ScalarEvolution analysis used.
LLVM_ABI const SCEVPredicate & getPredicate() const
LLVM_ABI unsigned getSmallConstantMaxTripCount()
Returns the upper bound of the loop trip count as a normal unsigned value, or 0 if the trip count is ...
LLVM_ABI const SCEV * getBackedgeTakenCount()
Get the (predicated) backedge count for the analyzed loop.
LLVM_ABI const SCEV * getSCEV(Value *V)
Returns the SCEV expression of V, in the context of the current SCEV predicate.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
PreservedAnalyses & preserveSet()
Mark an analysis set as preserved.
PreservedAnalyses & preserve()
Mark an analysis as preserved.
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
Analysis providing profile information.
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
static bool isFMulAddIntrinsic(Instruction *I)
Returns true if the instruction is a call to the llvm.fmuladd intrinsic.
FastMathFlags getFastMathFlags() const
Instruction * getLoopExitInstr() const
static LLVM_ABI unsigned getOpcode(RecurKind Kind)
Returns the opcode corresponding to the RecurrenceKind.
Type * getRecurrenceType() const
Returns the type of the recurrence.
const SmallPtrSet< Instruction *, 8 > & getCastInsts() const
Returns a reference to the instructions used for type-promoting the recurrence.
unsigned getMinWidthCastToRecurrenceTypeInBits() const
Returns the minimum width used by the recurrence in bits.
TrackingVH< Value > getRecurrenceStartValue() const
LLVM_ABI SmallVector< Instruction *, 4 > getReductionOpChain(PHINode *Phi, Loop *L) const
Attempts to find a chain of operations from Phi to LoopExitInst that can be treated as a set of reduc...
static bool isAnyOfRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
bool isSigned() const
Returns true if all source operands of the recurrence are SExtInsts.
RecurKind getRecurrenceKind() const
bool isOrdered() const
Expose an ordered FP reduction to the instance users.
static LLVM_ABI bool isFloatingPointRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is a floating point kind.
static bool isFindIVRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
Value * getSentinelValue() const
Returns the sentinel value for FindFirstIV & FindLastIV recurrences to replace the start value.
static bool isMinMaxRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is any min/max kind.
std::optional< ArrayRef< PointerDiffInfo > > getDiffChecks() const
const SmallVectorImpl< RuntimePointerCheck > & getChecks() const
Returns the checks that generateChecks created.
This class uses information about analyze scalars to rewrite expressions in canonical form.
ScalarEvolution * getSE()
bool isInsertedInstruction(Instruction *I) const
Return true if the specified instruction was inserted by the code rewriter.
LLVM_ABI Value * expandCodeForPredicate(const SCEVPredicate *Pred, Instruction *Loc)
Generates a code sequence that evaluates this predicate.
void eraseDeadInstructions(Value *Root)
Remove inserted instructions that are dead, e.g.
virtual bool isAlwaysTrue() const =0
Returns true if the predicate is always true.
This class represents an analyzed expression in the program.
LLVM_ABI bool isZero() const
Return true if the expression is a constant zero.
LLVM_ABI Type * getType() const
Return the LLVM type of this SCEV expression.
Analysis pass that exposes the ScalarEvolution for a function.
The main scalar evolution driver.
LLVM_ABI const SCEV * getURemExpr(const SCEV *LHS, const SCEV *RHS)
Represents an unsigned remainder expression based on unsigned division.
LLVM_ABI const SCEV * getBackedgeTakenCount(const Loop *L, ExitCountKind Kind=Exact)
If the specified loop has a predictable backedge-taken count, return it, otherwise return a SCEVCould...
LLVM_ABI const SCEV * getConstant(ConstantInt *V)
LLVM_ABI const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
LLVM_ABI const SCEV * getTripCountFromExitCount(const SCEV *ExitCount)
A version of getTripCountFromExitCount below which always picks an evaluation type which can not resu...
const SCEV * getOne(Type *Ty)
Return a SCEV for the constant 1 of a specific type.
LLVM_ABI void forgetLoop(const Loop *L)
This method should be called by the client when it has changed a loop in a way that may effect Scalar...
LLVM_ABI bool isLoopInvariant(const SCEV *S, const Loop *L)
Return true if the value of the given SCEV is unchanging in the specified loop.
LLVM_ABI bool isSCEVable(Type *Ty) const
Test if values of the given type are analyzable within the SCEV framework.
LLVM_ABI const SCEV * getElementCount(Type *Ty, ElementCount EC, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
LLVM_ABI void forgetValue(Value *V)
This method should be called by the client when it has changed a value in a way that may effect its v...
LLVM_ABI void forgetBlockAndLoopDispositions(Value *V=nullptr)
Called when the client has changed the disposition of values in a loop or block.
const SCEV * getMinusOne(Type *Ty)
Return a SCEV for the constant -1 of a specific type.
LLVM_ABI void forgetLcssaPhiWithNewPredecessor(Loop *L, PHINode *V)
Forget LCSSA phi node V of loop L to which a new predecessor was added, such that it may no longer be...
LLVM_ABI unsigned getSmallConstantTripCount(const Loop *L)
Returns the exact trip count of the loop if we can compute it, and the result is a small constant.
APInt getUnsignedRangeMax(const SCEV *S)
Determine the max of the unsigned range for a particular SCEV.
LLVM_ABI const SCEV * applyLoopGuards(const SCEV *Expr, const Loop *L)
Try to apply information from loop guards for L to Expr.
LLVM_ABI const SCEV * getAddExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap, unsigned Depth=0)
Get a canonical add expression, or something simpler if possible.
LLVM_ABI bool isKnownPredicate(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
Test if the given expression is known to satisfy the condition described by Pred, LHS,...
This class represents the LLVM 'select' instruction.
A vector that has set insertion semantics.
size_type size() const
Determine the number of elements in the SetVector.
void insert_range(Range &&R)
size_type count(const key_type &key) const
Count the number of elements of a given key in the SetVector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
Analysis pass providing the TargetTransformInfo.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
This class represents a truncation of integer types.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
This class implements a switch-like dispatch statement for a value of 'T' using dyn_cast functionalit...
TypeSwitch< T, ResultT > & Case(CallableT &&caseFn)
Add a case on the given type.
The instances of the Type class are immutable: once they are created, they are never changed.
LLVM_ABI unsigned getIntegerBitWidth() const
bool isVectorTy() const
True if this is an instance of VectorType.
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
static LLVM_ABI Type * getVoidTy(LLVMContext &C)
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static LLVM_ABI IntegerType * getInt1Ty(LLVMContext &C)
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isVoidTy() const
Return true if this is 'void'.
A Use represents the edge between a Value definition and its users.
LLVM_ABI bool replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
iterator_range< op_iterator > op_range
Value * getOperand(unsigned i) const
static SmallVector< VFInfo, 8 > getMappings(const CallInst &CI)
Retrieve all the VFInfo instances associated to the CallInst CI.
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
void appendRecipe(VPRecipeBase *Recipe)
Augment the existing recipes of a VPBasicBlock with an additional Recipe as the last recipe.
RecipeListTy::iterator iterator
Instruction iterators...
iterator begin()
Recipe iterator methods.
iterator_range< iterator > phis()
Returns an iterator range over the PHI-like recipes in the block.
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
VPRegionBlock * getEnclosingLoopRegion()
void insert(VPRecipeBase *Recipe, iterator InsertPt)
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
VPRegionBlock * getParent()
const VPBasicBlock * getExitingBasicBlock() const
void setName(const Twine &newName)
size_t getNumSuccessors() const
void swapSuccessors()
Swap successors of the block. The block must have exactly 2 successors.
size_t getNumPredecessors() const
VPBlockBase * getSinglePredecessor() const
const VPBasicBlock * getEntryBasicBlock() const
VPBlockBase * getSingleSuccessor() const
const VPBlocksTy & getSuccessors() const
static auto blocksOnly(const T &Range)
Return an iterator range over Range which only includes BlockTy blocks.
static void insertOnEdge(VPBlockBase *From, VPBlockBase *To, VPBlockBase *BlockPtr)
Inserts BlockPtr on the edge between From and To.
static void connectBlocks(VPBlockBase *From, VPBlockBase *To, unsigned PredIdx=-1u, unsigned SuccIdx=-1u)
Connect VPBlockBases From and To bi-directionally.
static void reassociateBlocks(VPBlockBase *Old, VPBlockBase *New)
Reassociate all the blocks connected to Old so that they now point to New.
VPlan-based builder utility analogous to IRBuilder.
VPDerivedIVRecipe * createDerivedIV(InductionDescriptor::InductionKind Kind, FPMathOperator *FPBinOp, VPValue *Start, VPValue *Current, VPValue *Step, const Twine &Name="")
Convert the input value Current to the corresponding value of an induction with Start and Step values...
VPPhi * createScalarPhi(ArrayRef< VPValue * > IncomingValues, DebugLoc DL, const Twine &Name="")
VPInstruction * createNaryOp(unsigned Opcode, ArrayRef< VPValue * > Operands, Instruction *Inst=nullptr, const Twine &Name="")
Create an N-ary operation with Opcode, Operands and set Inst as its underlying Instruction.
VPInstruction * createScalarCast(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy, DebugLoc DL)
unsigned getNumDefinedValues() const
Returns the number of values defined by the VPDef.
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
void execute(VPTransformState &State) override
Generate the transformed value of the induction at offset StartValue (1.
VPValue * getStepValue() const
VPValue * getStartValue() const
A recipe representing a sequence of load -> update -> store as part of a histogram operation.
A special type of VPBasicBlock that wraps an existing IR basic block.
This is a concrete Recipe that models a single VPlan-level instruction.
@ ComputeAnyOfResult
Compute the final result of a AnyOf reduction with select(cmp(),x,y), where one of (x,...
@ ExtractPenultimateElement
@ ResumeForEpilogue
Explicit user for the resume phi of the canonical induction in the main VPlan, used by the epilogue v...
@ FirstOrderRecurrenceSplice
@ ReductionStartVector
Start vector for reductions with 3 operands: the original start value, the identity value for the red...
unsigned getOpcode() const
VPInterleaveRecipe is a recipe for transforming an interleave group of load or stores into one wide l...
In what follows, the term "input IR" refers to code that is fed into the vectorizer whereas the term ...
A recipe for forming partial reductions.
detail::zippy< llvm::detail::zip_first, VPUser::const_operand_range, const_incoming_blocks_range > incoming_values_and_blocks() const
Returns an iterator range over pairs of incoming values and corresponding incoming blocks.
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
VPBasicBlock * getParent()
DebugLoc getDebugLoc() const
Returns the debug location of the recipe.
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Helper class to create VPRecipies from IR instructions.
VPRecipeBase * tryToCreateWidenRecipe(VPSingleDefRecipe *R, VFRange &Range)
Create and return a widened recipe for R if one can be created within the given VF Range.
VPValue * getBlockInMask(VPBasicBlock *VPBB) const
Returns the entry mask for block VPBB or null if the mask is all-true.
VPValue * getVPValueOrAddLiveIn(Value *V)
std::optional< unsigned > getScalingForReduction(const Instruction *ExitInst)
void collectScaledReductions(VFRange &Range)
Find all possible partial reductions in the loop and track all of those that are valid so recipes can...
VPReplicateRecipe * handleReplication(Instruction *I, ArrayRef< VPValue * > Operands, VFRange &Range)
Build a VPReplicationRecipe for I using Operands.
VPRecipeBase * tryToCreatePartialReduction(Instruction *Reduction, ArrayRef< VPValue * > Operands, unsigned ScaleFactor)
Create and return a partial reduction recipe for a reduction instruction along with binary operation ...
A recipe for handling reduction phis.
bool isInLoop() const
Returns true, if the phi is part of an in-loop reduction.
RecurKind getRecurrenceKind() const
Returns the recurrence kind of the reduction.
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
const VPBlockBase * getEntry() const
VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...
VPSingleDef is a base class for recipes for modeling a sequence of one or more output IR that define ...
Instruction * getUnderlyingInstr()
Returns the underlying instruction.
An analysis for type-inference for VPValues.
Type * inferScalarType(const VPValue *V)
Infer the type of V. Returns the scalar type of V.
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
void setOperand(unsigned I, VPValue *New)
VPValue * getOperand(unsigned N) const
void addOperand(VPValue *Operand)
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
Value * getLiveInIRValue() const
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
Value * getUnderlyingValue() const
Return the underlying Value attached to this VPValue.
void replaceAllUsesWith(VPValue *New)
user_iterator user_begin()
unsigned getNumUsers() const
void replaceUsesWithIf(VPValue *New, llvm::function_ref< bool(VPUser &U, unsigned Idx)> ShouldReplace)
Go through the uses list for this VPValue and make each use point to New if the callback ShouldReplac...
A recipe to compute a pointer to the last element of each part of a widened memory access for widened...
VPWidenCastRecipe is a recipe to create vector cast instructions.
A recipe for handling GEP instructions.
Base class for widened induction (VPWidenIntOrFpInductionRecipe and VPWidenPointerInductionRecipe),...
VPValue * getStepValue()
Returns the step value of the induction.
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
A recipe for handling phi nodes of integer and floating-point inductions, producing their vector valu...
A common base class for widening memory operations.
A recipe for widened phis.
VPWidenRecipe is a recipe for producing a widened instruction using the opcode and operands of the re...
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
bool hasVF(ElementCount VF) const
VPBasicBlock * getEntry()
VPValue & getVectorTripCount()
The vector trip count.
VPValue & getVF()
Returns the VF of the vector loop region.
VPValue * getTripCount() const
The trip count of the original loop.
iterator_range< SmallSetVector< ElementCount, 2 >::iterator > vectorFactors() const
Returns an iterator range over all VFs of the plan.
bool hasUF(unsigned UF) const
ArrayRef< VPIRBasicBlock * > getExitBlocks() const
Return an ArrayRef containing VPIRBasicBlocks wrapping the exit blocks of the original scalar loop.
LLVM_ABI_FOR_TEST VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
bool hasEarlyExit() const
Returns true if the VPlan is based on a loop with an early exit.
InstructionCost cost(ElementCount VF, VPCostContext &Ctx)
Return the cost of this plan.
void resetTripCount(VPValue *NewTripCount)
Resets the trip count for the VPlan.
VPBasicBlock * getMiddleBlock()
Returns the 'middle' block of the plan, that is the block that selects whether to execute the scalar ...
VPValue * getOrAddLiveIn(Value *V)
Gets the live-in VPValue for V or adds a new live-in (if none exists yet) for V.
bool hasScalarVFOnly() const
VPBasicBlock * getScalarPreheader() const
Return the VPBasicBlock for the preheader of the scalar loop.
void execute(VPTransformState *State)
Generate the IR code for this VPlan.
VPCanonicalIVPHIRecipe * getCanonicalIV()
Returns the canonical induction recipe of the vector loop.
VPIRBasicBlock * getScalarHeader() const
Return the VPIRBasicBlock wrapping the header of the scalar loop.
VPBasicBlock * getVectorPreheader()
Returns the preheader of the vector loop region, if one exists, or null otherwise.
VPlan * duplicate()
Clone the current VPlan, update all VPValues of the new VPlan and cloned recipes to refer to the clon...
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
LLVM_ABI bool hasOneUser() const
Return true if there is exactly one user of this value.
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
iterator_range< user_iterator > users()
LLVM_ABI LLVMContext & getContext() const
All values hold a context through their type.
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
std::pair< iterator, bool > insert(const ValueT &V)
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
constexpr bool hasKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns true if there exists a value X where RHS.multiplyCoefficientBy(X) will result in a value whos...
constexpr ScalarTy getFixedValue() const
static constexpr bool isKnownLE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
constexpr bool isNonZero() const
constexpr ScalarTy getKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns a value X where RHS.multiplyCoefficientBy(X) will result in a value whose quantity matches ou...
static constexpr bool isKnownLT(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr LeafTy multiplyCoefficientBy(ScalarTy RHS) const
constexpr bool isFixed() const
Returns true if the quantity is not scaled by vscale.
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr bool isZero() const
static constexpr bool isKnownGT(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
static constexpr bool isKnownGE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
self_iterator getIterator()
This class implements an extremely fast bulk output stream that can only output to a stream.
A raw_ostream that writes to an std::string.
This provides a very simple, boring adaptor for a begin and end iterator into a range type.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ PredicateElseScalarEpilogue
@ PredicateOrDontVectorize
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
@ C
The default llvm calling convention, compatible with C.
@ BasicBlock
Various leaf nodes.
std::variant< std::monostate, Loc::Single, Loc::Multi, Loc::MMI, Loc::EntryValue > Variant
Alias for the std::variant specialization base class of DbgVariable.
SpecificConstantMatch m_ZeroInt()
Convenience matchers for specific integer values.
BinaryOp_match< SpecificConstantMatch, SrcTy, TargetOpcode::G_SUB > m_Neg(const SrcTy &&Src)
Matches a register negated by a G_SUB.
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
class_match< BinaryOperator > m_BinOp()
Match an arbitrary binary operation and ignore it.
OneOps_match< OpTy, Instruction::Freeze > m_Freeze(const OpTy &Op)
Matches FreezeInst.
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
bool match(Val *V, const Pattern &P)
bind_ty< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
BinaryOp_match< LHS, RHS, Instruction::Mul > m_Mul(const LHS &L, const RHS &R)
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
SpecificCmpClass_match< LHS, RHS, ICmpInst > m_SpecificICmp(CmpPredicate MatchPred, const LHS &L, const RHS &R)
class_match< CmpInst > m_Cmp()
Matches any compare instruction and ignore it.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
match_combine_or< CastInst_match< OpTy, ZExtInst >, CastInst_match< OpTy, SExtInst > > m_ZExtOrSExt(const OpTy &Op)
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
MatchFunctor< Val, Pattern > match_fn(const Pattern &P)
A match functor that can be used as a UnaryPredicate in functional algorithms like all_of.
class_match< const SCEVVScale > m_SCEVVScale()
bind_cst_ty m_scev_APInt(const APInt *&C)
Match an SCEV constant and bind it to an APInt.
specificloop_ty m_SpecificLoop(const Loop *L)
cst_pred_ty< is_specific_signed_cst > m_scev_SpecificSInt(int64_t V)
Match an SCEV constant with a plain signed integer (sign-extended value will be matched)
SCEVAffineAddRec_match< Op0_t, Op1_t, class_match< const Loop > > m_scev_AffineAddRec(const Op0_t &Op0, const Op1_t &Op1)
SCEVBinaryExpr_match< SCEVMulExpr, Op0_t, Op1_t > m_scev_Mul(const Op0_t &Op0, const Op1_t &Op1)
bool match(const SCEV *S, const Pattern &P)
class_match< const SCEV > m_SCEV()
match_combine_or< AllRecipe_match< Instruction::ZExt, Op0_t >, AllRecipe_match< Instruction::SExt, Op0_t > > m_ZExtOrSExt(const Op0_t &Op0)
VPInstruction_match< VPInstruction::ExtractLastElement, Op0_t > m_ExtractLastElement(const Op0_t &Op0)
class_match< VPValue > m_VPValue()
Match an arbitrary VPValue and ignore it.
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
Add a small namespace to avoid name clashes with the classes used in the streaming interface.
DiagnosticInfoOptimizationBase::Argument NV
NodeAddr< InstrNode * > Instr
NodeAddr< PhiNode * > Phi
friend class Instruction
Iterator for Instructions in a `BasicBlock.
bool isSingleScalar(const VPValue *VPV)
Returns true if VPV is a single scalar, either because it produces the same value for all lanes or on...
VPValue * getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr)
Get or create a VPValue that corresponds to the expansion of Expr.
VPBasicBlock * getFirstLoopHeader(VPlan &Plan, VPDominatorTree &VPDT)
Returns the header block of the first, top-level loop, or null if none exist.
const SCEV * getSCEVExprForVPValue(VPValue *V, ScalarEvolution &SE)
Return the SCEV expression for V.
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI bool simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI, ScalarEvolution *SE, AssumptionCache *AC, MemorySSAUpdater *MSSAU, bool PreserveLCSSA)
Simplify each loop in a loop nest recursively.
LLVM_ABI void ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI, Instruction *I)
Replace the instruction specified by BI with the instruction specified by I.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
FunctionAddr VTableAddr Value
LLVM_ABI Value * addRuntimeChecks(Instruction *Loc, Loop *TheLoop, const SmallVectorImpl< RuntimePointerCheck > &PointerChecks, SCEVExpander &Expander, bool HoistRuntimeChecks=false)
Add code that checks at runtime if the accessed arrays in PointerChecks overlap.
auto cast_if_present(const Y &Val)
cast_if_present<X> - Functionally identical to cast, except that a null value is accepted.
LLVM_ABI bool RemoveRedundantDbgInstrs(BasicBlock *BB)
Try to remove redundant dbg.value instructions from given basic block.
cl::opt< bool > VerifyEachVPlan
LLVM_ABI std::optional< unsigned > getLoopEstimatedTripCount(Loop *L, unsigned *EstimatedLoopInvocationWeight=nullptr)
Return either:
static void reportVectorization(OptimizationRemarkEmitter *ORE, Loop *TheLoop, VectorizationFactor VF, unsigned IC)
Report successful vectorization of the loop.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
unsigned getLoadStoreAddressSpace(const Value *I)
A helper function that returns the address space of the pointer operand of load or store instruction.
LLVM_ABI Intrinsic::ID getMinMaxReductionIntrinsicOp(Intrinsic::ID RdxID)
Returns the min/max intrinsic used when expanding a min/max reduction.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
LLVM_ABI_FOR_TEST bool verifyVPlanIsValid(const VPlan &Plan, bool VerifyLate=false)
Verify invariants for general VPlans.
LLVM_ABI Intrinsic::ID getVectorIntrinsicIDForCall(const CallInst *CI, const TargetLibraryInfo *TLI)
Returns intrinsic ID for call.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
LLVM_ABI bool verifyFunction(const Function &F, raw_ostream *OS=nullptr)
Check a function for errors, useful for use when debugging a pass.
const Value * getLoadStorePointerOperand(const Value *V)
A helper function that returns the pointer operand of a load or store instruction.
OuterAnalysisManagerProxy< ModuleAnalysisManager, Function > ModuleAnalysisManagerFunctionProxy
Provide the ModuleAnalysisManager to Function proxy.
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
LLVM_ABI bool formLCSSARecursively(Loop &L, const DominatorTree &DT, const LoopInfo *LI, ScalarEvolution *SE)
Put a loop nest into LCSSA form.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
LLVM_ABI bool shouldOptimizeForSize(const MachineFunction *MF, ProfileSummaryInfo *PSI, const MachineBlockFrequencyInfo *BFI, PGSOQueryType QueryType=PGSOQueryType::Other)
Returns true if machine function MF is suggested to be size-optimized based on the profile.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Align getLoadStoreAlignment(const Value *I)
A helper function that returns the alignment of load or store instruction.
iterator_range< df_iterator< VPBlockShallowTraversalWrapper< VPBlockBase * > > > vp_depth_first_shallow(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order.
LLVM_ABI bool isSafeToSpeculativelyExecute(const Instruction *I, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr, bool UseVariableInfo=true, bool IgnoreUBImplyingAttrs=true)
Return true if the instruction does not have any effects besides calculating the result and does not ...
bool isa_and_nonnull(const Y &Val)
iterator_range< df_iterator< VPBlockDeepTraversalWrapper< VPBlockBase * > > > vp_depth_first_deep(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order while traversing t...
SmallVector< VPRegisterUsage, 8 > calculateRegisterUsageForPlan(VPlan &Plan, ArrayRef< ElementCount > VFs, const TargetTransformInfo &TTI, const SmallPtrSetImpl< const Value * > &ValuesToIgnore)
Estimate the register usage for Plan and vectorization factors in VFs by calculating the highest numb...
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
auto dyn_cast_or_null(const Y &Val)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
void collectEphemeralRecipesForVPlan(VPlan &Plan, DenseSet< VPRecipeBase * > &EphRecipes)
auto reverse(ContainerTy &&C)
LLVM_ABI void setBranchWeights(Instruction &I, ArrayRef< uint32_t > Weights, bool IsExpected)
Create a new branch_weights metadata node and add or overwrite a prof metadata reference to instructi...
bool containsIrreducibleCFG(RPOTraversalT &RPOTraversal, const LoopInfoT &LI)
Return true if the control flow in RPOTraversal is irreducible.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
void sort(IteratorTy Start, IteratorTy End)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
LLVM_ABI cl::opt< bool > EnableLoopVectorization
LLVM_ABI bool wouldInstructionBeTriviallyDead(const Instruction *I, const TargetLibraryInfo *TLI=nullptr)
Return true if the result produced by the instruction would have no side effects if it was not used.
FunctionAddr VTableAddr Count
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
Type * toVectorizedTy(Type *Ty, ElementCount EC)
A helper for converting to vectorized types.
LLVM_ABI void llvm_unreachable_internal(const char *msg=nullptr, const char *file=nullptr, unsigned line=0)
This function calls abort(), and prints the optional message to stderr.
T * find_singleton(R &&Range, Predicate P, bool AllowRepeats=false)
Return the single value in Range that satisfies P(<member of Range> *, AllowRepeats)->T * returning n...
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
cl::opt< unsigned > ForceTargetInstructionCost
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
bool canVectorizeTy(Type *Ty)
Returns true if Ty is a valid vector element type, void, or an unpacked literal struct where all elem...
static void reportVectorizationInfo(const StringRef Msg, const StringRef ORETag, OptimizationRemarkEmitter *ORE, Loop *TheLoop, Instruction *I=nullptr, DebugLoc DL={})
Reports an informative message: print Msg for debugging purposes as well as an optimization remark.
LLVM_ABI bool isAssignmentTrackingEnabled(const Module &M)
Return true if assignment tracking is enabled for module M.
RecurKind
These are the kinds of recurrences that we support.
@ Or
Bitwise or logical OR of integers.
@ FMulAdd
Sum of float products with llvm.fmuladd(a * b + sum).
@ Sub
Subtraction of integers.
@ AddChainWithSubs
A chain of adds and subs.
LLVM_ABI Value * getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF)
Given information about an recurrence kind, return the identity for the @llvm.vector....
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
LLVM_ABI void reportVectorizationFailure(const StringRef DebugMsg, const StringRef OREMsg, const StringRef ORETag, OptimizationRemarkEmitter *ORE, Loop *TheLoop, Instruction *I=nullptr)
Reports a vectorization failure: print DebugMsg for debugging purposes along with the corresponding o...
DWARFExpression::Operation Op
@ CM_ScalarEpilogueNotAllowedLowTripLoop
@ CM_ScalarEpilogueNotNeededUsePredicate
@ CM_ScalarEpilogueNotAllowedOptSize
@ CM_ScalarEpilogueAllowed
@ CM_ScalarEpilogueNotAllowedUsePredicate
LLVM_ABI bool isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Return true if this function can prove that V does not have undef bits and is never poison.
ArrayRef(const T &OneElt) -> ArrayRef< T >
Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)
Return a value for Step multiplied by VF.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
LLVM_ABI BasicBlock * SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="", bool Before=false)
Split the specified block at the specified instruction.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
auto predecessors(const MachineBasicBlock *BB)
iterator_range< pointer_iterator< WrappedIteratorT > > make_pointer_range(RangeT &&Range)
cl::opt< bool > EnableVPlanNativePath
Type * getLoadStoreType(const Value *I)
A helper function that returns the type of a load or store instruction.
ArrayRef< Type * > getContainedTypes(Type *const &Ty)
Returns the types contained in Ty.
LLVM_ABI Value * addDiffRuntimeChecks(Instruction *Loc, ArrayRef< PointerDiffInfo > Checks, SCEVExpander &Expander, function_ref< Value *(IRBuilderBase &, unsigned)> GetVF, unsigned IC)
bool pred_empty(const BasicBlock *BB)
@ DataAndControlFlowWithoutRuntimeCheck
Use predicate to control both data and control flow, but modify the trip count so that a runtime over...
@ None
Don't use tail folding.
@ DataWithEVL
Use predicated EVL instructions for tail-folding.
@ DataAndControlFlow
Use predicate to control both data and control flow.
@ DataWithoutLaneMask
Same as Data, but avoids using the get.active.lane.mask intrinsic to calculate the mask and instead i...
@ Data
Use predicate only to mask operations on data in the loop.
unsigned getPredBlockCostDivisor(TargetTransformInfo::TargetCostKind CostKind)
A helper function that returns how much we should divide the cost of a predicated block by.
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
LLVM_ABI bool hasBranchWeightMD(const Instruction &I)
Checks if an instructions has Branch Weight Metadata.
hash_code hash_combine(const Ts &...args)
Combine values into a single hash_code.
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
Type * toVectorTy(Type *Scalar, ElementCount EC)
A helper function for converting Scalar types to vector types.
std::unique_ptr< VPlan > VPlanPtr
constexpr detail::IsaCheckPredicate< Types... > IsaPred
Function object wrapper for the llvm::isa type check.
LLVM_ABI MapVector< Instruction *, uint64_t > computeMinimumValueSizes(ArrayRef< BasicBlock * > Blocks, DemandedBits &DB, const TargetTransformInfo *TTI=nullptr)
Compute a map of integer instructions to their minimum legal type size.
hash_code hash_combine_range(InputIteratorT first, InputIteratorT last)
Compute a hash_code for a sequence of values.
LLVM_ABI cl::opt< bool > EnableLoopInterleaving
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
A special type used by analysis passes to provide an address that identifies that particular analysis...
static LLVM_ABI void collectEphemeralValues(const Loop *L, AssumptionCache *AC, SmallPtrSetImpl< const Value * > &EphValues)
Collect a loop's ephemeral values (those used only by an assume or similar intrinsics in the loop).
An information struct used to provide DenseMap with the various necessary components for a given valu...
Encapsulate information regarding vectorization of a loop and its epilogue.
EpilogueLoopVectorizationInfo(ElementCount MVF, unsigned MUF, ElementCount EVF, unsigned EUF, VPlan &EpiloguePlan)
BasicBlock * MainLoopIterationCountCheck
BasicBlock * EpilogueIterationCountCheck
A class that represents two vectorization factors (initialized with 0 by default).
static FixedScalableVFPair getNone()
This holds details about a histogram operation – a load -> update -> store sequence where each lane i...
Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...
std::optional< unsigned > MaskPos
LLVM_ABI LoopVectorizeResult runImpl(Function &F)
LLVM_ABI bool processLoop(Loop *L)
LoopAccessInfoManager * LAIs
LLVM_ABI void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
LLVM_ABI LoopVectorizePass(LoopVectorizeOptions Opts={})
LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
OptimizationRemarkEmitter * ORE
TargetTransformInfo * TTI
Storage for information about made changes.
A chain of instructions that form a partial reduction.
Instruction * Reduction
The top-level binary operation that forms the reduction to a scalar after the loop body.
Instruction * ExtendA
The extension of each of the inner binary operation's operands.
A CRTP mix-in to automatically provide informational APIs needed for passes.
Holds the VFShape for a specific scalar to vector function mapping.
std::optional< unsigned > getParamIndexForOptionalMask() const
Instruction Set Architecture.
Encapsulates information needed to describe a parameter.
A range of powers-of-2 vectorization factors with fixed start and adjustable end.
Struct to hold various analysis needed for cost computations.
LoopVectorizationCostModel & CM
bool isLegacyUniformAfterVectorization(Instruction *I, ElementCount VF) const
Return true if I is considered uniform-after-vectorization in the legacy cost model for VF.
bool skipCostComputation(Instruction *UI, bool IsVector) const
Return true if the cost for UI shouldn't be computed, e.g.
InstructionCost getLegacyCost(Instruction *UI, ElementCount VF) const
Return the cost for UI with VF using the legacy cost model as fallback until computing the cost of al...
SmallPtrSet< Instruction *, 8 > SkipCostComputation
A recipe for handling first-order recurrence phis.
A struct that represents some properties of the register usage of a loop.
A recipe for widening select instructions.
TODO: The following VectorizationFactor was pulled out of LoopVectorizationCostModel class.
InstructionCost Cost
Cost of the loop with that width.
ElementCount MinProfitableTripCount
The minimum trip count required to make vectorization profitable, e.g.
ElementCount Width
Vector width with best cost.
InstructionCost ScalarCost
Cost of the scalar loop.
static VectorizationFactor Disabled()
Width 1 means no vectorization, cost 0 means uncomputed cost.
static LLVM_ABI bool HoistRuntimeChecks