Thanks to visit codestin.com
Credit goes to llvm.org

LLVM 22.0.0git
LoopVectorize.cpp
Go to the documentation of this file.
1//===- LoopVectorize.cpp - A Loop Vectorizer ------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This is the LLVM loop vectorizer. This pass modifies 'vectorizable' loops
10// and generates target-independent LLVM-IR.
11// The vectorizer uses the TargetTransformInfo analysis to estimate the costs
12// of instructions in order to estimate the profitability of vectorization.
13//
14// The loop vectorizer combines consecutive loop iterations into a single
15// 'wide' iteration. After this transformation the index is incremented
16// by the SIMD vector width, and not by one.
17//
18// This pass has three parts:
19// 1. The main loop pass that drives the different parts.
20// 2. LoopVectorizationLegality - A unit that checks for the legality
21// of the vectorization.
22// 3. InnerLoopVectorizer - A unit that performs the actual
23// widening of instructions.
24// 4. LoopVectorizationCostModel - A unit that checks for the profitability
25// of vectorization. It decides on the optimal vector width, which
26// can be one, if vectorization is not profitable.
27//
28// There is a development effort going on to migrate loop vectorizer to the
29// VPlan infrastructure and to introduce outer loop vectorization support (see
30// docs/VectorizationPlan.rst and
31// http://lists.llvm.org/pipermail/llvm-dev/2017-December/119523.html). For this
32// purpose, we temporarily introduced the VPlan-native vectorization path: an
33// alternative vectorization path that is natively implemented on top of the
34// VPlan infrastructure. See EnableVPlanNativePath for enabling.
35//
36//===----------------------------------------------------------------------===//
37//
38// The reduction-variable vectorization is based on the paper:
39// D. Nuzman and R. Henderson. Multi-platform Auto-vectorization.
40//
41// Variable uniformity checks are inspired by:
42// Karrenberg, R. and Hack, S. Whole Function Vectorization.
43//
44// The interleaved access vectorization is based on the paper:
45// Dorit Nuzman, Ira Rosen and Ayal Zaks. Auto-Vectorization of Interleaved
46// Data for SIMD
47//
48// Other ideas/concepts are from:
49// A. Zaks and D. Nuzman. Autovectorization in GCC-two years later.
50//
51// S. Maleki, Y. Gao, M. Garzaran, T. Wong and D. Padua. An Evaluation of
52// Vectorizing Compilers.
53//
54//===----------------------------------------------------------------------===//
55
58#include "VPRecipeBuilder.h"
59#include "VPlan.h"
60#include "VPlanAnalysis.h"
61#include "VPlanCFG.h"
62#include "VPlanHelpers.h"
63#include "VPlanPatternMatch.h"
64#include "VPlanTransforms.h"
65#include "VPlanUtils.h"
66#include "VPlanVerifier.h"
67#include "llvm/ADT/APInt.h"
68#include "llvm/ADT/ArrayRef.h"
69#include "llvm/ADT/DenseMap.h"
71#include "llvm/ADT/Hashing.h"
72#include "llvm/ADT/MapVector.h"
73#include "llvm/ADT/STLExtras.h"
76#include "llvm/ADT/Statistic.h"
77#include "llvm/ADT/StringRef.h"
78#include "llvm/ADT/Twine.h"
79#include "llvm/ADT/TypeSwitch.h"
84#include "llvm/Analysis/CFG.h"
101#include "llvm/IR/Attributes.h"
102#include "llvm/IR/BasicBlock.h"
103#include "llvm/IR/CFG.h"
104#include "llvm/IR/Constant.h"
105#include "llvm/IR/Constants.h"
106#include "llvm/IR/DataLayout.h"
107#include "llvm/IR/DebugInfo.h"
108#include "llvm/IR/DebugLoc.h"
109#include "llvm/IR/DerivedTypes.h"
111#include "llvm/IR/Dominators.h"
112#include "llvm/IR/Function.h"
113#include "llvm/IR/IRBuilder.h"
114#include "llvm/IR/InstrTypes.h"
115#include "llvm/IR/Instruction.h"
116#include "llvm/IR/Instructions.h"
118#include "llvm/IR/Intrinsics.h"
119#include "llvm/IR/MDBuilder.h"
120#include "llvm/IR/Metadata.h"
121#include "llvm/IR/Module.h"
122#include "llvm/IR/Operator.h"
123#include "llvm/IR/PatternMatch.h"
125#include "llvm/IR/Type.h"
126#include "llvm/IR/Use.h"
127#include "llvm/IR/User.h"
128#include "llvm/IR/Value.h"
129#include "llvm/IR/Verifier.h"
130#include "llvm/Support/Casting.h"
132#include "llvm/Support/Debug.h"
147#include <algorithm>
148#include <cassert>
149#include <cstdint>
150#include <functional>
151#include <iterator>
152#include <limits>
153#include <memory>
154#include <string>
155#include <tuple>
156#include <utility>
157
158using namespace llvm;
159using namespace SCEVPatternMatch;
160
161#define LV_NAME "loop-vectorize"
162#define DEBUG_TYPE LV_NAME
163
164#ifndef NDEBUG
165const char VerboseDebug[] = DEBUG_TYPE "-verbose";
166#endif
167
168STATISTIC(LoopsVectorized, "Number of loops vectorized");
169STATISTIC(LoopsAnalyzed, "Number of loops analyzed for vectorization");
170STATISTIC(LoopsEpilogueVectorized, "Number of epilogues vectorized");
171STATISTIC(LoopsEarlyExitVectorized, "Number of early exit loops vectorized");
172
174 "enable-epilogue-vectorization", cl::init(true), cl::Hidden,
175 cl::desc("Enable vectorization of epilogue loops."));
176
178 "epilogue-vectorization-force-VF", cl::init(1), cl::Hidden,
179 cl::desc("When epilogue vectorization is enabled, and a value greater than "
180 "1 is specified, forces the given VF for all applicable epilogue "
181 "loops."));
182
184 "epilogue-vectorization-minimum-VF", cl::Hidden,
185 cl::desc("Only loops with vectorization factor equal to or larger than "
186 "the specified value are considered for epilogue vectorization."));
187
188/// Loops with a known constant trip count below this number are vectorized only
189/// if no scalar iteration overheads are incurred.
191 "vectorizer-min-trip-count", cl::init(16), cl::Hidden,
192 cl::desc("Loops with a constant trip count that is smaller than this "
193 "value are vectorized only if no scalar iteration overheads "
194 "are incurred."));
195
197 "vectorize-memory-check-threshold", cl::init(128), cl::Hidden,
198 cl::desc("The maximum allowed number of runtime memory checks"));
199
200// Option prefer-predicate-over-epilogue indicates that an epilogue is undesired,
201// that predication is preferred, and this lists all options. I.e., the
202// vectorizer will try to fold the tail-loop (epilogue) into the vector body
203// and predicate the instructions accordingly. If tail-folding fails, there are
204// different fallback strategies depending on these values:
211} // namespace PreferPredicateTy
212
214 "prefer-predicate-over-epilogue",
217 cl::desc("Tail-folding and predication preferences over creating a scalar "
218 "epilogue loop."),
220 "scalar-epilogue",
221 "Don't tail-predicate loops, create scalar epilogue"),
223 "predicate-else-scalar-epilogue",
224 "prefer tail-folding, create scalar epilogue if tail "
225 "folding fails."),
227 "predicate-dont-vectorize",
228 "prefers tail-folding, don't attempt vectorization if "
229 "tail-folding fails.")));
230
232 "force-tail-folding-style", cl::desc("Force the tail folding style"),
235 clEnumValN(TailFoldingStyle::None, "none", "Disable tail folding"),
238 "Create lane mask for data only, using active.lane.mask intrinsic"),
240 "data-without-lane-mask",
241 "Create lane mask with compare/stepvector"),
243 "Create lane mask using active.lane.mask intrinsic, and use "
244 "it for both data and control flow"),
246 "data-and-control-without-rt-check",
247 "Similar to data-and-control, but remove the runtime check"),
249 "Use predicated EVL instructions for tail folding. If EVL "
250 "is unsupported, fallback to data-without-lane-mask.")));
251
253 "vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden,
254 cl::desc("Maximize bandwidth when selecting vectorization factor which "
255 "will be determined by the smallest type in loop."));
256
258 "enable-interleaved-mem-accesses", cl::init(false), cl::Hidden,
259 cl::desc("Enable vectorization on interleaved memory accesses in a loop"));
260
261/// An interleave-group may need masking if it resides in a block that needs
262/// predication, or in order to mask away gaps.
264 "enable-masked-interleaved-mem-accesses", cl::init(false), cl::Hidden,
265 cl::desc("Enable vectorization on masked interleaved memory accesses in a loop"));
266
268 "force-target-num-scalar-regs", cl::init(0), cl::Hidden,
269 cl::desc("A flag that overrides the target's number of scalar registers."));
270
272 "force-target-num-vector-regs", cl::init(0), cl::Hidden,
273 cl::desc("A flag that overrides the target's number of vector registers."));
274
276 "force-target-max-scalar-interleave", cl::init(0), cl::Hidden,
277 cl::desc("A flag that overrides the target's max interleave factor for "
278 "scalar loops."));
279
281 "force-target-max-vector-interleave", cl::init(0), cl::Hidden,
282 cl::desc("A flag that overrides the target's max interleave factor for "
283 "vectorized loops."));
284
286 "force-target-instruction-cost", cl::init(0), cl::Hidden,
287 cl::desc("A flag that overrides the target's expected cost for "
288 "an instruction to a single constant value. Mostly "
289 "useful for getting consistent testing."));
290
292 "force-target-supports-scalable-vectors", cl::init(false), cl::Hidden,
293 cl::desc(
294 "Pretend that scalable vectors are supported, even if the target does "
295 "not support them. This flag should only be used for testing."));
296
298 "small-loop-cost", cl::init(20), cl::Hidden,
299 cl::desc(
300 "The cost of a loop that is considered 'small' by the interleaver."));
301
303 "loop-vectorize-with-block-frequency", cl::init(true), cl::Hidden,
304 cl::desc("Enable the use of the block frequency analysis to access PGO "
305 "heuristics minimizing code growth in cold regions and being more "
306 "aggressive in hot regions."));
307
308// Runtime interleave loops for load/store throughput.
310 "enable-loadstore-runtime-interleave", cl::init(true), cl::Hidden,
311 cl::desc(
312 "Enable runtime interleaving until load/store ports are saturated"));
313
314/// The number of stores in a loop that are allowed to need predication.
316 "vectorize-num-stores-pred", cl::init(1), cl::Hidden,
317 cl::desc("Max number of stores to be predicated behind an if."));
318
320 "enable-ind-var-reg-heur", cl::init(true), cl::Hidden,
321 cl::desc("Count the induction variable only once when interleaving"));
322
324 "enable-cond-stores-vec", cl::init(true), cl::Hidden,
325 cl::desc("Enable if predication of stores during vectorization."));
326
328 "max-nested-scalar-reduction-interleave", cl::init(2), cl::Hidden,
329 cl::desc("The maximum interleave count to use when interleaving a scalar "
330 "reduction in a nested loop."));
331
332static cl::opt<bool>
333 PreferInLoopReductions("prefer-inloop-reductions", cl::init(false),
335 cl::desc("Prefer in-loop vector reductions, "
336 "overriding the targets preference."));
337
339 "force-ordered-reductions", cl::init(false), cl::Hidden,
340 cl::desc("Enable the vectorisation of loops with in-order (strict) "
341 "FP reductions"));
342
344 "prefer-predicated-reduction-select", cl::init(false), cl::Hidden,
345 cl::desc(
346 "Prefer predicating a reduction operation over an after loop select."));
347
349 "enable-vplan-native-path", cl::Hidden,
350 cl::desc("Enable VPlan-native vectorization path with "
351 "support for outer loop vectorization."));
352
354 llvm::VerifyEachVPlan("vplan-verify-each",
355#ifdef EXPENSIVE_CHECKS
356 cl::init(true),
357#else
358 cl::init(false),
359#endif
361 cl::desc("Verfiy VPlans after VPlan transforms."));
362
363// This flag enables the stress testing of the VPlan H-CFG construction in the
364// VPlan-native vectorization path. It must be used in conjuction with
365// -enable-vplan-native-path. -vplan-verify-hcfg can also be used to enable the
366// verification of the H-CFGs built.
368 "vplan-build-stress-test", cl::init(false), cl::Hidden,
369 cl::desc(
370 "Build VPlan for every supported loop nest in the function and bail "
371 "out right after the build (stress test the VPlan H-CFG construction "
372 "in the VPlan-native vectorization path)."));
373
375 "interleave-loops", cl::init(true), cl::Hidden,
376 cl::desc("Enable loop interleaving in Loop vectorization passes"));
378 "vectorize-loops", cl::init(true), cl::Hidden,
379 cl::desc("Run the Loop vectorization passes"));
380
382 "force-widen-divrem-via-safe-divisor", cl::Hidden,
383 cl::desc(
384 "Override cost based safe divisor widening for div/rem instructions"));
385
387 "vectorizer-maximize-bandwidth-for-vector-calls", cl::init(true),
389 cl::desc("Try wider VFs if they enable the use of vector variants"));
390
392 "enable-early-exit-vectorization", cl::init(true), cl::Hidden,
393 cl::desc(
394 "Enable vectorization of early exit loops with uncountable exits."));
395
397 "vectorizer-consider-reg-pressure", cl::init(false), cl::Hidden,
398 cl::desc("Discard VFs if their register pressure is too high."));
399
400// Likelyhood of bypassing the vectorized loop because there are zero trips left
401// after prolog. See `emitIterationCountCheck`.
402static constexpr uint32_t MinItersBypassWeights[] = {1, 127};
403
404/// A helper function that returns true if the given type is irregular. The
405/// type is irregular if its allocated size doesn't equal the store size of an
406/// element of the corresponding vector type.
407static bool hasIrregularType(Type *Ty, const DataLayout &DL) {
408 // Determine if an array of N elements of type Ty is "bitcast compatible"
409 // with a <N x Ty> vector.
410 // This is only true if there is no padding between the array elements.
411 return DL.getTypeAllocSizeInBits(Ty) != DL.getTypeSizeInBits(Ty);
412}
413
414/// A version of ScalarEvolution::getSmallConstantTripCount that returns an
415/// ElementCount to include loops whose trip count is a function of vscale.
417 const Loop *L) {
418 if (unsigned ExpectedTC = SE->getSmallConstantTripCount(L))
419 return ElementCount::getFixed(ExpectedTC);
420
421 const SCEV *BTC = SE->getBackedgeTakenCount(L);
423 return ElementCount::getFixed(0);
424
425 const SCEV *ExitCount = SE->getTripCountFromExitCount(BTC, BTC->getType(), L);
426 if (isa<SCEVVScale>(ExitCount))
428
429 const APInt *Scale;
430 if (match(ExitCount, m_scev_Mul(m_scev_APInt(Scale), m_SCEVVScale())))
431 if (cast<SCEVMulExpr>(ExitCount)->hasNoUnsignedWrap())
432 if (Scale->getActiveBits() <= 32)
434
435 return ElementCount::getFixed(0);
436}
437
438/// Returns "best known" trip count, which is either a valid positive trip count
439/// or std::nullopt when an estimate cannot be made (including when the trip
440/// count would overflow), for the specified loop \p L as defined by the
441/// following procedure:
442/// 1) Returns exact trip count if it is known.
443/// 2) Returns expected trip count according to profile data if any.
444/// 3) Returns upper bound estimate if known, and if \p CanUseConstantMax.
445/// 4) Returns std::nullopt if all of the above failed.
446static std::optional<ElementCount>
448 bool CanUseConstantMax = true) {
449 // Check if exact trip count is known.
450 if (auto ExpectedTC = getSmallConstantTripCount(PSE.getSE(), L))
451 return ExpectedTC;
452
453 // Check if there is an expected trip count available from profile data.
455 if (auto EstimatedTC = getLoopEstimatedTripCount(L))
456 return ElementCount::getFixed(*EstimatedTC);
457
458 if (!CanUseConstantMax)
459 return std::nullopt;
460
461 // Check if upper bound estimate is known.
462 if (unsigned ExpectedTC = PSE.getSmallConstantMaxTripCount())
463 return ElementCount::getFixed(ExpectedTC);
464
465 return std::nullopt;
466}
467
468namespace {
469// Forward declare GeneratedRTChecks.
470class GeneratedRTChecks;
471
472using SCEV2ValueTy = DenseMap<const SCEV *, Value *>;
473} // namespace
474
475namespace llvm {
476
478
479/// InnerLoopVectorizer vectorizes loops which contain only one basic
480/// block to a specified vectorization factor (VF).
481/// This class performs the widening of scalars into vectors, or multiple
482/// scalars. This class also implements the following features:
483/// * It inserts an epilogue loop for handling loops that don't have iteration
484/// counts that are known to be a multiple of the vectorization factor.
485/// * It handles the code generation for reduction variables.
486/// * Scalarization (implementation using scalars) of un-vectorizable
487/// instructions.
488/// InnerLoopVectorizer does not perform any vectorization-legality
489/// checks, and relies on the caller to check for the different legality
490/// aspects. The InnerLoopVectorizer relies on the
491/// LoopVectorizationLegality class to provide information about the induction
492/// and reduction variables that were found to a given vectorization factor.
494public:
498 ElementCount VecWidth, unsigned UnrollFactor,
500 ProfileSummaryInfo *PSI, GeneratedRTChecks &RTChecks,
501 VPlan &Plan)
502 : OrigLoop(OrigLoop), PSE(PSE), LI(LI), DT(DT), TTI(TTI), AC(AC),
503 VF(VecWidth), UF(UnrollFactor), Builder(PSE.getSE()->getContext()),
506 Plan.getVectorLoopRegion()->getSinglePredecessor())) {}
507
508 virtual ~InnerLoopVectorizer() = default;
509
510 /// Creates a basic block for the scalar preheader. Both
511 /// EpilogueVectorizerMainLoop and EpilogueVectorizerEpilogueLoop overwrite
512 /// the method to create additional blocks and checks needed for epilogue
513 /// vectorization.
515
516 /// Fix the vectorized code, taking care of header phi's, and more.
518
519 /// Fix the non-induction PHIs in \p Plan.
521
522 /// Returns the original loop trip count.
523 Value *getTripCount() const { return TripCount; }
524
525 /// Used to set the trip count after ILV's construction and after the
526 /// preheader block has been executed. Note that this always holds the trip
527 /// count of the original loop for both main loop and epilogue vectorization.
528 void setTripCount(Value *TC) { TripCount = TC; }
529
530protected:
532
533 /// Create and return a new IR basic block for the scalar preheader whose name
534 /// is prefixed with \p Prefix.
536
537 /// Allow subclasses to override and print debug traces before/after vplan
538 /// execution, when trace information is requested.
539 virtual void printDebugTracesAtStart() {}
540 virtual void printDebugTracesAtEnd() {}
541
542 /// The original loop.
544
545 /// A wrapper around ScalarEvolution used to add runtime SCEV checks. Applies
546 /// dynamic knowledge to simplify SCEV expressions and converts them to a
547 /// more usable form.
549
550 /// Loop Info.
552
553 /// Dominator Tree.
555
556 /// Target Transform Info.
558
559 /// Assumption Cache.
561
562 /// The vectorization SIMD factor to use. Each vector will have this many
563 /// vector elements.
565
566 /// The vectorization unroll factor to use. Each scalar is vectorized to this
567 /// many different vector instructions.
568 unsigned UF;
569
570 /// The builder that we use
572
573 // --- Vectorization state ---
574
575 /// Trip count of the original loop.
576 Value *TripCount = nullptr;
577
578 /// The profitablity analysis.
580
581 /// BFI and PSI are used to check for profile guided size optimizations.
584
585 /// Structure to hold information about generated runtime checks, responsible
586 /// for cleaning the checks, if vectorization turns out unprofitable.
587 GeneratedRTChecks &RTChecks;
588
590
591 /// The vector preheader block of \p Plan, used as target for check blocks
592 /// introduced during skeleton creation.
594};
595
596/// Encapsulate information regarding vectorization of a loop and its epilogue.
597/// This information is meant to be updated and used across two stages of
598/// epilogue vectorization.
601 unsigned MainLoopUF = 0;
603 unsigned EpilogueUF = 0;
606 Value *TripCount = nullptr;
609
611 ElementCount EVF, unsigned EUF,
613 : MainLoopVF(MVF), MainLoopUF(MUF), EpilogueVF(EVF), EpilogueUF(EUF),
615 assert(EUF == 1 &&
616 "A high UF for the epilogue loop is likely not beneficial.");
617 }
618};
619
620/// An extension of the inner loop vectorizer that creates a skeleton for a
621/// vectorized loop that has its epilogue (residual) also vectorized.
622/// The idea is to run the vplan on a given loop twice, firstly to setup the
623/// skeleton and vectorize the main loop, and secondly to complete the skeleton
624/// from the first step and vectorize the epilogue. This is achieved by
625/// deriving two concrete strategy classes from this base class and invoking
626/// them in succession from the loop vectorizer planner.
628public:
639
640 /// Holds and updates state information required to vectorize the main loop
641 /// and its epilogue in two separate passes. This setup helps us avoid
642 /// regenerating and recomputing runtime safety checks. It also helps us to
643 /// shorten the iteration-count-check path length for the cases where the
644 /// iteration count of the loop is so small that the main vector loop is
645 /// completely skipped.
647
648protected:
650};
651
652/// A specialized derived class of inner loop vectorizer that performs
653/// vectorization of *main* loops in the process of vectorizing loops and their
654/// epilogues.
656public:
668 /// Implements the interface for creating a vectorized skeleton using the
669 /// *main loop* strategy (i.e., the first pass of VPlan execution).
671
672protected:
673 /// Introduces a new VPIRBasicBlock for \p CheckIRBB to Plan between the
674 /// vector preheader and its predecessor, also connecting the new block to the
675 /// scalar preheader.
676 void introduceCheckBlockInVPlan(BasicBlock *CheckIRBB);
677
678 // Create a check to see if the main vector loop should be executed
680 unsigned UF) const;
681
682 /// Emits an iteration count bypass check once for the main loop (when \p
683 /// ForEpilogue is false) and once for the epilogue loop (when \p
684 /// ForEpilogue is true).
686 bool ForEpilogue);
687 void printDebugTracesAtStart() override;
688 void printDebugTracesAtEnd() override;
689};
690
691// A specialized derived class of inner loop vectorizer that performs
692// vectorization of *epilogue* loops in the process of vectorizing loops and
693// their epilogues.
695public:
705 /// Implements the interface for creating a vectorized skeleton using the
706 /// *epilogue loop* strategy (i.e., the second pass of VPlan execution).
708
709protected:
710 void printDebugTracesAtStart() override;
711 void printDebugTracesAtEnd() override;
712};
713} // end namespace llvm
714
715/// Look for a meaningful debug location on the instruction or its operands.
717 if (!I)
718 return DebugLoc::getUnknown();
719
721 if (I->getDebugLoc() != Empty)
722 return I->getDebugLoc();
723
724 for (Use &Op : I->operands()) {
725 if (Instruction *OpInst = dyn_cast<Instruction>(Op))
726 if (OpInst->getDebugLoc() != Empty)
727 return OpInst->getDebugLoc();
728 }
729
730 return I->getDebugLoc();
731}
732
733/// Write a \p DebugMsg about vectorization to the debug output stream. If \p I
734/// is passed, the message relates to that particular instruction.
735#ifndef NDEBUG
736static void debugVectorizationMessage(const StringRef Prefix,
737 const StringRef DebugMsg,
738 Instruction *I) {
739 dbgs() << "LV: " << Prefix << DebugMsg;
740 if (I != nullptr)
741 dbgs() << " " << *I;
742 else
743 dbgs() << '.';
744 dbgs() << '\n';
745}
746#endif
747
748/// Create an analysis remark that explains why vectorization failed
749///
750/// \p PassName is the name of the pass (e.g. can be AlwaysPrint). \p
751/// RemarkName is the identifier for the remark. If \p I is passed it is an
752/// instruction that prevents vectorization. Otherwise \p TheLoop is used for
753/// the location of the remark. If \p DL is passed, use it as debug location for
754/// the remark. \return the remark object that can be streamed to.
755static OptimizationRemarkAnalysis
756createLVAnalysis(const char *PassName, StringRef RemarkName, Loop *TheLoop,
757 Instruction *I, DebugLoc DL = {}) {
758 BasicBlock *CodeRegion = I ? I->getParent() : TheLoop->getHeader();
759 // If debug location is attached to the instruction, use it. Otherwise if DL
760 // was not provided, use the loop's.
761 if (I && I->getDebugLoc())
762 DL = I->getDebugLoc();
763 else if (!DL)
764 DL = TheLoop->getStartLoc();
765
766 return OptimizationRemarkAnalysis(PassName, RemarkName, DL, CodeRegion);
767}
768
769namespace llvm {
770
771/// Return a value for Step multiplied by VF.
773 int64_t Step) {
774 assert(Ty->isIntegerTy() && "Expected an integer step");
775 ElementCount VFxStep = VF.multiplyCoefficientBy(Step);
776 assert(isPowerOf2_64(VF.getKnownMinValue()) && "must pass power-of-2 VF");
777 if (VF.isScalable() && isPowerOf2_64(Step)) {
778 return B.CreateShl(
779 B.CreateVScale(Ty),
780 ConstantInt::get(Ty, Log2_64(VFxStep.getKnownMinValue())), "", true);
781 }
782 return B.CreateElementCount(Ty, VFxStep);
783}
784
785/// Return the runtime value for VF.
787 return B.CreateElementCount(Ty, VF);
788}
789
791 const StringRef OREMsg, const StringRef ORETag,
792 OptimizationRemarkEmitter *ORE, Loop *TheLoop,
793 Instruction *I) {
794 LLVM_DEBUG(debugVectorizationMessage("Not vectorizing: ", DebugMsg, I));
795 LoopVectorizeHints Hints(TheLoop, true /* doesn't matter */, *ORE);
796 ORE->emit(
797 createLVAnalysis(Hints.vectorizeAnalysisPassName(), ORETag, TheLoop, I)
798 << "loop not vectorized: " << OREMsg);
799}
800
801/// Reports an informative message: print \p Msg for debugging purposes as well
802/// as an optimization remark. Uses either \p I as location of the remark, or
803/// otherwise \p TheLoop. If \p DL is passed, use it as debug location for the
804/// remark. If \p DL is passed, use it as debug location for the remark.
805static void reportVectorizationInfo(const StringRef Msg, const StringRef ORETag,
807 Loop *TheLoop, Instruction *I = nullptr,
808 DebugLoc DL = {}) {
810 LoopVectorizeHints Hints(TheLoop, true /* doesn't matter */, *ORE);
811 ORE->emit(createLVAnalysis(Hints.vectorizeAnalysisPassName(), ORETag, TheLoop,
812 I, DL)
813 << Msg);
814}
815
816/// Report successful vectorization of the loop. In case an outer loop is
817/// vectorized, prepend "outer" to the vectorization remark.
819 VectorizationFactor VF, unsigned IC) {
821 "Vectorizing: ", TheLoop->isInnermost() ? "innermost loop" : "outer loop",
822 nullptr));
823 StringRef LoopType = TheLoop->isInnermost() ? "" : "outer ";
824 ORE->emit([&]() {
825 return OptimizationRemark(LV_NAME, "Vectorized", TheLoop->getStartLoc(),
826 TheLoop->getHeader())
827 << "vectorized " << LoopType << "loop (vectorization width: "
828 << ore::NV("VectorizationFactor", VF.Width)
829 << ", interleaved count: " << ore::NV("InterleaveCount", IC) << ")";
830 });
831}
832
833} // end namespace llvm
834
835namespace llvm {
836
837// Loop vectorization cost-model hints how the scalar epilogue loop should be
838// lowered.
840
841 // The default: allowing scalar epilogues.
843
844 // Vectorization with OptForSize: don't allow epilogues.
846
847 // A special case of vectorisation with OptForSize: loops with a very small
848 // trip count are considered for vectorization under OptForSize, thereby
849 // making sure the cost of their loop body is dominant, free of runtime
850 // guards and scalar iteration overheads.
852
853 // Loop hint predicate indicating an epilogue is undesired.
855
856 // Directive indicating we must either tail fold or not vectorize
858};
859
860/// LoopVectorizationCostModel - estimates the expected speedups due to
861/// vectorization.
862/// In many cases vectorization is not profitable. This can happen because of
863/// a number of reasons. In this class we mainly attempt to predict the
864/// expected speedup/slowdowns due to the supported instruction set. We use the
865/// TargetTransformInfo to query the different backends for the cost of
866/// different operations.
869
870public:
881 : ScalarEpilogueStatus(SEL), TheLoop(L), PSE(PSE), LI(LI), Legal(Legal),
882 TTI(TTI), TLI(TLI), DB(DB), AC(AC), ORE(ORE), TheFunction(F),
883 Hints(Hints), InterleaveInfo(IAI) {
884 if (TTI.supportsScalableVectors() || ForceTargetSupportsScalableVectors)
885 initializeVScaleForTuning();
887 // Query this against the original loop and save it here because the profile
888 // of the original loop header may change as the transformation happens.
889 OptForSize = llvm::shouldOptimizeForSize(L->getHeader(), PSI, BFI,
891 }
892
893 /// \return An upper bound for the vectorization factors (both fixed and
894 /// scalable). If the factors are 0, vectorization and interleaving should be
895 /// avoided up front.
896 FixedScalableVFPair computeMaxVF(ElementCount UserVF, unsigned UserIC);
897
898 /// \return True if runtime checks are required for vectorization, and false
899 /// otherwise.
900 bool runtimeChecksRequired();
901
902 /// Setup cost-based decisions for user vectorization factor.
903 /// \return true if the UserVF is a feasible VF to be chosen.
906 return expectedCost(UserVF).isValid();
907 }
908
909 /// \return True if maximizing vector bandwidth is enabled by the target or
910 /// user options, for the given register kind.
911 bool useMaxBandwidth(TargetTransformInfo::RegisterKind RegKind);
912
913 /// \return True if register pressure should be considered for the given VF.
914 bool shouldConsiderRegPressureForVF(ElementCount VF);
915
916 /// \return The size (in bits) of the smallest and widest types in the code
917 /// that needs to be vectorized. We ignore values that remain scalar such as
918 /// 64 bit loop indices.
919 std::pair<unsigned, unsigned> getSmallestAndWidestTypes();
920
921 /// Memory access instruction may be vectorized in more than one way.
922 /// Form of instruction after vectorization depends on cost.
923 /// This function takes cost-based decisions for Load/Store instructions
924 /// and collects them in a map. This decisions map is used for building
925 /// the lists of loop-uniform and loop-scalar instructions.
926 /// The calculated cost is saved with widening decision in order to
927 /// avoid redundant calculations.
928 void setCostBasedWideningDecision(ElementCount VF);
929
930 /// A call may be vectorized in different ways depending on whether we have
931 /// vectorized variants available and whether the target supports masking.
932 /// This function analyzes all calls in the function at the supplied VF,
933 /// makes a decision based on the costs of available options, and stores that
934 /// decision in a map for use in planning and plan execution.
935 void setVectorizedCallDecision(ElementCount VF);
936
937 /// Collect values we want to ignore in the cost model.
938 void collectValuesToIgnore();
939
940 /// Collect all element types in the loop for which widening is needed.
941 void collectElementTypesForWidening();
942
943 /// Split reductions into those that happen in the loop, and those that happen
944 /// outside. In loop reductions are collected into InLoopReductions.
945 void collectInLoopReductions();
946
947 /// Returns true if we should use strict in-order reductions for the given
948 /// RdxDesc. This is true if the -enable-strict-reductions flag is passed,
949 /// the IsOrdered flag of RdxDesc is set and we do not allow reordering
950 /// of FP operations.
951 bool useOrderedReductions(const RecurrenceDescriptor &RdxDesc) const {
952 return !Hints->allowReordering() && RdxDesc.isOrdered();
953 }
954
955 /// \returns The smallest bitwidth each instruction can be represented with.
956 /// The vector equivalents of these instructions should be truncated to this
957 /// type.
959 return MinBWs;
960 }
961
962 /// \returns True if it is more profitable to scalarize instruction \p I for
963 /// vectorization factor \p VF.
965 assert(VF.isVector() &&
966 "Profitable to scalarize relevant only for VF > 1.");
967 assert(
968 TheLoop->isInnermost() &&
969 "cost-model should not be used for outer loops (in VPlan-native path)");
970
971 auto Scalars = InstsToScalarize.find(VF);
972 assert(Scalars != InstsToScalarize.end() &&
973 "VF not yet analyzed for scalarization profitability");
974 return Scalars->second.contains(I);
975 }
976
977 /// Returns true if \p I is known to be uniform after vectorization.
979 assert(
980 TheLoop->isInnermost() &&
981 "cost-model should not be used for outer loops (in VPlan-native path)");
982 // Pseudo probe needs to be duplicated for each unrolled iteration and
983 // vector lane so that profiled loop trip count can be accurately
984 // accumulated instead of being under counted.
986 return false;
987
988 if (VF.isScalar())
989 return true;
990
991 auto UniformsPerVF = Uniforms.find(VF);
992 assert(UniformsPerVF != Uniforms.end() &&
993 "VF not yet analyzed for uniformity");
994 return UniformsPerVF->second.count(I);
995 }
996
997 /// Returns true if \p I is known to be scalar after vectorization.
999 assert(
1000 TheLoop->isInnermost() &&
1001 "cost-model should not be used for outer loops (in VPlan-native path)");
1002 if (VF.isScalar())
1003 return true;
1004
1005 auto ScalarsPerVF = Scalars.find(VF);
1006 assert(ScalarsPerVF != Scalars.end() &&
1007 "Scalar values are not calculated for VF");
1008 return ScalarsPerVF->second.count(I);
1009 }
1010
1011 /// \returns True if instruction \p I can be truncated to a smaller bitwidth
1012 /// for vectorization factor \p VF.
1014 return VF.isVector() && MinBWs.contains(I) &&
1015 !isProfitableToScalarize(I, VF) &&
1017 }
1018
1019 /// Decision that was taken during cost calculation for memory instruction.
1022 CM_Widen, // For consecutive accesses with stride +1.
1023 CM_Widen_Reverse, // For consecutive accesses with stride -1.
1029 };
1030
1031 /// Save vectorization decision \p W and \p Cost taken by the cost model for
1032 /// instruction \p I and vector width \p VF.
1035 assert(VF.isVector() && "Expected VF >=2");
1036 WideningDecisions[{I, VF}] = {W, Cost};
1037 }
1038
1039 /// Save vectorization decision \p W and \p Cost taken by the cost model for
1040 /// interleaving group \p Grp and vector width \p VF.
1044 assert(VF.isVector() && "Expected VF >=2");
1045 /// Broadcast this decicion to all instructions inside the group.
1046 /// When interleaving, the cost will only be assigned one instruction, the
1047 /// insert position. For other cases, add the appropriate fraction of the
1048 /// total cost to each instruction. This ensures accurate costs are used,
1049 /// even if the insert position instruction is not used.
1050 InstructionCost InsertPosCost = Cost;
1051 InstructionCost OtherMemberCost = 0;
1052 if (W != CM_Interleave)
1053 OtherMemberCost = InsertPosCost = Cost / Grp->getNumMembers();
1054 ;
1055 for (unsigned Idx = 0; Idx < Grp->getFactor(); ++Idx) {
1056 if (auto *I = Grp->getMember(Idx)) {
1057 if (Grp->getInsertPos() == I)
1058 WideningDecisions[{I, VF}] = {W, InsertPosCost};
1059 else
1060 WideningDecisions[{I, VF}] = {W, OtherMemberCost};
1061 }
1062 }
1063 }
1064
1065 /// Return the cost model decision for the given instruction \p I and vector
1066 /// width \p VF. Return CM_Unknown if this instruction did not pass
1067 /// through the cost modeling.
1069 assert(VF.isVector() && "Expected VF to be a vector VF");
1070 assert(
1071 TheLoop->isInnermost() &&
1072 "cost-model should not be used for outer loops (in VPlan-native path)");
1073
1074 std::pair<Instruction *, ElementCount> InstOnVF(I, VF);
1075 auto Itr = WideningDecisions.find(InstOnVF);
1076 if (Itr == WideningDecisions.end())
1077 return CM_Unknown;
1078 return Itr->second.first;
1079 }
1080
1081 /// Return the vectorization cost for the given instruction \p I and vector
1082 /// width \p VF.
1084 assert(VF.isVector() && "Expected VF >=2");
1085 std::pair<Instruction *, ElementCount> InstOnVF(I, VF);
1086 assert(WideningDecisions.contains(InstOnVF) &&
1087 "The cost is not calculated");
1088 return WideningDecisions[InstOnVF].second;
1089 }
1090
1098
1100 Function *Variant, Intrinsic::ID IID,
1101 std::optional<unsigned> MaskPos,
1103 assert(!VF.isScalar() && "Expected vector VF");
1104 CallWideningDecisions[{CI, VF}] = {Kind, Variant, IID, MaskPos, Cost};
1105 }
1106
1108 ElementCount VF) const {
1109 assert(!VF.isScalar() && "Expected vector VF");
1110 auto I = CallWideningDecisions.find({CI, VF});
1111 if (I == CallWideningDecisions.end())
1112 return {CM_Unknown, nullptr, Intrinsic::not_intrinsic, std::nullopt, 0};
1113 return I->second;
1114 }
1115
1116 /// Return True if instruction \p I is an optimizable truncate whose operand
1117 /// is an induction variable. Such a truncate will be removed by adding a new
1118 /// induction variable with the destination type.
1120 // If the instruction is not a truncate, return false.
1121 auto *Trunc = dyn_cast<TruncInst>(I);
1122 if (!Trunc)
1123 return false;
1124
1125 // Get the source and destination types of the truncate.
1126 Type *SrcTy = toVectorTy(Trunc->getSrcTy(), VF);
1127 Type *DestTy = toVectorTy(Trunc->getDestTy(), VF);
1128
1129 // If the truncate is free for the given types, return false. Replacing a
1130 // free truncate with an induction variable would add an induction variable
1131 // update instruction to each iteration of the loop. We exclude from this
1132 // check the primary induction variable since it will need an update
1133 // instruction regardless.
1134 Value *Op = Trunc->getOperand(0);
1135 if (Op != Legal->getPrimaryInduction() && TTI.isTruncateFree(SrcTy, DestTy))
1136 return false;
1137
1138 // If the truncated value is not an induction variable, return false.
1139 return Legal->isInductionPhi(Op);
1140 }
1141
1142 /// Collects the instructions to scalarize for each predicated instruction in
1143 /// the loop.
1144 void collectInstsToScalarize(ElementCount VF);
1145
1146 /// Collect values that will not be widened, including Uniforms, Scalars, and
1147 /// Instructions to Scalarize for the given \p VF.
1148 /// The sets depend on CM decision for Load/Store instructions
1149 /// that may be vectorized as interleave, gather-scatter or scalarized.
1150 /// Also make a decision on what to do about call instructions in the loop
1151 /// at that VF -- scalarize, call a known vector routine, or call a
1152 /// vector intrinsic.
1154 // Do the analysis once.
1155 if (VF.isScalar() || Uniforms.contains(VF))
1156 return;
1158 collectLoopUniforms(VF);
1160 collectLoopScalars(VF);
1162 }
1163
1164 /// Returns true if the target machine supports masked store operation
1165 /// for the given \p DataType and kind of access to \p Ptr.
1166 bool isLegalMaskedStore(Type *DataType, Value *Ptr, Align Alignment,
1167 unsigned AddressSpace) const {
1168 return Legal->isConsecutivePtr(DataType, Ptr) &&
1169 TTI.isLegalMaskedStore(DataType, Alignment, AddressSpace);
1170 }
1171
1172 /// Returns true if the target machine supports masked load operation
1173 /// for the given \p DataType and kind of access to \p Ptr.
1174 bool isLegalMaskedLoad(Type *DataType, Value *Ptr, Align Alignment,
1175 unsigned AddressSpace) const {
1176 return Legal->isConsecutivePtr(DataType, Ptr) &&
1177 TTI.isLegalMaskedLoad(DataType, Alignment, AddressSpace);
1178 }
1179
1180 /// Returns true if the target machine can represent \p V as a masked gather
1181 /// or scatter operation.
1183 bool LI = isa<LoadInst>(V);
1184 bool SI = isa<StoreInst>(V);
1185 if (!LI && !SI)
1186 return false;
1187 auto *Ty = getLoadStoreType(V);
1189 if (VF.isVector())
1190 Ty = VectorType::get(Ty, VF);
1191 return (LI && TTI.isLegalMaskedGather(Ty, Align)) ||
1192 (SI && TTI.isLegalMaskedScatter(Ty, Align));
1193 }
1194
1195 /// Returns true if the target machine supports all of the reduction
1196 /// variables found for the given VF.
1198 return (all_of(Legal->getReductionVars(), [&](auto &Reduction) -> bool {
1199 const RecurrenceDescriptor &RdxDesc = Reduction.second;
1200 return TTI.isLegalToVectorizeReduction(RdxDesc, VF);
1201 }));
1202 }
1203
1204 /// Given costs for both strategies, return true if the scalar predication
1205 /// lowering should be used for div/rem. This incorporates an override
1206 /// option so it is not simply a cost comparison.
1208 InstructionCost SafeDivisorCost) const {
1209 switch (ForceSafeDivisor) {
1210 case cl::BOU_UNSET:
1211 return ScalarCost < SafeDivisorCost;
1212 case cl::BOU_TRUE:
1213 return false;
1214 case cl::BOU_FALSE:
1215 return true;
1216 }
1217 llvm_unreachable("impossible case value");
1218 }
1219
1220 /// Returns true if \p I is an instruction which requires predication and
1221 /// for which our chosen predication strategy is scalarization (i.e. we
1222 /// don't have an alternate strategy such as masking available).
1223 /// \p VF is the vectorization factor that will be used to vectorize \p I.
1224 bool isScalarWithPredication(Instruction *I, ElementCount VF) const;
1225
1226 /// Returns true if \p I is an instruction that needs to be predicated
1227 /// at runtime. The result is independent of the predication mechanism.
1228 /// Superset of instructions that return true for isScalarWithPredication.
1229 bool isPredicatedInst(Instruction *I) const;
1230
1231 /// Return the costs for our two available strategies for lowering a
1232 /// div/rem operation which requires speculating at least one lane.
1233 /// First result is for scalarization (will be invalid for scalable
1234 /// vectors); second is for the safe-divisor strategy.
1235 std::pair<InstructionCost, InstructionCost>
1236 getDivRemSpeculationCost(Instruction *I,
1237 ElementCount VF) const;
1238
1239 /// Returns true if \p I is a memory instruction with consecutive memory
1240 /// access that can be widened.
1241 bool memoryInstructionCanBeWidened(Instruction *I, ElementCount VF);
1242
1243 /// Returns true if \p I is a memory instruction in an interleaved-group
1244 /// of memory accesses that can be vectorized with wide vector loads/stores
1245 /// and shuffles.
1246 bool interleavedAccessCanBeWidened(Instruction *I, ElementCount VF) const;
1247
1248 /// Check if \p Instr belongs to any interleaved access group.
1250 return InterleaveInfo.isInterleaved(Instr);
1251 }
1252
1253 /// Get the interleaved access group that \p Instr belongs to.
1256 return InterleaveInfo.getInterleaveGroup(Instr);
1257 }
1258
1259 /// Returns true if we're required to use a scalar epilogue for at least
1260 /// the final iteration of the original loop.
1261 bool requiresScalarEpilogue(bool IsVectorizing) const {
1262 if (!isScalarEpilogueAllowed()) {
1263 LLVM_DEBUG(dbgs() << "LV: Loop does not require scalar epilogue\n");
1264 return false;
1265 }
1266 // If we might exit from anywhere but the latch and early exit vectorization
1267 // is disabled, we must run the exiting iteration in scalar form.
1268 if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch() &&
1269 !(EnableEarlyExitVectorization && Legal->hasUncountableEarlyExit())) {
1270 LLVM_DEBUG(dbgs() << "LV: Loop requires scalar epilogue: not exiting "
1271 "from latch block\n");
1272 return true;
1273 }
1274 if (IsVectorizing && InterleaveInfo.requiresScalarEpilogue()) {
1275 LLVM_DEBUG(dbgs() << "LV: Loop requires scalar epilogue: "
1276 "interleaved group requires scalar epilogue\n");
1277 return true;
1278 }
1279 LLVM_DEBUG(dbgs() << "LV: Loop does not require scalar epilogue\n");
1280 return false;
1281 }
1282
1283 /// Returns true if a scalar epilogue is not allowed due to optsize or a
1284 /// loop hint annotation.
1286 return ScalarEpilogueStatus == CM_ScalarEpilogueAllowed;
1287 }
1288
1289 /// Returns the TailFoldingStyle that is best for the current loop.
1290 TailFoldingStyle getTailFoldingStyle(bool IVUpdateMayOverflow = true) const {
1291 if (!ChosenTailFoldingStyle)
1293 return IVUpdateMayOverflow ? ChosenTailFoldingStyle->first
1294 : ChosenTailFoldingStyle->second;
1295 }
1296
1297 /// Selects and saves TailFoldingStyle for 2 options - if IV update may
1298 /// overflow or not.
1299 /// \param IsScalableVF true if scalable vector factors enabled.
1300 /// \param UserIC User specific interleave count.
1301 void setTailFoldingStyles(bool IsScalableVF, unsigned UserIC) {
1302 assert(!ChosenTailFoldingStyle && "Tail folding must not be selected yet.");
1303 if (!Legal->canFoldTailByMasking()) {
1304 ChosenTailFoldingStyle = {TailFoldingStyle::None, TailFoldingStyle::None};
1305 return;
1306 }
1307
1308 // Default to TTI preference, but allow command line override.
1309 ChosenTailFoldingStyle = {
1310 TTI.getPreferredTailFoldingStyle(/*IVUpdateMayOverflow=*/true),
1311 TTI.getPreferredTailFoldingStyle(/*IVUpdateMayOverflow=*/false)};
1312 if (ForceTailFoldingStyle.getNumOccurrences())
1313 ChosenTailFoldingStyle = {ForceTailFoldingStyle.getValue(),
1314 ForceTailFoldingStyle.getValue()};
1315
1316 if (ChosenTailFoldingStyle->first != TailFoldingStyle::DataWithEVL &&
1317 ChosenTailFoldingStyle->second != TailFoldingStyle::DataWithEVL)
1318 return;
1319 // Override EVL styles if needed.
1320 // FIXME: Investigate opportunity for fixed vector factor.
1321 bool EVLIsLegal = UserIC <= 1 && IsScalableVF &&
1322 TTI.hasActiveVectorLength() && !EnableVPlanNativePath;
1323 if (EVLIsLegal)
1324 return;
1325 // If for some reason EVL mode is unsupported, fallback to a scalar epilogue
1326 // if it's allowed, or DataWithoutLaneMask otherwise.
1327 if (ScalarEpilogueStatus == CM_ScalarEpilogueAllowed ||
1328 ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUsePredicate)
1329 ChosenTailFoldingStyle = {TailFoldingStyle::None, TailFoldingStyle::None};
1330 else
1331 ChosenTailFoldingStyle = {TailFoldingStyle::DataWithoutLaneMask,
1333
1334 LLVM_DEBUG(
1335 dbgs() << "LV: Preference for VP intrinsics indicated. Will "
1336 "not try to generate VP Intrinsics "
1337 << (UserIC > 1
1338 ? "since interleave count specified is greater than 1.\n"
1339 : "due to non-interleaving reasons.\n"));
1340 }
1341
1342 /// Returns true if all loop blocks should be masked to fold tail loop.
1343 bool foldTailByMasking() const {
1344 // TODO: check if it is possible to check for None style independent of
1345 // IVUpdateMayOverflow flag in getTailFoldingStyle.
1347 }
1348
1349 /// Return maximum safe number of elements to be processed per vector
1350 /// iteration, which do not prevent store-load forwarding and are safe with
1351 /// regard to the memory dependencies. Required for EVL-based VPlans to
1352 /// correctly calculate AVL (application vector length) as min(remaining AVL,
1353 /// MaxSafeElements).
1354 /// TODO: need to consider adjusting cost model to use this value as a
1355 /// vectorization factor for EVL-based vectorization.
1356 std::optional<unsigned> getMaxSafeElements() const { return MaxSafeElements; }
1357
1358 /// Returns true if the instructions in this block requires predication
1359 /// for any reason, e.g. because tail folding now requires a predicate
1360 /// or because the block in the original loop was predicated.
1362 return foldTailByMasking() || Legal->blockNeedsPredication(BB);
1363 }
1364
1365 /// Returns true if VP intrinsics with explicit vector length support should
1366 /// be generated in the tail folded loop.
1370
1371 /// Returns true if the Phi is part of an inloop reduction.
1372 bool isInLoopReduction(PHINode *Phi) const {
1373 return InLoopReductions.contains(Phi);
1374 }
1375
1376 /// Returns true if the predicated reduction select should be used to set the
1377 /// incoming value for the reduction phi.
1379 // Force to use predicated reduction select since the EVL of the
1380 // second-to-last iteration might not be VF*UF.
1381 if (foldTailWithEVL())
1382 return true;
1384 TTI.preferPredicatedReductionSelect();
1385 }
1386
1387 /// Estimate cost of an intrinsic call instruction CI if it were vectorized
1388 /// with factor VF. Return the cost of the instruction, including
1389 /// scalarization overhead if it's needed.
1390 InstructionCost getVectorIntrinsicCost(CallInst *CI, ElementCount VF) const;
1391
1392 /// Estimate cost of a call instruction CI if it were vectorized with factor
1393 /// VF. Return the cost of the instruction, including scalarization overhead
1394 /// if it's needed.
1395 InstructionCost getVectorCallCost(CallInst *CI, ElementCount VF) const;
1396
1397 /// Invalidates decisions already taken by the cost model.
1399 WideningDecisions.clear();
1400 CallWideningDecisions.clear();
1401 Uniforms.clear();
1402 Scalars.clear();
1403 }
1404
1405 /// Returns the expected execution cost. The unit of the cost does
1406 /// not matter because we use the 'cost' units to compare different
1407 /// vector widths. The cost that is returned is *not* normalized by
1408 /// the factor width.
1409 InstructionCost expectedCost(ElementCount VF);
1410
1411 bool hasPredStores() const { return NumPredStores > 0; }
1412
1413 /// Returns true if epilogue vectorization is considered profitable, and
1414 /// false otherwise.
1415 /// \p VF is the vectorization factor chosen for the original loop.
1416 /// \p Multiplier is an aditional scaling factor applied to VF before
1417 /// comparing to EpilogueVectorizationMinVF.
1418 bool isEpilogueVectorizationProfitable(const ElementCount VF,
1419 const unsigned IC) const;
1420
1421 /// Returns the execution time cost of an instruction for a given vector
1422 /// width. Vector width of one means scalar.
1423 InstructionCost getInstructionCost(Instruction *I, ElementCount VF);
1424
1425 /// Return the cost of instructions in an inloop reduction pattern, if I is
1426 /// part of that pattern.
1427 std::optional<InstructionCost> getReductionPatternCost(Instruction *I,
1428 ElementCount VF,
1429 Type *VectorTy) const;
1430
1431 /// Returns true if \p Op should be considered invariant and if it is
1432 /// trivially hoistable.
1433 bool shouldConsiderInvariant(Value *Op);
1434
1435 /// Return the value of vscale used for tuning the cost model.
1436 std::optional<unsigned> getVScaleForTuning() const { return VScaleForTuning; }
1437
1438private:
1439 unsigned NumPredStores = 0;
1440
1441 /// Used to store the value of vscale used for tuning the cost model. It is
1442 /// initialized during object construction.
1443 std::optional<unsigned> VScaleForTuning;
1444
1445 /// Initializes the value of vscale used for tuning the cost model. If
1446 /// vscale_range.min == vscale_range.max then return vscale_range.max, else
1447 /// return the value returned by the corresponding TTI method.
1448 void initializeVScaleForTuning() {
1449 const Function *Fn = TheLoop->getHeader()->getParent();
1450 if (Fn->hasFnAttribute(Attribute::VScaleRange)) {
1451 auto Attr = Fn->getFnAttribute(Attribute::VScaleRange);
1452 auto Min = Attr.getVScaleRangeMin();
1453 auto Max = Attr.getVScaleRangeMax();
1454 if (Max && Min == Max) {
1455 VScaleForTuning = Max;
1456 return;
1457 }
1458 }
1459
1460 VScaleForTuning = TTI.getVScaleForTuning();
1461 }
1462
1463 /// \return An upper bound for the vectorization factors for both
1464 /// fixed and scalable vectorization, where the minimum-known number of
1465 /// elements is a power-of-2 larger than zero. If scalable vectorization is
1466 /// disabled or unsupported, then the scalable part will be equal to
1467 /// ElementCount::getScalable(0).
1468 FixedScalableVFPair computeFeasibleMaxVF(unsigned MaxTripCount,
1469 ElementCount UserVF,
1470 bool FoldTailByMasking);
1471
1472 /// If \p VF > MaxTripcount, clamps it to the next lower VF that is <=
1473 /// MaxTripCount.
1474 ElementCount clampVFByMaxTripCount(ElementCount VF, unsigned MaxTripCount,
1475 bool FoldTailByMasking) const;
1476
1477 /// \return the maximized element count based on the targets vector
1478 /// registers and the loop trip-count, but limited to a maximum safe VF.
1479 /// This is a helper function of computeFeasibleMaxVF.
1480 ElementCount getMaximizedVFForTarget(unsigned MaxTripCount,
1481 unsigned SmallestType,
1482 unsigned WidestType,
1483 ElementCount MaxSafeVF,
1484 bool FoldTailByMasking);
1485
1486 /// Checks if scalable vectorization is supported and enabled. Caches the
1487 /// result to avoid repeated debug dumps for repeated queries.
1488 bool isScalableVectorizationAllowed();
1489
1490 /// \return the maximum legal scalable VF, based on the safe max number
1491 /// of elements.
1492 ElementCount getMaxLegalScalableVF(unsigned MaxSafeElements);
1493
1494 /// Calculate vectorization cost of memory instruction \p I.
1495 InstructionCost getMemoryInstructionCost(Instruction *I, ElementCount VF);
1496
1497 /// The cost computation for scalarized memory instruction.
1498 InstructionCost getMemInstScalarizationCost(Instruction *I, ElementCount VF);
1499
1500 /// The cost computation for interleaving group of memory instructions.
1501 InstructionCost getInterleaveGroupCost(Instruction *I, ElementCount VF);
1502
1503 /// The cost computation for Gather/Scatter instruction.
1504 InstructionCost getGatherScatterCost(Instruction *I, ElementCount VF);
1505
1506 /// The cost computation for widening instruction \p I with consecutive
1507 /// memory access.
1508 InstructionCost getConsecutiveMemOpCost(Instruction *I, ElementCount VF);
1509
1510 /// The cost calculation for Load/Store instruction \p I with uniform pointer -
1511 /// Load: scalar load + broadcast.
1512 /// Store: scalar store + (loop invariant value stored? 0 : extract of last
1513 /// element)
1514 InstructionCost getUniformMemOpCost(Instruction *I, ElementCount VF);
1515
1516 /// Estimate the overhead of scalarizing an instruction. This is a
1517 /// convenience wrapper for the type-based getScalarizationOverhead API.
1519 ElementCount VF) const;
1520
1521 /// Returns true if an artificially high cost for emulated masked memrefs
1522 /// should be used.
1523 bool useEmulatedMaskMemRefHack(Instruction *I, ElementCount VF);
1524
1525 /// Map of scalar integer values to the smallest bitwidth they can be legally
1526 /// represented as. The vector equivalents of these values should be truncated
1527 /// to this type.
1528 MapVector<Instruction *, uint64_t> MinBWs;
1529
1530 /// A type representing the costs for instructions if they were to be
1531 /// scalarized rather than vectorized. The entries are Instruction-Cost
1532 /// pairs.
1533 using ScalarCostsTy = MapVector<Instruction *, InstructionCost>;
1534
1535 /// A set containing all BasicBlocks that are known to present after
1536 /// vectorization as a predicated block.
1537 DenseMap<ElementCount, SmallPtrSet<BasicBlock *, 4>>
1538 PredicatedBBsAfterVectorization;
1539
1540 /// Records whether it is allowed to have the original scalar loop execute at
1541 /// least once. This may be needed as a fallback loop in case runtime
1542 /// aliasing/dependence checks fail, or to handle the tail/remainder
1543 /// iterations when the trip count is unknown or doesn't divide by the VF,
1544 /// or as a peel-loop to handle gaps in interleave-groups.
1545 /// Under optsize and when the trip count is very small we don't allow any
1546 /// iterations to execute in the scalar loop.
1547 ScalarEpilogueLowering ScalarEpilogueStatus = CM_ScalarEpilogueAllowed;
1548
1549 /// Control finally chosen tail folding style. The first element is used if
1550 /// the IV update may overflow, the second element - if it does not.
1551 std::optional<std::pair<TailFoldingStyle, TailFoldingStyle>>
1552 ChosenTailFoldingStyle;
1553
1554 /// true if scalable vectorization is supported and enabled.
1555 std::optional<bool> IsScalableVectorizationAllowed;
1556
1557 /// Maximum safe number of elements to be processed per vector iteration,
1558 /// which do not prevent store-load forwarding and are safe with regard to the
1559 /// memory dependencies. Required for EVL-based veectorization, where this
1560 /// value is used as the upper bound of the safe AVL.
1561 std::optional<unsigned> MaxSafeElements;
1562
1563 /// A map holding scalar costs for different vectorization factors. The
1564 /// presence of a cost for an instruction in the mapping indicates that the
1565 /// instruction will be scalarized when vectorizing with the associated
1566 /// vectorization factor. The entries are VF-ScalarCostTy pairs.
1567 MapVector<ElementCount, ScalarCostsTy> InstsToScalarize;
1568
1569 /// Holds the instructions known to be uniform after vectorization.
1570 /// The data is collected per VF.
1571 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Uniforms;
1572
1573 /// Holds the instructions known to be scalar after vectorization.
1574 /// The data is collected per VF.
1575 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Scalars;
1576
1577 /// Holds the instructions (address computations) that are forced to be
1578 /// scalarized.
1579 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> ForcedScalars;
1580
1581 /// PHINodes of the reductions that should be expanded in-loop.
1582 SmallPtrSet<PHINode *, 4> InLoopReductions;
1583
1584 /// A Map of inloop reduction operations and their immediate chain operand.
1585 /// FIXME: This can be removed once reductions can be costed correctly in
1586 /// VPlan. This was added to allow quick lookup of the inloop operations.
1587 DenseMap<Instruction *, Instruction *> InLoopReductionImmediateChains;
1588
1589 /// Returns the expected difference in cost from scalarizing the expression
1590 /// feeding a predicated instruction \p PredInst. The instructions to
1591 /// scalarize and their scalar costs are collected in \p ScalarCosts. A
1592 /// non-negative return value implies the expression will be scalarized.
1593 /// Currently, only single-use chains are considered for scalarization.
1594 InstructionCost computePredInstDiscount(Instruction *PredInst,
1595 ScalarCostsTy &ScalarCosts,
1596 ElementCount VF);
1597
1598 /// Collect the instructions that are uniform after vectorization. An
1599 /// instruction is uniform if we represent it with a single scalar value in
1600 /// the vectorized loop corresponding to each vector iteration. Examples of
1601 /// uniform instructions include pointer operands of consecutive or
1602 /// interleaved memory accesses. Note that although uniformity implies an
1603 /// instruction will be scalar, the reverse is not true. In general, a
1604 /// scalarized instruction will be represented by VF scalar values in the
1605 /// vectorized loop, each corresponding to an iteration of the original
1606 /// scalar loop.
1607 void collectLoopUniforms(ElementCount VF);
1608
1609 /// Collect the instructions that are scalar after vectorization. An
1610 /// instruction is scalar if it is known to be uniform or will be scalarized
1611 /// during vectorization. collectLoopScalars should only add non-uniform nodes
1612 /// to the list if they are used by a load/store instruction that is marked as
1613 /// CM_Scalarize. Non-uniform scalarized instructions will be represented by
1614 /// VF values in the vectorized loop, each corresponding to an iteration of
1615 /// the original scalar loop.
1616 void collectLoopScalars(ElementCount VF);
1617
1618 /// Keeps cost model vectorization decision and cost for instructions.
1619 /// Right now it is used for memory instructions only.
1620 using DecisionList = DenseMap<std::pair<Instruction *, ElementCount>,
1621 std::pair<InstWidening, InstructionCost>>;
1622
1623 DecisionList WideningDecisions;
1624
1625 using CallDecisionList =
1626 DenseMap<std::pair<CallInst *, ElementCount>, CallWideningDecision>;
1627
1628 CallDecisionList CallWideningDecisions;
1629
1630 /// Returns true if \p V is expected to be vectorized and it needs to be
1631 /// extracted.
1632 bool needsExtract(Value *V, ElementCount VF) const {
1634 if (VF.isScalar() || !I || !TheLoop->contains(I) ||
1635 TheLoop->isLoopInvariant(I) ||
1636 getWideningDecision(I, VF) == CM_Scalarize ||
1637 (isa<CallInst>(I) &&
1638 getCallWideningDecision(cast<CallInst>(I), VF).Kind == CM_Scalarize))
1639 return false;
1640
1641 // Assume we can vectorize V (and hence we need extraction) if the
1642 // scalars are not computed yet. This can happen, because it is called
1643 // via getScalarizationOverhead from setCostBasedWideningDecision, before
1644 // the scalars are collected. That should be a safe assumption in most
1645 // cases, because we check if the operands have vectorizable types
1646 // beforehand in LoopVectorizationLegality.
1647 return !Scalars.contains(VF) || !isScalarAfterVectorization(I, VF);
1648 };
1649
1650 /// Returns a range containing only operands needing to be extracted.
1651 SmallVector<Value *, 4> filterExtractingOperands(Instruction::op_range Ops,
1652 ElementCount VF) const {
1653
1654 SmallPtrSet<const Value *, 4> UniqueOperands;
1656 for (Value *Op : Ops) {
1657 if (isa<Constant>(Op) || !UniqueOperands.insert(Op).second ||
1658 !needsExtract(Op, VF))
1659 continue;
1660 Res.push_back(Op);
1661 }
1662 return Res;
1663 }
1664
1665public:
1666 /// The loop that we evaluate.
1668
1669 /// Predicated scalar evolution analysis.
1671
1672 /// Loop Info analysis.
1674
1675 /// Vectorization legality.
1677
1678 /// Vector target information.
1680
1681 /// Target Library Info.
1683
1684 /// Demanded bits analysis.
1686
1687 /// Assumption cache.
1689
1690 /// Interface to emit optimization remarks.
1692
1694
1695 /// Loop Vectorize Hint.
1697
1698 /// The interleave access information contains groups of interleaved accesses
1699 /// with the same stride and close to each other.
1701
1702 /// Values to ignore in the cost model.
1704
1705 /// Values to ignore in the cost model when VF > 1.
1707
1708 /// All element types found in the loop.
1710
1711 /// The kind of cost that we are calculating
1713
1714 /// Whether this loop should be optimized for size based on function attribute
1715 /// or profile information.
1717
1718 /// The highest VF possible for this loop, without using MaxBandwidth.
1720};
1721} // end namespace llvm
1722
1723namespace {
1724/// Helper struct to manage generating runtime checks for vectorization.
1725///
1726/// The runtime checks are created up-front in temporary blocks to allow better
1727/// estimating the cost and un-linked from the existing IR. After deciding to
1728/// vectorize, the checks are moved back. If deciding not to vectorize, the
1729/// temporary blocks are completely removed.
1730class GeneratedRTChecks {
1731 /// Basic block which contains the generated SCEV checks, if any.
1732 BasicBlock *SCEVCheckBlock = nullptr;
1733
1734 /// The value representing the result of the generated SCEV checks. If it is
1735 /// nullptr no SCEV checks have been generated.
1736 Value *SCEVCheckCond = nullptr;
1737
1738 /// Basic block which contains the generated memory runtime checks, if any.
1739 BasicBlock *MemCheckBlock = nullptr;
1740
1741 /// The value representing the result of the generated memory runtime checks.
1742 /// If it is nullptr no memory runtime checks have been generated.
1743 Value *MemRuntimeCheckCond = nullptr;
1744
1745 DominatorTree *DT;
1746 LoopInfo *LI;
1748
1749 SCEVExpander SCEVExp;
1750 SCEVExpander MemCheckExp;
1751
1752 bool CostTooHigh = false;
1753
1754 Loop *OuterLoop = nullptr;
1755
1757
1758 /// The kind of cost that we are calculating
1760
1761public:
1762 GeneratedRTChecks(PredicatedScalarEvolution &PSE, DominatorTree *DT,
1765 : DT(DT), LI(LI), TTI(TTI),
1766 SCEVExp(*PSE.getSE(), DL, "scev.check", /*PreserveLCSSA=*/false),
1767 MemCheckExp(*PSE.getSE(), DL, "scev.check", /*PreserveLCSSA=*/false),
1768 PSE(PSE), CostKind(CostKind) {}
1769
1770 /// Generate runtime checks in SCEVCheckBlock and MemCheckBlock, so we can
1771 /// accurately estimate the cost of the runtime checks. The blocks are
1772 /// un-linked from the IR and are added back during vector code generation. If
1773 /// there is no vector code generation, the check blocks are removed
1774 /// completely.
1775 void create(Loop *L, const LoopAccessInfo &LAI,
1776 const SCEVPredicate &UnionPred, ElementCount VF, unsigned IC) {
1777
1778 // Hard cutoff to limit compile-time increase in case a very large number of
1779 // runtime checks needs to be generated.
1780 // TODO: Skip cutoff if the loop is guaranteed to execute, e.g. due to
1781 // profile info.
1782 CostTooHigh =
1784 if (CostTooHigh)
1785 return;
1786
1787 BasicBlock *LoopHeader = L->getHeader();
1788 BasicBlock *Preheader = L->getLoopPreheader();
1789
1790 // Use SplitBlock to create blocks for SCEV & memory runtime checks to
1791 // ensure the blocks are properly added to LoopInfo & DominatorTree. Those
1792 // may be used by SCEVExpander. The blocks will be un-linked from their
1793 // predecessors and removed from LI & DT at the end of the function.
1794 if (!UnionPred.isAlwaysTrue()) {
1795 SCEVCheckBlock = SplitBlock(Preheader, Preheader->getTerminator(), DT, LI,
1796 nullptr, "vector.scevcheck");
1797
1798 SCEVCheckCond = SCEVExp.expandCodeForPredicate(
1799 &UnionPred, SCEVCheckBlock->getTerminator());
1800 if (isa<Constant>(SCEVCheckCond)) {
1801 // Clean up directly after expanding the predicate to a constant, to
1802 // avoid further expansions re-using anything left over from SCEVExp.
1803 SCEVExpanderCleaner SCEVCleaner(SCEVExp);
1804 SCEVCleaner.cleanup();
1805 }
1806 }
1807
1808 const auto &RtPtrChecking = *LAI.getRuntimePointerChecking();
1809 if (RtPtrChecking.Need) {
1810 auto *Pred = SCEVCheckBlock ? SCEVCheckBlock : Preheader;
1811 MemCheckBlock = SplitBlock(Pred, Pred->getTerminator(), DT, LI, nullptr,
1812 "vector.memcheck");
1813
1814 auto DiffChecks = RtPtrChecking.getDiffChecks();
1815 if (DiffChecks) {
1816 Value *RuntimeVF = nullptr;
1817 MemRuntimeCheckCond = addDiffRuntimeChecks(
1818 MemCheckBlock->getTerminator(), *DiffChecks, MemCheckExp,
1819 [VF, &RuntimeVF](IRBuilderBase &B, unsigned Bits) {
1820 if (!RuntimeVF)
1821 RuntimeVF = getRuntimeVF(B, B.getIntNTy(Bits), VF);
1822 return RuntimeVF;
1823 },
1824 IC);
1825 } else {
1826 MemRuntimeCheckCond = addRuntimeChecks(
1827 MemCheckBlock->getTerminator(), L, RtPtrChecking.getChecks(),
1829 }
1830 assert(MemRuntimeCheckCond &&
1831 "no RT checks generated although RtPtrChecking "
1832 "claimed checks are required");
1833 }
1834
1835 SCEVExp.eraseDeadInstructions(SCEVCheckCond);
1836
1837 if (!MemCheckBlock && !SCEVCheckBlock)
1838 return;
1839
1840 // Unhook the temporary block with the checks, update various places
1841 // accordingly.
1842 if (SCEVCheckBlock)
1843 SCEVCheckBlock->replaceAllUsesWith(Preheader);
1844 if (MemCheckBlock)
1845 MemCheckBlock->replaceAllUsesWith(Preheader);
1846
1847 if (SCEVCheckBlock) {
1848 SCEVCheckBlock->getTerminator()->moveBefore(
1849 Preheader->getTerminator()->getIterator());
1850 auto *UI = new UnreachableInst(Preheader->getContext(), SCEVCheckBlock);
1851 UI->setDebugLoc(DebugLoc::getTemporary());
1852 Preheader->getTerminator()->eraseFromParent();
1853 }
1854 if (MemCheckBlock) {
1855 MemCheckBlock->getTerminator()->moveBefore(
1856 Preheader->getTerminator()->getIterator());
1857 auto *UI = new UnreachableInst(Preheader->getContext(), MemCheckBlock);
1858 UI->setDebugLoc(DebugLoc::getTemporary());
1859 Preheader->getTerminator()->eraseFromParent();
1860 }
1861
1862 DT->changeImmediateDominator(LoopHeader, Preheader);
1863 if (MemCheckBlock) {
1864 DT->eraseNode(MemCheckBlock);
1865 LI->removeBlock(MemCheckBlock);
1866 }
1867 if (SCEVCheckBlock) {
1868 DT->eraseNode(SCEVCheckBlock);
1869 LI->removeBlock(SCEVCheckBlock);
1870 }
1871
1872 // Outer loop is used as part of the later cost calculations.
1873 OuterLoop = L->getParentLoop();
1874 }
1875
1877 if (SCEVCheckBlock || MemCheckBlock)
1878 LLVM_DEBUG(dbgs() << "Calculating cost of runtime checks:\n");
1879
1880 if (CostTooHigh) {
1882 Cost.setInvalid();
1883 LLVM_DEBUG(dbgs() << " number of checks exceeded threshold\n");
1884 return Cost;
1885 }
1886
1887 InstructionCost RTCheckCost = 0;
1888 if (SCEVCheckBlock)
1889 for (Instruction &I : *SCEVCheckBlock) {
1890 if (SCEVCheckBlock->getTerminator() == &I)
1891 continue;
1893 LLVM_DEBUG(dbgs() << " " << C << " for " << I << "\n");
1894 RTCheckCost += C;
1895 }
1896 if (MemCheckBlock) {
1897 InstructionCost MemCheckCost = 0;
1898 for (Instruction &I : *MemCheckBlock) {
1899 if (MemCheckBlock->getTerminator() == &I)
1900 continue;
1902 LLVM_DEBUG(dbgs() << " " << C << " for " << I << "\n");
1903 MemCheckCost += C;
1904 }
1905
1906 // If the runtime memory checks are being created inside an outer loop
1907 // we should find out if these checks are outer loop invariant. If so,
1908 // the checks will likely be hoisted out and so the effective cost will
1909 // reduce according to the outer loop trip count.
1910 if (OuterLoop) {
1911 ScalarEvolution *SE = MemCheckExp.getSE();
1912 // TODO: If profitable, we could refine this further by analysing every
1913 // individual memory check, since there could be a mixture of loop
1914 // variant and invariant checks that mean the final condition is
1915 // variant.
1916 const SCEV *Cond = SE->getSCEV(MemRuntimeCheckCond);
1917 if (SE->isLoopInvariant(Cond, OuterLoop)) {
1918 // It seems reasonable to assume that we can reduce the effective
1919 // cost of the checks even when we know nothing about the trip
1920 // count. Assume that the outer loop executes at least twice.
1921 unsigned BestTripCount = 2;
1922
1923 // Get the best known TC estimate.
1924 if (auto EstimatedTC = getSmallBestKnownTC(
1925 PSE, OuterLoop, /* CanUseConstantMax = */ false))
1926 if (EstimatedTC->isFixed())
1927 BestTripCount = EstimatedTC->getFixedValue();
1928
1929 InstructionCost NewMemCheckCost = MemCheckCost / BestTripCount;
1930
1931 // Let's ensure the cost is always at least 1.
1932 NewMemCheckCost = std::max(NewMemCheckCost.getValue(),
1933 (InstructionCost::CostType)1);
1934
1935 if (BestTripCount > 1)
1937 << "We expect runtime memory checks to be hoisted "
1938 << "out of the outer loop. Cost reduced from "
1939 << MemCheckCost << " to " << NewMemCheckCost << '\n');
1940
1941 MemCheckCost = NewMemCheckCost;
1942 }
1943 }
1944
1945 RTCheckCost += MemCheckCost;
1946 }
1947
1948 if (SCEVCheckBlock || MemCheckBlock)
1949 LLVM_DEBUG(dbgs() << "Total cost of runtime checks: " << RTCheckCost
1950 << "\n");
1951
1952 return RTCheckCost;
1953 }
1954
1955 /// Remove the created SCEV & memory runtime check blocks & instructions, if
1956 /// unused.
1957 ~GeneratedRTChecks() {
1958 SCEVExpanderCleaner SCEVCleaner(SCEVExp);
1959 SCEVExpanderCleaner MemCheckCleaner(MemCheckExp);
1960 bool SCEVChecksUsed = !SCEVCheckBlock || !pred_empty(SCEVCheckBlock);
1961 bool MemChecksUsed = !MemCheckBlock || !pred_empty(MemCheckBlock);
1962 if (SCEVChecksUsed)
1963 SCEVCleaner.markResultUsed();
1964
1965 if (MemChecksUsed) {
1966 MemCheckCleaner.markResultUsed();
1967 } else {
1968 auto &SE = *MemCheckExp.getSE();
1969 // Memory runtime check generation creates compares that use expanded
1970 // values. Remove them before running the SCEVExpanderCleaners.
1971 for (auto &I : make_early_inc_range(reverse(*MemCheckBlock))) {
1972 if (MemCheckExp.isInsertedInstruction(&I))
1973 continue;
1974 SE.forgetValue(&I);
1975 I.eraseFromParent();
1976 }
1977 }
1978 MemCheckCleaner.cleanup();
1979 SCEVCleaner.cleanup();
1980
1981 if (!SCEVChecksUsed)
1982 SCEVCheckBlock->eraseFromParent();
1983 if (!MemChecksUsed)
1984 MemCheckBlock->eraseFromParent();
1985 }
1986
1987 /// Retrieves the SCEVCheckCond and SCEVCheckBlock that were generated as IR
1988 /// outside VPlan.
1989 std::pair<Value *, BasicBlock *> getSCEVChecks() const {
1990 using namespace llvm::PatternMatch;
1991 if (!SCEVCheckCond || match(SCEVCheckCond, m_ZeroInt()))
1992 return {nullptr, nullptr};
1993
1994 return {SCEVCheckCond, SCEVCheckBlock};
1995 }
1996
1997 /// Retrieves the MemCheckCond and MemCheckBlock that were generated as IR
1998 /// outside VPlan.
1999 std::pair<Value *, BasicBlock *> getMemRuntimeChecks() const {
2000 using namespace llvm::PatternMatch;
2001 if (MemRuntimeCheckCond && match(MemRuntimeCheckCond, m_ZeroInt()))
2002 return {nullptr, nullptr};
2003 return {MemRuntimeCheckCond, MemCheckBlock};
2004 }
2005
2006 /// Return true if any runtime checks have been added
2007 bool hasChecks() const {
2008 return getSCEVChecks().first || getMemRuntimeChecks().first;
2009 }
2010};
2011} // namespace
2012
2018
2023
2024// Return true if \p OuterLp is an outer loop annotated with hints for explicit
2025// vectorization. The loop needs to be annotated with #pragma omp simd
2026// simdlen(#) or #pragma clang vectorize(enable) vectorize_width(#). If the
2027// vector length information is not provided, vectorization is not considered
2028// explicit. Interleave hints are not allowed either. These limitations will be
2029// relaxed in the future.
2030// Please, note that we are currently forced to abuse the pragma 'clang
2031// vectorize' semantics. This pragma provides *auto-vectorization hints*
2032// (i.e., LV must check that vectorization is legal) whereas pragma 'omp simd'
2033// provides *explicit vectorization hints* (LV can bypass legal checks and
2034// assume that vectorization is legal). However, both hints are implemented
2035// using the same metadata (llvm.loop.vectorize, processed by
2036// LoopVectorizeHints). This will be fixed in the future when the native IR
2037// representation for pragma 'omp simd' is introduced.
2038static bool isExplicitVecOuterLoop(Loop *OuterLp,
2040 assert(!OuterLp->isInnermost() && "This is not an outer loop");
2041 LoopVectorizeHints Hints(OuterLp, true /*DisableInterleaving*/, *ORE);
2042
2043 // Only outer loops with an explicit vectorization hint are supported.
2044 // Unannotated outer loops are ignored.
2046 return false;
2047
2048 Function *Fn = OuterLp->getHeader()->getParent();
2049 if (!Hints.allowVectorization(Fn, OuterLp,
2050 true /*VectorizeOnlyWhenForced*/)) {
2051 LLVM_DEBUG(dbgs() << "LV: Loop hints prevent outer loop vectorization.\n");
2052 return false;
2053 }
2054
2055 if (Hints.getInterleave() > 1) {
2056 // TODO: Interleave support is future work.
2057 LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Interleave is not supported for "
2058 "outer loops.\n");
2059 Hints.emitRemarkWithHints();
2060 return false;
2061 }
2062
2063 return true;
2064}
2065
2069 // Collect inner loops and outer loops without irreducible control flow. For
2070 // now, only collect outer loops that have explicit vectorization hints. If we
2071 // are stress testing the VPlan H-CFG construction, we collect the outermost
2072 // loop of every loop nest.
2073 if (L.isInnermost() || VPlanBuildStressTest ||
2075 LoopBlocksRPO RPOT(&L);
2076 RPOT.perform(LI);
2078 V.push_back(&L);
2079 // TODO: Collect inner loops inside marked outer loops in case
2080 // vectorization fails for the outer loop. Do not invoke
2081 // 'containsIrreducibleCFG' again for inner loops when the outer loop is
2082 // already known to be reducible. We can use an inherited attribute for
2083 // that.
2084 return;
2085 }
2086 }
2087 for (Loop *InnerL : L)
2088 collectSupportedLoops(*InnerL, LI, ORE, V);
2089}
2090
2091//===----------------------------------------------------------------------===//
2092// Implementation of LoopVectorizationLegality, InnerLoopVectorizer and
2093// LoopVectorizationCostModel and LoopVectorizationPlanner.
2094//===----------------------------------------------------------------------===//
2095
2096/// Compute the transformed value of Index at offset StartValue using step
2097/// StepValue.
2098/// For integer induction, returns StartValue + Index * StepValue.
2099/// For pointer induction, returns StartValue[Index * StepValue].
2100/// FIXME: The newly created binary instructions should contain nsw/nuw
2101/// flags, which can be found from the original scalar operations.
2102static Value *
2104 Value *Step,
2106 const BinaryOperator *InductionBinOp) {
2107 using namespace llvm::PatternMatch;
2108 Type *StepTy = Step->getType();
2109 Value *CastedIndex = StepTy->isIntegerTy()
2110 ? B.CreateSExtOrTrunc(Index, StepTy)
2111 : B.CreateCast(Instruction::SIToFP, Index, StepTy);
2112 if (CastedIndex != Index) {
2113 CastedIndex->setName(CastedIndex->getName() + ".cast");
2114 Index = CastedIndex;
2115 }
2116
2117 // Note: the IR at this point is broken. We cannot use SE to create any new
2118 // SCEV and then expand it, hoping that SCEV's simplification will give us
2119 // a more optimal code. Unfortunately, attempt of doing so on invalid IR may
2120 // lead to various SCEV crashes. So all we can do is to use builder and rely
2121 // on InstCombine for future simplifications. Here we handle some trivial
2122 // cases only.
2123 auto CreateAdd = [&B](Value *X, Value *Y) {
2124 assert(X->getType() == Y->getType() && "Types don't match!");
2125 if (match(X, m_ZeroInt()))
2126 return Y;
2127 if (match(Y, m_ZeroInt()))
2128 return X;
2129 return B.CreateAdd(X, Y);
2130 };
2131
2132 // We allow X to be a vector type, in which case Y will potentially be
2133 // splatted into a vector with the same element count.
2134 auto CreateMul = [&B](Value *X, Value *Y) {
2135 assert(X->getType()->getScalarType() == Y->getType() &&
2136 "Types don't match!");
2137 if (match(X, m_One()))
2138 return Y;
2139 if (match(Y, m_One()))
2140 return X;
2141 VectorType *XVTy = dyn_cast<VectorType>(X->getType());
2142 if (XVTy && !isa<VectorType>(Y->getType()))
2143 Y = B.CreateVectorSplat(XVTy->getElementCount(), Y);
2144 return B.CreateMul(X, Y);
2145 };
2146
2147 switch (InductionKind) {
2149 assert(!isa<VectorType>(Index->getType()) &&
2150 "Vector indices not supported for integer inductions yet");
2151 assert(Index->getType() == StartValue->getType() &&
2152 "Index type does not match StartValue type");
2153 if (isa<ConstantInt>(Step) && cast<ConstantInt>(Step)->isMinusOne())
2154 return B.CreateSub(StartValue, Index);
2155 auto *Offset = CreateMul(Index, Step);
2156 return CreateAdd(StartValue, Offset);
2157 }
2159 return B.CreatePtrAdd(StartValue, CreateMul(Index, Step));
2161 assert(!isa<VectorType>(Index->getType()) &&
2162 "Vector indices not supported for FP inductions yet");
2163 assert(Step->getType()->isFloatingPointTy() && "Expected FP Step value");
2164 assert(InductionBinOp &&
2165 (InductionBinOp->getOpcode() == Instruction::FAdd ||
2166 InductionBinOp->getOpcode() == Instruction::FSub) &&
2167 "Original bin op should be defined for FP induction");
2168
2169 Value *MulExp = B.CreateFMul(Step, Index);
2170 return B.CreateBinOp(InductionBinOp->getOpcode(), StartValue, MulExp,
2171 "induction");
2172 }
2174 return nullptr;
2175 }
2176 llvm_unreachable("invalid enum");
2177}
2178
2179static std::optional<unsigned> getMaxVScale(const Function &F,
2180 const TargetTransformInfo &TTI) {
2181 if (std::optional<unsigned> MaxVScale = TTI.getMaxVScale())
2182 return MaxVScale;
2183
2184 if (F.hasFnAttribute(Attribute::VScaleRange))
2185 return F.getFnAttribute(Attribute::VScaleRange).getVScaleRangeMax();
2186
2187 return std::nullopt;
2188}
2189
2190/// For the given VF and UF and maximum trip count computed for the loop, return
2191/// whether the induction variable might overflow in the vectorized loop. If not,
2192/// then we know a runtime overflow check always evaluates to false and can be
2193/// removed.
2195 const LoopVectorizationCostModel *Cost,
2196 ElementCount VF, std::optional<unsigned> UF = std::nullopt) {
2197 // Always be conservative if we don't know the exact unroll factor.
2198 unsigned MaxUF = UF ? *UF : Cost->TTI.getMaxInterleaveFactor(VF);
2199
2200 IntegerType *IdxTy = Cost->Legal->getWidestInductionType();
2201 APInt MaxUIntTripCount = IdxTy->getMask();
2202
2203 // We know the runtime overflow check is known false iff the (max) trip-count
2204 // is known and (max) trip-count + (VF * UF) does not overflow in the type of
2205 // the vector loop induction variable.
2206 if (unsigned TC = Cost->PSE.getSmallConstantMaxTripCount()) {
2207 uint64_t MaxVF = VF.getKnownMinValue();
2208 if (VF.isScalable()) {
2209 std::optional<unsigned> MaxVScale =
2210 getMaxVScale(*Cost->TheFunction, Cost->TTI);
2211 if (!MaxVScale)
2212 return false;
2213 MaxVF *= *MaxVScale;
2214 }
2215
2216 return (MaxUIntTripCount - TC).ugt(MaxVF * MaxUF);
2217 }
2218
2219 return false;
2220}
2221
2222// Return whether we allow using masked interleave-groups (for dealing with
2223// strided loads/stores that reside in predicated blocks, or for dealing
2224// with gaps).
2226 // If an override option has been passed in for interleaved accesses, use it.
2227 if (EnableMaskedInterleavedMemAccesses.getNumOccurrences() > 0)
2229
2230 return TTI.enableMaskedInterleavedAccessVectorization();
2231}
2232
2234 BasicBlock *CheckIRBB) {
2235 // Note: The block with the minimum trip-count check is already connected
2236 // during earlier VPlan construction.
2237 VPBlockBase *ScalarPH = Plan.getScalarPreheader();
2238 VPBlockBase *PreVectorPH = VectorPHVPBB->getSinglePredecessor();
2239 assert(PreVectorPH->getNumSuccessors() == 2 && "Expected 2 successors");
2240 assert(PreVectorPH->getSuccessors()[0] == ScalarPH && "Unexpected successor");
2241 VPIRBasicBlock *CheckVPIRBB = Plan.createVPIRBasicBlock(CheckIRBB);
2242 VPBlockUtils::insertOnEdge(PreVectorPH, VectorPHVPBB, CheckVPIRBB);
2243 PreVectorPH = CheckVPIRBB;
2244 VPBlockUtils::connectBlocks(PreVectorPH, ScalarPH);
2245 PreVectorPH->swapSuccessors();
2246
2247 // We just connected a new block to the scalar preheader. Update all
2248 // VPPhis by adding an incoming value for it, replicating the last value.
2249 unsigned NumPredecessors = ScalarPH->getNumPredecessors();
2250 for (VPRecipeBase &R : cast<VPBasicBlock>(ScalarPH)->phis()) {
2251 assert(isa<VPPhi>(&R) && "Phi expected to be VPPhi");
2252 assert(cast<VPPhi>(&R)->getNumIncoming() == NumPredecessors - 1 &&
2253 "must have incoming values for all operands");
2254 R.addOperand(R.getOperand(NumPredecessors - 2));
2255 }
2256}
2257
2259 BasicBlock *VectorPH, ElementCount VF, unsigned UF) const {
2260 // Generate code to check if the loop's trip count is less than VF * UF, or
2261 // equal to it in case a scalar epilogue is required; this implies that the
2262 // vector trip count is zero. This check also covers the case where adding one
2263 // to the backedge-taken count overflowed leading to an incorrect trip count
2264 // of zero. In this case we will also jump to the scalar loop.
2265 auto P = Cost->requiresScalarEpilogue(VF.isVector()) ? ICmpInst::ICMP_ULE
2267
2268 // Reuse existing vector loop preheader for TC checks.
2269 // Note that new preheader block is generated for vector loop.
2270 BasicBlock *const TCCheckBlock = VectorPH;
2272 TCCheckBlock->getContext(),
2273 InstSimplifyFolder(TCCheckBlock->getDataLayout()));
2274 Builder.SetInsertPoint(TCCheckBlock->getTerminator());
2275
2276 // If tail is to be folded, vector loop takes care of all iterations.
2278 Type *CountTy = Count->getType();
2279 Value *CheckMinIters = Builder.getFalse();
2280 auto CreateStep = [&]() -> Value * {
2281 // Create step with max(MinProTripCount, UF * VF).
2282 if (UF * VF.getKnownMinValue() >= MinProfitableTripCount.getKnownMinValue())
2283 return createStepForVF(Builder, CountTy, VF, UF);
2284
2285 Value *MinProfTC =
2286 Builder.CreateElementCount(CountTy, MinProfitableTripCount);
2287 if (!VF.isScalable())
2288 return MinProfTC;
2289 return Builder.CreateBinaryIntrinsic(
2290 Intrinsic::umax, MinProfTC, createStepForVF(Builder, CountTy, VF, UF));
2291 };
2292
2293 TailFoldingStyle Style = Cost->getTailFoldingStyle();
2294 if (Style == TailFoldingStyle::None) {
2295 Value *Step = CreateStep();
2296 ScalarEvolution &SE = *PSE.getSE();
2297 // TODO: Emit unconditional branch to vector preheader instead of
2298 // conditional branch with known condition.
2299 const SCEV *TripCountSCEV = SE.applyLoopGuards(SE.getSCEV(Count), OrigLoop);
2300 // Check if the trip count is < the step.
2301 if (SE.isKnownPredicate(P, TripCountSCEV, SE.getSCEV(Step))) {
2302 // TODO: Ensure step is at most the trip count when determining max VF and
2303 // UF, w/o tail folding.
2304 CheckMinIters = Builder.getTrue();
2306 TripCountSCEV, SE.getSCEV(Step))) {
2307 // Generate the minimum iteration check only if we cannot prove the
2308 // check is known to be true, or known to be false.
2309 CheckMinIters = Builder.CreateICmp(P, Count, Step, "min.iters.check");
2310 } // else step known to be < trip count, use CheckMinIters preset to false.
2311 } else if (VF.isScalable() && !TTI->isVScaleKnownToBeAPowerOfTwo() &&
2314 // vscale is not necessarily a power-of-2, which means we cannot guarantee
2315 // an overflow to zero when updating induction variables and so an
2316 // additional overflow check is required before entering the vector loop.
2317
2318 // Get the maximum unsigned value for the type.
2319 Value *MaxUIntTripCount =
2320 ConstantInt::get(CountTy, cast<IntegerType>(CountTy)->getMask());
2321 Value *LHS = Builder.CreateSub(MaxUIntTripCount, Count);
2322
2323 // Don't execute the vector loop if (UMax - n) < (VF * UF).
2324 CheckMinIters = Builder.CreateICmp(ICmpInst::ICMP_ULT, LHS, CreateStep());
2325 }
2326 return CheckMinIters;
2327}
2328
2329/// Replace \p VPBB with a VPIRBasicBlock wrapping \p IRBB. All recipes from \p
2330/// VPBB are moved to the end of the newly created VPIRBasicBlock. All
2331/// predecessors and successors of VPBB, if any, are rewired to the new
2332/// VPIRBasicBlock. If \p VPBB may be unreachable, \p Plan must be passed.
2334 BasicBlock *IRBB,
2335 VPlan *Plan = nullptr) {
2336 if (!Plan)
2337 Plan = VPBB->getPlan();
2338 VPIRBasicBlock *IRVPBB = Plan->createVPIRBasicBlock(IRBB);
2339 auto IP = IRVPBB->begin();
2340 for (auto &R : make_early_inc_range(VPBB->phis()))
2341 R.moveBefore(*IRVPBB, IP);
2342
2343 for (auto &R :
2345 R.moveBefore(*IRVPBB, IRVPBB->end());
2346
2347 VPBlockUtils::reassociateBlocks(VPBB, IRVPBB);
2348 // VPBB is now dead and will be cleaned up when the plan gets destroyed.
2349 return IRVPBB;
2350}
2351
2353 BasicBlock *VectorPH = OrigLoop->getLoopPreheader();
2354 assert(VectorPH && "Invalid loop structure");
2355 assert((OrigLoop->getUniqueLatchExitBlock() ||
2356 Cost->requiresScalarEpilogue(VF.isVector())) &&
2357 "loops not exiting via the latch without required epilogue?");
2358
2359 // NOTE: The Plan's scalar preheader VPBB isn't replaced with a VPIRBasicBlock
2360 // wrapping the newly created scalar preheader here at the moment, because the
2361 // Plan's scalar preheader may be unreachable at this point. Instead it is
2362 // replaced in executePlan.
2363 return SplitBlock(VectorPH, VectorPH->getTerminator(), DT, LI, nullptr,
2364 Twine(Prefix) + "scalar.ph");
2365}
2366
2367/// Return the expanded step for \p ID using \p ExpandedSCEVs to look up SCEV
2368/// expansion results.
2370 const SCEV2ValueTy &ExpandedSCEVs) {
2371 const SCEV *Step = ID.getStep();
2372 if (auto *C = dyn_cast<SCEVConstant>(Step))
2373 return C->getValue();
2374 if (auto *U = dyn_cast<SCEVUnknown>(Step))
2375 return U->getValue();
2376 Value *V = ExpandedSCEVs.lookup(Step);
2377 assert(V && "SCEV must be expanded at this point");
2378 return V;
2379}
2380
2381/// Knowing that loop \p L executes a single vector iteration, add instructions
2382/// that will get simplified and thus should not have any cost to \p
2383/// InstsToIgnore.
2386 SmallPtrSetImpl<Instruction *> &InstsToIgnore) {
2387 auto *Cmp = L->getLatchCmpInst();
2388 if (Cmp)
2389 InstsToIgnore.insert(Cmp);
2390 for (const auto &KV : IL) {
2391 // Extract the key by hand so that it can be used in the lambda below. Note
2392 // that captured structured bindings are a C++20 extension.
2393 const PHINode *IV = KV.first;
2394
2395 // Get next iteration value of the induction variable.
2396 Instruction *IVInst =
2397 cast<Instruction>(IV->getIncomingValueForBlock(L->getLoopLatch()));
2398 if (all_of(IVInst->users(),
2399 [&](const User *U) { return U == IV || U == Cmp; }))
2400 InstsToIgnore.insert(IVInst);
2401 }
2402}
2403
2405 // Create a new IR basic block for the scalar preheader.
2406 BasicBlock *ScalarPH = createScalarPreheader("");
2407 return ScalarPH->getSinglePredecessor();
2408}
2409
2410namespace {
2411
2412struct CSEDenseMapInfo {
2413 static bool canHandle(const Instruction *I) {
2416 }
2417
2418 static inline Instruction *getEmptyKey() {
2420 }
2421
2422 static inline Instruction *getTombstoneKey() {
2423 return DenseMapInfo<Instruction *>::getTombstoneKey();
2424 }
2425
2426 static unsigned getHashValue(const Instruction *I) {
2427 assert(canHandle(I) && "Unknown instruction!");
2428 return hash_combine(I->getOpcode(),
2429 hash_combine_range(I->operand_values()));
2430 }
2431
2432 static bool isEqual(const Instruction *LHS, const Instruction *RHS) {
2433 if (LHS == getEmptyKey() || RHS == getEmptyKey() ||
2434 LHS == getTombstoneKey() || RHS == getTombstoneKey())
2435 return LHS == RHS;
2436 return LHS->isIdenticalTo(RHS);
2437 }
2438};
2439
2440} // end anonymous namespace
2441
2442/// FIXME: This legacy common-subexpression-elimination routine is scheduled for
2443/// removal, in favor of the VPlan-based one.
2444static void legacyCSE(BasicBlock *BB) {
2445 // Perform simple cse.
2447 for (Instruction &In : llvm::make_early_inc_range(*BB)) {
2448 if (!CSEDenseMapInfo::canHandle(&In))
2449 continue;
2450
2451 // Check if we can replace this instruction with any of the
2452 // visited instructions.
2453 if (Instruction *V = CSEMap.lookup(&In)) {
2454 In.replaceAllUsesWith(V);
2455 In.eraseFromParent();
2456 continue;
2457 }
2458
2459 CSEMap[&In] = &In;
2460 }
2461}
2462
2463/// This function attempts to return a value that represents the ElementCount
2464/// at runtime. For fixed-width VFs we know this precisely at compile
2465/// time, but for scalable VFs we calculate it based on an estimate of the
2466/// vscale value.
2468 std::optional<unsigned> VScale) {
2469 unsigned EstimatedVF = VF.getKnownMinValue();
2470 if (VF.isScalable())
2471 if (VScale)
2472 EstimatedVF *= *VScale;
2473 assert(EstimatedVF >= 1 && "Estimated VF shouldn't be less than 1");
2474 return EstimatedVF;
2475}
2476
2479 ElementCount VF) const {
2480 // We only need to calculate a cost if the VF is scalar; for actual vectors
2481 // we should already have a pre-calculated cost at each VF.
2482 if (!VF.isScalar())
2483 return getCallWideningDecision(CI, VF).Cost;
2484
2485 Type *RetTy = CI->getType();
2487 if (auto RedCost = getReductionPatternCost(CI, VF, RetTy))
2488 return *RedCost;
2489
2491 for (auto &ArgOp : CI->args())
2492 Tys.push_back(ArgOp->getType());
2493
2494 InstructionCost ScalarCallCost =
2495 TTI.getCallInstrCost(CI->getCalledFunction(), RetTy, Tys, CostKind);
2496
2497 // If this is an intrinsic we may have a lower cost for it.
2500 return std::min(ScalarCallCost, IntrinsicCost);
2501 }
2502 return ScalarCallCost;
2503}
2504
2506 if (VF.isScalar() || !canVectorizeTy(Ty))
2507 return Ty;
2508 return toVectorizedTy(Ty, VF);
2509}
2510
2513 ElementCount VF) const {
2515 assert(ID && "Expected intrinsic call!");
2516 Type *RetTy = maybeVectorizeType(CI->getType(), VF);
2517 FastMathFlags FMF;
2518 if (auto *FPMO = dyn_cast<FPMathOperator>(CI))
2519 FMF = FPMO->getFastMathFlags();
2520
2523 SmallVector<Type *> ParamTys;
2524 std::transform(FTy->param_begin(), FTy->param_end(),
2525 std::back_inserter(ParamTys),
2526 [&](Type *Ty) { return maybeVectorizeType(Ty, VF); });
2527
2528 IntrinsicCostAttributes CostAttrs(ID, RetTy, Arguments, ParamTys, FMF,
2531 return TTI.getIntrinsicInstrCost(CostAttrs, CostKind);
2532}
2533
2535 // Fix widened non-induction PHIs by setting up the PHI operands.
2536 fixNonInductionPHIs(State);
2537
2538 // Don't apply optimizations below when no (vector) loop remains, as they all
2539 // require one at the moment.
2540 VPBasicBlock *HeaderVPBB =
2541 vputils::getFirstLoopHeader(*State.Plan, State.VPDT);
2542 if (!HeaderVPBB)
2543 return;
2544
2545 BasicBlock *HeaderBB = State.CFG.VPBB2IRBB[HeaderVPBB];
2546
2547 // Remove redundant induction instructions.
2548 legacyCSE(HeaderBB);
2549}
2550
2552 auto Iter = vp_depth_first_shallow(Plan.getEntry());
2554 for (VPRecipeBase &P : VPBB->phis()) {
2556 if (!VPPhi)
2557 continue;
2558 PHINode *NewPhi = cast<PHINode>(State.get(VPPhi));
2559 // Make sure the builder has a valid insert point.
2560 Builder.SetInsertPoint(NewPhi);
2561 for (const auto &[Inc, VPBB] : VPPhi->incoming_values_and_blocks())
2562 NewPhi->addIncoming(State.get(Inc), State.CFG.VPBB2IRBB[VPBB]);
2563 }
2564 }
2565}
2566
2567void LoopVectorizationCostModel::collectLoopScalars(ElementCount VF) {
2568 // We should not collect Scalars more than once per VF. Right now, this
2569 // function is called from collectUniformsAndScalars(), which already does
2570 // this check. Collecting Scalars for VF=1 does not make any sense.
2571 assert(VF.isVector() && !Scalars.contains(VF) &&
2572 "This function should not be visited twice for the same VF");
2573
2574 // This avoids any chances of creating a REPLICATE recipe during planning
2575 // since that would result in generation of scalarized code during execution,
2576 // which is not supported for scalable vectors.
2577 if (VF.isScalable()) {
2578 Scalars[VF].insert_range(Uniforms[VF]);
2579 return;
2580 }
2581
2583
2584 // These sets are used to seed the analysis with pointers used by memory
2585 // accesses that will remain scalar.
2587 SmallPtrSet<Instruction *, 8> PossibleNonScalarPtrs;
2588 auto *Latch = TheLoop->getLoopLatch();
2589
2590 // A helper that returns true if the use of Ptr by MemAccess will be scalar.
2591 // The pointer operands of loads and stores will be scalar as long as the
2592 // memory access is not a gather or scatter operation. The value operand of a
2593 // store will remain scalar if the store is scalarized.
2594 auto IsScalarUse = [&](Instruction *MemAccess, Value *Ptr) {
2595 InstWidening WideningDecision = getWideningDecision(MemAccess, VF);
2596 assert(WideningDecision != CM_Unknown &&
2597 "Widening decision should be ready at this moment");
2598 if (auto *Store = dyn_cast<StoreInst>(MemAccess))
2599 if (Ptr == Store->getValueOperand())
2600 return WideningDecision == CM_Scalarize;
2601 assert(Ptr == getLoadStorePointerOperand(MemAccess) &&
2602 "Ptr is neither a value or pointer operand");
2603 return WideningDecision != CM_GatherScatter;
2604 };
2605
2606 // A helper that returns true if the given value is a getelementptr
2607 // instruction contained in the loop.
2608 auto IsLoopVaryingGEP = [&](Value *V) {
2609 return isa<GetElementPtrInst>(V) && !TheLoop->isLoopInvariant(V);
2610 };
2611
2612 // A helper that evaluates a memory access's use of a pointer. If the use will
2613 // be a scalar use and the pointer is only used by memory accesses, we place
2614 // the pointer in ScalarPtrs. Otherwise, the pointer is placed in
2615 // PossibleNonScalarPtrs.
2616 auto EvaluatePtrUse = [&](Instruction *MemAccess, Value *Ptr) {
2617 // We only care about bitcast and getelementptr instructions contained in
2618 // the loop.
2619 if (!IsLoopVaryingGEP(Ptr))
2620 return;
2621
2622 // If the pointer has already been identified as scalar (e.g., if it was
2623 // also identified as uniform), there's nothing to do.
2624 auto *I = cast<Instruction>(Ptr);
2625 if (Worklist.count(I))
2626 return;
2627
2628 // If the use of the pointer will be a scalar use, and all users of the
2629 // pointer are memory accesses, place the pointer in ScalarPtrs. Otherwise,
2630 // place the pointer in PossibleNonScalarPtrs.
2631 if (IsScalarUse(MemAccess, Ptr) &&
2633 ScalarPtrs.insert(I);
2634 else
2635 PossibleNonScalarPtrs.insert(I);
2636 };
2637
2638 // We seed the scalars analysis with three classes of instructions: (1)
2639 // instructions marked uniform-after-vectorization and (2) bitcast,
2640 // getelementptr and (pointer) phi instructions used by memory accesses
2641 // requiring a scalar use.
2642 //
2643 // (1) Add to the worklist all instructions that have been identified as
2644 // uniform-after-vectorization.
2645 Worklist.insert_range(Uniforms[VF]);
2646
2647 // (2) Add to the worklist all bitcast and getelementptr instructions used by
2648 // memory accesses requiring a scalar use. The pointer operands of loads and
2649 // stores will be scalar unless the operation is a gather or scatter.
2650 // The value operand of a store will remain scalar if the store is scalarized.
2651 for (auto *BB : TheLoop->blocks())
2652 for (auto &I : *BB) {
2653 if (auto *Load = dyn_cast<LoadInst>(&I)) {
2654 EvaluatePtrUse(Load, Load->getPointerOperand());
2655 } else if (auto *Store = dyn_cast<StoreInst>(&I)) {
2656 EvaluatePtrUse(Store, Store->getPointerOperand());
2657 EvaluatePtrUse(Store, Store->getValueOperand());
2658 }
2659 }
2660 for (auto *I : ScalarPtrs)
2661 if (!PossibleNonScalarPtrs.count(I)) {
2662 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *I << "\n");
2663 Worklist.insert(I);
2664 }
2665
2666 // Insert the forced scalars.
2667 // FIXME: Currently VPWidenPHIRecipe() often creates a dead vector
2668 // induction variable when the PHI user is scalarized.
2669 auto ForcedScalar = ForcedScalars.find(VF);
2670 if (ForcedScalar != ForcedScalars.end())
2671 for (auto *I : ForcedScalar->second) {
2672 LLVM_DEBUG(dbgs() << "LV: Found (forced) scalar instruction: " << *I << "\n");
2673 Worklist.insert(I);
2674 }
2675
2676 // Expand the worklist by looking through any bitcasts and getelementptr
2677 // instructions we've already identified as scalar. This is similar to the
2678 // expansion step in collectLoopUniforms(); however, here we're only
2679 // expanding to include additional bitcasts and getelementptr instructions.
2680 unsigned Idx = 0;
2681 while (Idx != Worklist.size()) {
2682 Instruction *Dst = Worklist[Idx++];
2683 if (!IsLoopVaryingGEP(Dst->getOperand(0)))
2684 continue;
2685 auto *Src = cast<Instruction>(Dst->getOperand(0));
2686 if (llvm::all_of(Src->users(), [&](User *U) -> bool {
2687 auto *J = cast<Instruction>(U);
2688 return !TheLoop->contains(J) || Worklist.count(J) ||
2689 ((isa<LoadInst>(J) || isa<StoreInst>(J)) &&
2690 IsScalarUse(J, Src));
2691 })) {
2692 Worklist.insert(Src);
2693 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Src << "\n");
2694 }
2695 }
2696
2697 // An induction variable will remain scalar if all users of the induction
2698 // variable and induction variable update remain scalar.
2699 for (const auto &Induction : Legal->getInductionVars()) {
2700 auto *Ind = Induction.first;
2701 auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch));
2702
2703 // If tail-folding is applied, the primary induction variable will be used
2704 // to feed a vector compare.
2705 if (Ind == Legal->getPrimaryInduction() && foldTailByMasking())
2706 continue;
2707
2708 // Returns true if \p Indvar is a pointer induction that is used directly by
2709 // load/store instruction \p I.
2710 auto IsDirectLoadStoreFromPtrIndvar = [&](Instruction *Indvar,
2711 Instruction *I) {
2712 return Induction.second.getKind() ==
2715 Indvar == getLoadStorePointerOperand(I) && IsScalarUse(I, Indvar);
2716 };
2717
2718 // Determine if all users of the induction variable are scalar after
2719 // vectorization.
2720 bool ScalarInd = all_of(Ind->users(), [&](User *U) -> bool {
2721 auto *I = cast<Instruction>(U);
2722 return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I) ||
2723 IsDirectLoadStoreFromPtrIndvar(Ind, I);
2724 });
2725 if (!ScalarInd)
2726 continue;
2727
2728 // If the induction variable update is a fixed-order recurrence, neither the
2729 // induction variable or its update should be marked scalar after
2730 // vectorization.
2731 auto *IndUpdatePhi = dyn_cast<PHINode>(IndUpdate);
2732 if (IndUpdatePhi && Legal->isFixedOrderRecurrence(IndUpdatePhi))
2733 continue;
2734
2735 // Determine if all users of the induction variable update instruction are
2736 // scalar after vectorization.
2737 bool ScalarIndUpdate = all_of(IndUpdate->users(), [&](User *U) -> bool {
2738 auto *I = cast<Instruction>(U);
2739 return I == Ind || !TheLoop->contains(I) || Worklist.count(I) ||
2740 IsDirectLoadStoreFromPtrIndvar(IndUpdate, I);
2741 });
2742 if (!ScalarIndUpdate)
2743 continue;
2744
2745 // The induction variable and its update instruction will remain scalar.
2746 Worklist.insert(Ind);
2747 Worklist.insert(IndUpdate);
2748 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Ind << "\n");
2749 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *IndUpdate
2750 << "\n");
2751 }
2752
2753 Scalars[VF].insert_range(Worklist);
2754}
2755
2757 Instruction *I, ElementCount VF) const {
2758 if (!isPredicatedInst(I))
2759 return false;
2760
2761 // Do we have a non-scalar lowering for this predicated
2762 // instruction? No - it is scalar with predication.
2763 switch(I->getOpcode()) {
2764 default:
2765 return true;
2766 case Instruction::Call:
2767 if (VF.isScalar())
2768 return true;
2770 case Instruction::Load:
2771 case Instruction::Store: {
2773 auto *Ty = getLoadStoreType(I);
2774 unsigned AS = getLoadStoreAddressSpace(I);
2775 Type *VTy = Ty;
2776 if (VF.isVector())
2777 VTy = VectorType::get(Ty, VF);
2778 const Align Alignment = getLoadStoreAlignment(I);
2779 return isa<LoadInst>(I) ? !(isLegalMaskedLoad(Ty, Ptr, Alignment, AS) ||
2780 TTI.isLegalMaskedGather(VTy, Alignment))
2781 : !(isLegalMaskedStore(Ty, Ptr, Alignment, AS) ||
2782 TTI.isLegalMaskedScatter(VTy, Alignment));
2783 }
2784 case Instruction::UDiv:
2785 case Instruction::SDiv:
2786 case Instruction::SRem:
2787 case Instruction::URem: {
2788 // We have the option to use the safe-divisor idiom to avoid predication.
2789 // The cost based decision here will always select safe-divisor for
2790 // scalable vectors as scalarization isn't legal.
2791 const auto [ScalarCost, SafeDivisorCost] = getDivRemSpeculationCost(I, VF);
2792 return isDivRemScalarWithPredication(ScalarCost, SafeDivisorCost);
2793 }
2794 }
2795}
2796
2797// TODO: Fold into LoopVectorizationLegality::isMaskRequired.
2799 // TODO: We can use the loop-preheader as context point here and get
2800 // context sensitive reasoning for isSafeToSpeculativelyExecute.
2802 (isa<LoadInst, StoreInst, CallInst>(I) && !Legal->isMaskRequired(I)) ||
2804 return false;
2805
2806 // If the instruction was executed conditionally in the original scalar loop,
2807 // predication is needed with a mask whose lanes are all possibly inactive.
2808 if (Legal->blockNeedsPredication(I->getParent()))
2809 return true;
2810
2811 // If we're not folding the tail by masking, predication is unnecessary.
2812 if (!foldTailByMasking())
2813 return false;
2814
2815 // All that remain are instructions with side-effects originally executed in
2816 // the loop unconditionally, but now execute under a tail-fold mask (only)
2817 // having at least one active lane (the first). If the side-effects of the
2818 // instruction are invariant, executing it w/o (the tail-folding) mask is safe
2819 // - it will cause the same side-effects as when masked.
2820 switch(I->getOpcode()) {
2821 default:
2823 "instruction should have been considered by earlier checks");
2824 case Instruction::Call:
2825 // Side-effects of a Call are assumed to be non-invariant, needing a
2826 // (fold-tail) mask.
2827 assert(Legal->isMaskRequired(I) &&
2828 "should have returned earlier for calls not needing a mask");
2829 return true;
2830 case Instruction::Load:
2831 // If the address is loop invariant no predication is needed.
2832 return !Legal->isInvariant(getLoadStorePointerOperand(I));
2833 case Instruction::Store: {
2834 // For stores, we need to prove both speculation safety (which follows from
2835 // the same argument as loads), but also must prove the value being stored
2836 // is correct. The easiest form of the later is to require that all values
2837 // stored are the same.
2838 return !(Legal->isInvariant(getLoadStorePointerOperand(I)) &&
2839 TheLoop->isLoopInvariant(cast<StoreInst>(I)->getValueOperand()));
2840 }
2841 case Instruction::UDiv:
2842 case Instruction::SDiv:
2843 case Instruction::SRem:
2844 case Instruction::URem:
2845 // If the divisor is loop-invariant no predication is needed.
2846 return !Legal->isInvariant(I->getOperand(1));
2847 }
2848}
2849
2850std::pair<InstructionCost, InstructionCost>
2852 ElementCount VF) const {
2853 assert(I->getOpcode() == Instruction::UDiv ||
2854 I->getOpcode() == Instruction::SDiv ||
2855 I->getOpcode() == Instruction::SRem ||
2856 I->getOpcode() == Instruction::URem);
2858
2859 // Scalarization isn't legal for scalable vector types
2860 InstructionCost ScalarizationCost = InstructionCost::getInvalid();
2861 if (!VF.isScalable()) {
2862 // Get the scalarization cost and scale this amount by the probability of
2863 // executing the predicated block. If the instruction is not predicated,
2864 // we fall through to the next case.
2865 ScalarizationCost = 0;
2866
2867 // These instructions have a non-void type, so account for the phi nodes
2868 // that we will create. This cost is likely to be zero. The phi node
2869 // cost, if any, should be scaled by the block probability because it
2870 // models a copy at the end of each predicated block.
2871 ScalarizationCost +=
2872 VF.getFixedValue() * TTI.getCFInstrCost(Instruction::PHI, CostKind);
2873
2874 // The cost of the non-predicated instruction.
2875 ScalarizationCost +=
2876 VF.getFixedValue() *
2877 TTI.getArithmeticInstrCost(I->getOpcode(), I->getType(), CostKind);
2878
2879 // The cost of insertelement and extractelement instructions needed for
2880 // scalarization.
2881 ScalarizationCost += getScalarizationOverhead(I, VF);
2882
2883 // Scale the cost by the probability of executing the predicated blocks.
2884 // This assumes the predicated block for each vector lane is equally
2885 // likely.
2886 ScalarizationCost = ScalarizationCost / getPredBlockCostDivisor(CostKind);
2887 }
2888
2889 InstructionCost SafeDivisorCost = 0;
2890 auto *VecTy = toVectorTy(I->getType(), VF);
2891 // The cost of the select guard to ensure all lanes are well defined
2892 // after we speculate above any internal control flow.
2893 SafeDivisorCost +=
2894 TTI.getCmpSelInstrCost(Instruction::Select, VecTy,
2895 toVectorTy(Type::getInt1Ty(I->getContext()), VF),
2897
2898 SmallVector<const Value *, 4> Operands(I->operand_values());
2899 SafeDivisorCost += TTI.getArithmeticInstrCost(
2900 I->getOpcode(), VecTy, CostKind,
2901 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
2902 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
2903 Operands, I);
2904 return {ScalarizationCost, SafeDivisorCost};
2905}
2906
2908 Instruction *I, ElementCount VF) const {
2909 assert(isAccessInterleaved(I) && "Expecting interleaved access.");
2911 "Decision should not be set yet.");
2912 auto *Group = getInterleavedAccessGroup(I);
2913 assert(Group && "Must have a group.");
2914 unsigned InterleaveFactor = Group->getFactor();
2915
2916 // If the instruction's allocated size doesn't equal its type size, it
2917 // requires padding and will be scalarized.
2918 auto &DL = I->getDataLayout();
2919 auto *ScalarTy = getLoadStoreType(I);
2920 if (hasIrregularType(ScalarTy, DL))
2921 return false;
2922
2923 // For scalable vectors, the interleave factors must be <= 8 since we require
2924 // the (de)interleaveN intrinsics instead of shufflevectors.
2925 if (VF.isScalable() && InterleaveFactor > 8)
2926 return false;
2927
2928 // If the group involves a non-integral pointer, we may not be able to
2929 // losslessly cast all values to a common type.
2930 bool ScalarNI = DL.isNonIntegralPointerType(ScalarTy);
2931 for (unsigned Idx = 0; Idx < InterleaveFactor; Idx++) {
2932 Instruction *Member = Group->getMember(Idx);
2933 if (!Member)
2934 continue;
2935 auto *MemberTy = getLoadStoreType(Member);
2936 bool MemberNI = DL.isNonIntegralPointerType(MemberTy);
2937 // Don't coerce non-integral pointers to integers or vice versa.
2938 if (MemberNI != ScalarNI)
2939 // TODO: Consider adding special nullptr value case here
2940 return false;
2941 if (MemberNI && ScalarNI &&
2942 ScalarTy->getPointerAddressSpace() !=
2943 MemberTy->getPointerAddressSpace())
2944 return false;
2945 }
2946
2947 // Check if masking is required.
2948 // A Group may need masking for one of two reasons: it resides in a block that
2949 // needs predication, or it was decided to use masking to deal with gaps
2950 // (either a gap at the end of a load-access that may result in a speculative
2951 // load, or any gaps in a store-access).
2952 bool PredicatedAccessRequiresMasking =
2953 blockNeedsPredicationForAnyReason(I->getParent()) &&
2954 Legal->isMaskRequired(I);
2955 bool LoadAccessWithGapsRequiresEpilogMasking =
2956 isa<LoadInst>(I) && Group->requiresScalarEpilogue() &&
2958 bool StoreAccessWithGapsRequiresMasking =
2959 isa<StoreInst>(I) && !Group->isFull();
2960 if (!PredicatedAccessRequiresMasking &&
2961 !LoadAccessWithGapsRequiresEpilogMasking &&
2962 !StoreAccessWithGapsRequiresMasking)
2963 return true;
2964
2965 // If masked interleaving is required, we expect that the user/target had
2966 // enabled it, because otherwise it either wouldn't have been created or
2967 // it should have been invalidated by the CostModel.
2969 "Masked interleave-groups for predicated accesses are not enabled.");
2970
2971 if (Group->isReverse())
2972 return false;
2973
2974 // TODO: Support interleaved access that requires a gap mask for scalable VFs.
2975 bool NeedsMaskForGaps = LoadAccessWithGapsRequiresEpilogMasking ||
2976 StoreAccessWithGapsRequiresMasking;
2977 if (VF.isScalable() && NeedsMaskForGaps)
2978 return false;
2979
2980 auto *Ty = getLoadStoreType(I);
2981 const Align Alignment = getLoadStoreAlignment(I);
2982 unsigned AS = getLoadStoreAddressSpace(I);
2983 return isa<LoadInst>(I) ? TTI.isLegalMaskedLoad(Ty, Alignment, AS)
2984 : TTI.isLegalMaskedStore(Ty, Alignment, AS);
2985}
2986
2988 Instruction *I, ElementCount VF) {
2989 // Get and ensure we have a valid memory instruction.
2990 assert((isa<LoadInst, StoreInst>(I)) && "Invalid memory instruction");
2991
2993 auto *ScalarTy = getLoadStoreType(I);
2994
2995 // In order to be widened, the pointer should be consecutive, first of all.
2996 if (!Legal->isConsecutivePtr(ScalarTy, Ptr))
2997 return false;
2998
2999 // If the instruction is a store located in a predicated block, it will be
3000 // scalarized.
3001 if (isScalarWithPredication(I, VF))
3002 return false;
3003
3004 // If the instruction's allocated size doesn't equal it's type size, it
3005 // requires padding and will be scalarized.
3006 auto &DL = I->getDataLayout();
3007 if (hasIrregularType(ScalarTy, DL))
3008 return false;
3009
3010 return true;
3011}
3012
3013void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
3014 // We should not collect Uniforms more than once per VF. Right now,
3015 // this function is called from collectUniformsAndScalars(), which
3016 // already does this check. Collecting Uniforms for VF=1 does not make any
3017 // sense.
3018
3019 assert(VF.isVector() && !Uniforms.contains(VF) &&
3020 "This function should not be visited twice for the same VF");
3021
3022 // Visit the list of Uniforms. If we find no uniform value, we won't
3023 // analyze again. Uniforms.count(VF) will return 1.
3024 Uniforms[VF].clear();
3025
3026 // Now we know that the loop is vectorizable!
3027 // Collect instructions inside the loop that will remain uniform after
3028 // vectorization.
3029
3030 // Global values, params and instructions outside of current loop are out of
3031 // scope.
3032 auto IsOutOfScope = [&](Value *V) -> bool {
3034 return (!I || !TheLoop->contains(I));
3035 };
3036
3037 // Worklist containing uniform instructions demanding lane 0.
3038 SetVector<Instruction *> Worklist;
3039
3040 // Add uniform instructions demanding lane 0 to the worklist. Instructions
3041 // that require predication must not be considered uniform after
3042 // vectorization, because that would create an erroneous replicating region
3043 // where only a single instance out of VF should be formed.
3044 auto AddToWorklistIfAllowed = [&](Instruction *I) -> void {
3045 if (IsOutOfScope(I)) {
3046 LLVM_DEBUG(dbgs() << "LV: Found not uniform due to scope: "
3047 << *I << "\n");
3048 return;
3049 }
3050 if (isPredicatedInst(I)) {
3051 LLVM_DEBUG(
3052 dbgs() << "LV: Found not uniform due to requiring predication: " << *I
3053 << "\n");
3054 return;
3055 }
3056 LLVM_DEBUG(dbgs() << "LV: Found uniform instruction: " << *I << "\n");
3057 Worklist.insert(I);
3058 };
3059
3060 // Start with the conditional branches exiting the loop. If the branch
3061 // condition is an instruction contained in the loop that is only used by the
3062 // branch, it is uniform. Note conditions from uncountable early exits are not
3063 // uniform.
3065 TheLoop->getExitingBlocks(Exiting);
3066 for (BasicBlock *E : Exiting) {
3067 if (Legal->hasUncountableEarlyExit() && TheLoop->getLoopLatch() != E)
3068 continue;
3069 auto *Cmp = dyn_cast<Instruction>(E->getTerminator()->getOperand(0));
3070 if (Cmp && TheLoop->contains(Cmp) && Cmp->hasOneUse())
3071 AddToWorklistIfAllowed(Cmp);
3072 }
3073
3074 auto PrevVF = VF.divideCoefficientBy(2);
3075 // Return true if all lanes perform the same memory operation, and we can
3076 // thus choose to execute only one.
3077 auto IsUniformMemOpUse = [&](Instruction *I) {
3078 // If the value was already known to not be uniform for the previous
3079 // (smaller VF), it cannot be uniform for the larger VF.
3080 if (PrevVF.isVector()) {
3081 auto Iter = Uniforms.find(PrevVF);
3082 if (Iter != Uniforms.end() && !Iter->second.contains(I))
3083 return false;
3084 }
3085 if (!Legal->isUniformMemOp(*I, VF))
3086 return false;
3087 if (isa<LoadInst>(I))
3088 // Loading the same address always produces the same result - at least
3089 // assuming aliasing and ordering which have already been checked.
3090 return true;
3091 // Storing the same value on every iteration.
3092 return TheLoop->isLoopInvariant(cast<StoreInst>(I)->getValueOperand());
3093 };
3094
3095 auto IsUniformDecision = [&](Instruction *I, ElementCount VF) {
3096 InstWidening WideningDecision = getWideningDecision(I, VF);
3097 assert(WideningDecision != CM_Unknown &&
3098 "Widening decision should be ready at this moment");
3099
3100 if (IsUniformMemOpUse(I))
3101 return true;
3102
3103 return (WideningDecision == CM_Widen ||
3104 WideningDecision == CM_Widen_Reverse ||
3105 WideningDecision == CM_Interleave);
3106 };
3107
3108 // Returns true if Ptr is the pointer operand of a memory access instruction
3109 // I, I is known to not require scalarization, and the pointer is not also
3110 // stored.
3111 auto IsVectorizedMemAccessUse = [&](Instruction *I, Value *Ptr) -> bool {
3112 if (isa<StoreInst>(I) && I->getOperand(0) == Ptr)
3113 return false;
3114 return getLoadStorePointerOperand(I) == Ptr &&
3115 (IsUniformDecision(I, VF) || Legal->isInvariant(Ptr));
3116 };
3117
3118 // Holds a list of values which are known to have at least one uniform use.
3119 // Note that there may be other uses which aren't uniform. A "uniform use"
3120 // here is something which only demands lane 0 of the unrolled iterations;
3121 // it does not imply that all lanes produce the same value (e.g. this is not
3122 // the usual meaning of uniform)
3123 SetVector<Value *> HasUniformUse;
3124
3125 // Scan the loop for instructions which are either a) known to have only
3126 // lane 0 demanded or b) are uses which demand only lane 0 of their operand.
3127 for (auto *BB : TheLoop->blocks())
3128 for (auto &I : *BB) {
3129 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(&I)) {
3130 switch (II->getIntrinsicID()) {
3131 case Intrinsic::sideeffect:
3132 case Intrinsic::experimental_noalias_scope_decl:
3133 case Intrinsic::assume:
3134 case Intrinsic::lifetime_start:
3135 case Intrinsic::lifetime_end:
3136 if (TheLoop->hasLoopInvariantOperands(&I))
3137 AddToWorklistIfAllowed(&I);
3138 break;
3139 default:
3140 break;
3141 }
3142 }
3143
3144 if (auto *EVI = dyn_cast<ExtractValueInst>(&I)) {
3145 if (IsOutOfScope(EVI->getAggregateOperand())) {
3146 AddToWorklistIfAllowed(EVI);
3147 continue;
3148 }
3149 // Only ExtractValue instructions where the aggregate value comes from a
3150 // call are allowed to be non-uniform.
3151 assert(isa<CallInst>(EVI->getAggregateOperand()) &&
3152 "Expected aggregate value to be call return value");
3153 }
3154
3155 // If there's no pointer operand, there's nothing to do.
3157 if (!Ptr)
3158 continue;
3159
3160 // If the pointer can be proven to be uniform, always add it to the
3161 // worklist.
3162 if (isa<Instruction>(Ptr) && Legal->isUniform(Ptr, VF))
3163 AddToWorklistIfAllowed(cast<Instruction>(Ptr));
3164
3165 if (IsUniformMemOpUse(&I))
3166 AddToWorklistIfAllowed(&I);
3167
3168 if (IsVectorizedMemAccessUse(&I, Ptr))
3169 HasUniformUse.insert(Ptr);
3170 }
3171
3172 // Add to the worklist any operands which have *only* uniform (e.g. lane 0
3173 // demanding) users. Since loops are assumed to be in LCSSA form, this
3174 // disallows uses outside the loop as well.
3175 for (auto *V : HasUniformUse) {
3176 if (IsOutOfScope(V))
3177 continue;
3178 auto *I = cast<Instruction>(V);
3179 bool UsersAreMemAccesses = all_of(I->users(), [&](User *U) -> bool {
3180 auto *UI = cast<Instruction>(U);
3181 return TheLoop->contains(UI) && IsVectorizedMemAccessUse(UI, V);
3182 });
3183 if (UsersAreMemAccesses)
3184 AddToWorklistIfAllowed(I);
3185 }
3186
3187 // Expand Worklist in topological order: whenever a new instruction
3188 // is added , its users should be already inside Worklist. It ensures
3189 // a uniform instruction will only be used by uniform instructions.
3190 unsigned Idx = 0;
3191 while (Idx != Worklist.size()) {
3192 Instruction *I = Worklist[Idx++];
3193
3194 for (auto *OV : I->operand_values()) {
3195 // isOutOfScope operands cannot be uniform instructions.
3196 if (IsOutOfScope(OV))
3197 continue;
3198 // First order recurrence Phi's should typically be considered
3199 // non-uniform.
3200 auto *OP = dyn_cast<PHINode>(OV);
3201 if (OP && Legal->isFixedOrderRecurrence(OP))
3202 continue;
3203 // If all the users of the operand are uniform, then add the
3204 // operand into the uniform worklist.
3205 auto *OI = cast<Instruction>(OV);
3206 if (llvm::all_of(OI->users(), [&](User *U) -> bool {
3207 auto *J = cast<Instruction>(U);
3208 return Worklist.count(J) || IsVectorizedMemAccessUse(J, OI);
3209 }))
3210 AddToWorklistIfAllowed(OI);
3211 }
3212 }
3213
3214 // For an instruction to be added into Worklist above, all its users inside
3215 // the loop should also be in Worklist. However, this condition cannot be
3216 // true for phi nodes that form a cyclic dependence. We must process phi
3217 // nodes separately. An induction variable will remain uniform if all users
3218 // of the induction variable and induction variable update remain uniform.
3219 // The code below handles both pointer and non-pointer induction variables.
3220 BasicBlock *Latch = TheLoop->getLoopLatch();
3221 for (const auto &Induction : Legal->getInductionVars()) {
3222 auto *Ind = Induction.first;
3223 auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch));
3224
3225 // Determine if all users of the induction variable are uniform after
3226 // vectorization.
3227 bool UniformInd = all_of(Ind->users(), [&](User *U) -> bool {
3228 auto *I = cast<Instruction>(U);
3229 return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I) ||
3230 IsVectorizedMemAccessUse(I, Ind);
3231 });
3232 if (!UniformInd)
3233 continue;
3234
3235 // Determine if all users of the induction variable update instruction are
3236 // uniform after vectorization.
3237 bool UniformIndUpdate = all_of(IndUpdate->users(), [&](User *U) -> bool {
3238 auto *I = cast<Instruction>(U);
3239 return I == Ind || Worklist.count(I) ||
3240 IsVectorizedMemAccessUse(I, IndUpdate);
3241 });
3242 if (!UniformIndUpdate)
3243 continue;
3244
3245 // The induction variable and its update instruction will remain uniform.
3246 AddToWorklistIfAllowed(Ind);
3247 AddToWorklistIfAllowed(IndUpdate);
3248 }
3249
3250 Uniforms[VF].insert_range(Worklist);
3251}
3252
3254 LLVM_DEBUG(dbgs() << "LV: Performing code size checks.\n");
3255
3256 if (Legal->getRuntimePointerChecking()->Need) {
3257 reportVectorizationFailure("Runtime ptr check is required with -Os/-Oz",
3258 "runtime pointer checks needed. Enable vectorization of this "
3259 "loop with '#pragma clang loop vectorize(enable)' when "
3260 "compiling with -Os/-Oz",
3261 "CantVersionLoopWithOptForSize", ORE, TheLoop);
3262 return true;
3263 }
3264
3265 if (!PSE.getPredicate().isAlwaysTrue()) {
3266 reportVectorizationFailure("Runtime SCEV check is required with -Os/-Oz",
3267 "runtime SCEV checks needed. Enable vectorization of this "
3268 "loop with '#pragma clang loop vectorize(enable)' when "
3269 "compiling with -Os/-Oz",
3270 "CantVersionLoopWithOptForSize", ORE, TheLoop);
3271 return true;
3272 }
3273
3274 // FIXME: Avoid specializing for stride==1 instead of bailing out.
3275 if (!Legal->getLAI()->getSymbolicStrides().empty()) {
3276 reportVectorizationFailure("Runtime stride check for small trip count",
3277 "runtime stride == 1 checks needed. Enable vectorization of "
3278 "this loop without such check by compiling with -Os/-Oz",
3279 "CantVersionLoopWithOptForSize", ORE, TheLoop);
3280 return true;
3281 }
3282
3283 return false;
3284}
3285
3286bool LoopVectorizationCostModel::isScalableVectorizationAllowed() {
3287 if (IsScalableVectorizationAllowed)
3288 return *IsScalableVectorizationAllowed;
3289
3290 IsScalableVectorizationAllowed = false;
3291 if (!TTI.supportsScalableVectors() && !ForceTargetSupportsScalableVectors)
3292 return false;
3293
3294 if (Hints->isScalableVectorizationDisabled()) {
3295 reportVectorizationInfo("Scalable vectorization is explicitly disabled",
3296 "ScalableVectorizationDisabled", ORE, TheLoop);
3297 return false;
3298 }
3299
3300 LLVM_DEBUG(dbgs() << "LV: Scalable vectorization is available\n");
3301
3302 auto MaxScalableVF = ElementCount::getScalable(
3303 std::numeric_limits<ElementCount::ScalarTy>::max());
3304
3305 // Test that the loop-vectorizer can legalize all operations for this MaxVF.
3306 // FIXME: While for scalable vectors this is currently sufficient, this should
3307 // be replaced by a more detailed mechanism that filters out specific VFs,
3308 // instead of invalidating vectorization for a whole set of VFs based on the
3309 // MaxVF.
3310
3311 // Disable scalable vectorization if the loop contains unsupported reductions.
3312 if (!canVectorizeReductions(MaxScalableVF)) {
3314 "Scalable vectorization not supported for the reduction "
3315 "operations found in this loop.",
3316 "ScalableVFUnfeasible", ORE, TheLoop);
3317 return false;
3318 }
3319
3320 // Disable scalable vectorization if the loop contains any instructions
3321 // with element types not supported for scalable vectors.
3322 if (any_of(ElementTypesInLoop, [&](Type *Ty) {
3323 return !Ty->isVoidTy() &&
3325 })) {
3326 reportVectorizationInfo("Scalable vectorization is not supported "
3327 "for all element types found in this loop.",
3328 "ScalableVFUnfeasible", ORE, TheLoop);
3329 return false;
3330 }
3331
3332 if (!Legal->isSafeForAnyVectorWidth() && !getMaxVScale(*TheFunction, TTI)) {
3333 reportVectorizationInfo("The target does not provide maximum vscale value "
3334 "for safe distance analysis.",
3335 "ScalableVFUnfeasible", ORE, TheLoop);
3336 return false;
3337 }
3338
3339 IsScalableVectorizationAllowed = true;
3340 return true;
3341}
3342
3343ElementCount
3344LoopVectorizationCostModel::getMaxLegalScalableVF(unsigned MaxSafeElements) {
3345 if (!isScalableVectorizationAllowed())
3346 return ElementCount::getScalable(0);
3347
3348 auto MaxScalableVF = ElementCount::getScalable(
3349 std::numeric_limits<ElementCount::ScalarTy>::max());
3350 if (Legal->isSafeForAnyVectorWidth())
3351 return MaxScalableVF;
3352
3353 std::optional<unsigned> MaxVScale = getMaxVScale(*TheFunction, TTI);
3354 // Limit MaxScalableVF by the maximum safe dependence distance.
3355 MaxScalableVF = ElementCount::getScalable(MaxSafeElements / *MaxVScale);
3356
3357 if (!MaxScalableVF)
3359 "Max legal vector width too small, scalable vectorization "
3360 "unfeasible.",
3361 "ScalableVFUnfeasible", ORE, TheLoop);
3362
3363 return MaxScalableVF;
3364}
3365
3366FixedScalableVFPair LoopVectorizationCostModel::computeFeasibleMaxVF(
3367 unsigned MaxTripCount, ElementCount UserVF, bool FoldTailByMasking) {
3368 MinBWs = computeMinimumValueSizes(TheLoop->getBlocks(), *DB, &TTI);
3369 unsigned SmallestType, WidestType;
3370 std::tie(SmallestType, WidestType) = getSmallestAndWidestTypes();
3371
3372 // Get the maximum safe dependence distance in bits computed by LAA.
3373 // It is computed by MaxVF * sizeOf(type) * 8, where type is taken from
3374 // the memory accesses that is most restrictive (involved in the smallest
3375 // dependence distance).
3376 unsigned MaxSafeElementsPowerOf2 =
3377 bit_floor(Legal->getMaxSafeVectorWidthInBits() / WidestType);
3378 if (!Legal->isSafeForAnyStoreLoadForwardDistances()) {
3379 unsigned SLDist = Legal->getMaxStoreLoadForwardSafeDistanceInBits();
3380 MaxSafeElementsPowerOf2 =
3381 std::min(MaxSafeElementsPowerOf2, SLDist / WidestType);
3382 }
3383 auto MaxSafeFixedVF = ElementCount::getFixed(MaxSafeElementsPowerOf2);
3384 auto MaxSafeScalableVF = getMaxLegalScalableVF(MaxSafeElementsPowerOf2);
3385
3386 if (!Legal->isSafeForAnyVectorWidth())
3387 this->MaxSafeElements = MaxSafeElementsPowerOf2;
3388
3389 LLVM_DEBUG(dbgs() << "LV: The max safe fixed VF is: " << MaxSafeFixedVF
3390 << ".\n");
3391 LLVM_DEBUG(dbgs() << "LV: The max safe scalable VF is: " << MaxSafeScalableVF
3392 << ".\n");
3393
3394 // First analyze the UserVF, fall back if the UserVF should be ignored.
3395 if (UserVF) {
3396 auto MaxSafeUserVF =
3397 UserVF.isScalable() ? MaxSafeScalableVF : MaxSafeFixedVF;
3398
3399 if (ElementCount::isKnownLE(UserVF, MaxSafeUserVF)) {
3400 // If `VF=vscale x N` is safe, then so is `VF=N`
3401 if (UserVF.isScalable())
3402 return FixedScalableVFPair(
3403 ElementCount::getFixed(UserVF.getKnownMinValue()), UserVF);
3404
3405 return UserVF;
3406 }
3407
3408 assert(ElementCount::isKnownGT(UserVF, MaxSafeUserVF));
3409
3410 // Only clamp if the UserVF is not scalable. If the UserVF is scalable, it
3411 // is better to ignore the hint and let the compiler choose a suitable VF.
3412 if (!UserVF.isScalable()) {
3413 LLVM_DEBUG(dbgs() << "LV: User VF=" << UserVF
3414 << " is unsafe, clamping to max safe VF="
3415 << MaxSafeFixedVF << ".\n");
3416 ORE->emit([&]() {
3417 return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationFactor",
3418 TheLoop->getStartLoc(),
3419 TheLoop->getHeader())
3420 << "User-specified vectorization factor "
3421 << ore::NV("UserVectorizationFactor", UserVF)
3422 << " is unsafe, clamping to maximum safe vectorization factor "
3423 << ore::NV("VectorizationFactor", MaxSafeFixedVF);
3424 });
3425 return MaxSafeFixedVF;
3426 }
3427
3429 LLVM_DEBUG(dbgs() << "LV: User VF=" << UserVF
3430 << " is ignored because scalable vectors are not "
3431 "available.\n");
3432 ORE->emit([&]() {
3433 return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationFactor",
3434 TheLoop->getStartLoc(),
3435 TheLoop->getHeader())
3436 << "User-specified vectorization factor "
3437 << ore::NV("UserVectorizationFactor", UserVF)
3438 << " is ignored because the target does not support scalable "
3439 "vectors. The compiler will pick a more suitable value.";
3440 });
3441 } else {
3442 LLVM_DEBUG(dbgs() << "LV: User VF=" << UserVF
3443 << " is unsafe. Ignoring scalable UserVF.\n");
3444 ORE->emit([&]() {
3445 return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationFactor",
3446 TheLoop->getStartLoc(),
3447 TheLoop->getHeader())
3448 << "User-specified vectorization factor "
3449 << ore::NV("UserVectorizationFactor", UserVF)
3450 << " is unsafe. Ignoring the hint to let the compiler pick a "
3451 "more suitable value.";
3452 });
3453 }
3454 }
3455
3456 LLVM_DEBUG(dbgs() << "LV: The Smallest and Widest types: " << SmallestType
3457 << " / " << WidestType << " bits.\n");
3458
3459 FixedScalableVFPair Result(ElementCount::getFixed(1),
3461 if (auto MaxVF =
3462 getMaximizedVFForTarget(MaxTripCount, SmallestType, WidestType,
3463 MaxSafeFixedVF, FoldTailByMasking))
3464 Result.FixedVF = MaxVF;
3465
3466 if (auto MaxVF =
3467 getMaximizedVFForTarget(MaxTripCount, SmallestType, WidestType,
3468 MaxSafeScalableVF, FoldTailByMasking))
3469 if (MaxVF.isScalable()) {
3470 Result.ScalableVF = MaxVF;
3471 LLVM_DEBUG(dbgs() << "LV: Found feasible scalable VF = " << MaxVF
3472 << "\n");
3473 }
3474
3475 return Result;
3476}
3477
3478FixedScalableVFPair
3480 if (Legal->getRuntimePointerChecking()->Need && TTI.hasBranchDivergence()) {
3481 // TODO: It may be useful to do since it's still likely to be dynamically
3482 // uniform if the target can skip.
3484 "Not inserting runtime ptr check for divergent target",
3485 "runtime pointer checks needed. Not enabled for divergent target",
3486 "CantVersionLoopWithDivergentTarget", ORE, TheLoop);
3488 }
3489
3490 ScalarEvolution *SE = PSE.getSE();
3492 unsigned MaxTC = PSE.getSmallConstantMaxTripCount();
3493 LLVM_DEBUG(dbgs() << "LV: Found trip count: " << TC << '\n');
3494 if (TC != ElementCount::getFixed(MaxTC))
3495 LLVM_DEBUG(dbgs() << "LV: Found maximum trip count: " << MaxTC << '\n');
3496 if (TC.isScalar()) {
3497 reportVectorizationFailure("Single iteration (non) loop",
3498 "loop trip count is one, irrelevant for vectorization",
3499 "SingleIterationLoop", ORE, TheLoop);
3501 }
3502
3503 // If BTC matches the widest induction type and is -1 then the trip count
3504 // computation will wrap to 0 and the vector trip count will be 0. Do not try
3505 // to vectorize.
3506 const SCEV *BTC = SE->getBackedgeTakenCount(TheLoop);
3507 if (!isa<SCEVCouldNotCompute>(BTC) &&
3508 BTC->getType()->getScalarSizeInBits() >=
3509 Legal->getWidestInductionType()->getScalarSizeInBits() &&
3511 SE->getMinusOne(BTC->getType()))) {
3513 "Trip count computation wrapped",
3514 "backedge-taken count is -1, loop trip count wrapped to 0",
3515 "TripCountWrapped", ORE, TheLoop);
3517 }
3518
3519 switch (ScalarEpilogueStatus) {
3521 return computeFeasibleMaxVF(MaxTC, UserVF, false);
3523 [[fallthrough]];
3525 LLVM_DEBUG(
3526 dbgs() << "LV: vector predicate hint/switch found.\n"
3527 << "LV: Not allowing scalar epilogue, creating predicated "
3528 << "vector loop.\n");
3529 break;
3531 // fallthrough as a special case of OptForSize
3533 if (ScalarEpilogueStatus == CM_ScalarEpilogueNotAllowedOptSize)
3534 LLVM_DEBUG(
3535 dbgs() << "LV: Not allowing scalar epilogue due to -Os/-Oz.\n");
3536 else
3537 LLVM_DEBUG(dbgs() << "LV: Not allowing scalar epilogue due to low trip "
3538 << "count.\n");
3539
3540 // Bail if runtime checks are required, which are not good when optimising
3541 // for size.
3544
3545 break;
3546 }
3547
3548 // Now try the tail folding
3549
3550 // Invalidate interleave groups that require an epilogue if we can't mask
3551 // the interleave-group.
3553 assert(WideningDecisions.empty() && Uniforms.empty() && Scalars.empty() &&
3554 "No decisions should have been taken at this point");
3555 // Note: There is no need to invalidate any cost modeling decisions here, as
3556 // none were taken so far.
3557 InterleaveInfo.invalidateGroupsRequiringScalarEpilogue();
3558 }
3559
3560 FixedScalableVFPair MaxFactors = computeFeasibleMaxVF(MaxTC, UserVF, true);
3561
3562 // Avoid tail folding if the trip count is known to be a multiple of any VF
3563 // we choose.
3564 std::optional<unsigned> MaxPowerOf2RuntimeVF =
3565 MaxFactors.FixedVF.getFixedValue();
3566 if (MaxFactors.ScalableVF) {
3567 std::optional<unsigned> MaxVScale = getMaxVScale(*TheFunction, TTI);
3568 if (MaxVScale && TTI.isVScaleKnownToBeAPowerOfTwo()) {
3569 MaxPowerOf2RuntimeVF = std::max<unsigned>(
3570 *MaxPowerOf2RuntimeVF,
3571 *MaxVScale * MaxFactors.ScalableVF.getKnownMinValue());
3572 } else
3573 MaxPowerOf2RuntimeVF = std::nullopt; // Stick with tail-folding for now.
3574 }
3575
3576 auto NoScalarEpilogueNeeded = [this, &UserIC](unsigned MaxVF) {
3577 // Return false if the loop is neither a single-latch-exit loop nor an
3578 // early-exit loop as tail-folding is not supported in that case.
3579 if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch() &&
3580 !Legal->hasUncountableEarlyExit())
3581 return false;
3582 unsigned MaxVFtimesIC = UserIC ? MaxVF * UserIC : MaxVF;
3583 ScalarEvolution *SE = PSE.getSE();
3584 // Calling getSymbolicMaxBackedgeTakenCount enables support for loops
3585 // with uncountable exits. For countable loops, the symbolic maximum must
3586 // remain identical to the known back-edge taken count.
3587 const SCEV *BackedgeTakenCount = PSE.getSymbolicMaxBackedgeTakenCount();
3588 assert((Legal->hasUncountableEarlyExit() ||
3589 BackedgeTakenCount == PSE.getBackedgeTakenCount()) &&
3590 "Invalid loop count");
3591 const SCEV *ExitCount = SE->getAddExpr(
3592 BackedgeTakenCount, SE->getOne(BackedgeTakenCount->getType()));
3593 const SCEV *Rem = SE->getURemExpr(
3594 SE->applyLoopGuards(ExitCount, TheLoop),
3595 SE->getConstant(BackedgeTakenCount->getType(), MaxVFtimesIC));
3596 return Rem->isZero();
3597 };
3598
3599 if (MaxPowerOf2RuntimeVF > 0u) {
3600 assert((UserVF.isNonZero() || isPowerOf2_32(*MaxPowerOf2RuntimeVF)) &&
3601 "MaxFixedVF must be a power of 2");
3602 if (NoScalarEpilogueNeeded(*MaxPowerOf2RuntimeVF)) {
3603 // Accept MaxFixedVF if we do not have a tail.
3604 LLVM_DEBUG(dbgs() << "LV: No tail will remain for any chosen VF.\n");
3605 return MaxFactors;
3606 }
3607 }
3608
3609 auto ExpectedTC = getSmallBestKnownTC(PSE, TheLoop);
3610 if (ExpectedTC && ExpectedTC->isFixed() &&
3611 ExpectedTC->getFixedValue() <=
3612 TTI.getMinTripCountTailFoldingThreshold()) {
3613 if (MaxPowerOf2RuntimeVF > 0u) {
3614 // If we have a low-trip-count, and the fixed-width VF is known to divide
3615 // the trip count but the scalable factor does not, use the fixed-width
3616 // factor in preference to allow the generation of a non-predicated loop.
3617 if (ScalarEpilogueStatus == CM_ScalarEpilogueNotAllowedLowTripLoop &&
3618 NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue())) {
3619 LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width so that no tail will "
3620 "remain for any chosen VF.\n");
3621 MaxFactors.ScalableVF = ElementCount::getScalable(0);
3622 return MaxFactors;
3623 }
3624 }
3625
3627 "The trip count is below the minial threshold value.",
3628 "loop trip count is too low, avoiding vectorization", "LowTripCount",
3629 ORE, TheLoop);
3631 }
3632
3633 // If we don't know the precise trip count, or if the trip count that we
3634 // found modulo the vectorization factor is not zero, try to fold the tail
3635 // by masking.
3636 // FIXME: look for a smaller MaxVF that does divide TC rather than masking.
3637 bool ContainsScalableVF = MaxFactors.ScalableVF.isNonZero();
3638 setTailFoldingStyles(ContainsScalableVF, UserIC);
3639 if (foldTailByMasking()) {
3641 LLVM_DEBUG(
3642 dbgs()
3643 << "LV: tail is folded with EVL, forcing unroll factor to be 1. Will "
3644 "try to generate VP Intrinsics with scalable vector "
3645 "factors only.\n");
3646 // Tail folded loop using VP intrinsics restricts the VF to be scalable
3647 // for now.
3648 // TODO: extend it for fixed vectors, if required.
3649 assert(ContainsScalableVF && "Expected scalable vector factor.");
3650
3651 MaxFactors.FixedVF = ElementCount::getFixed(1);
3652 }
3653 return MaxFactors;
3654 }
3655
3656 // If there was a tail-folding hint/switch, but we can't fold the tail by
3657 // masking, fallback to a vectorization with a scalar epilogue.
3658 if (ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUsePredicate) {
3659 LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking: vectorize with a "
3660 "scalar epilogue instead.\n");
3661 ScalarEpilogueStatus = CM_ScalarEpilogueAllowed;
3662 return MaxFactors;
3663 }
3664
3665 if (ScalarEpilogueStatus == CM_ScalarEpilogueNotAllowedUsePredicate) {
3666 LLVM_DEBUG(dbgs() << "LV: Can't fold tail by masking: don't vectorize\n");
3668 }
3669
3670 if (TC.isZero()) {
3672 "unable to calculate the loop count due to complex control flow",
3673 "UnknownLoopCountComplexCFG", ORE, TheLoop);
3675 }
3676
3678 "Cannot optimize for size and vectorize at the same time.",
3679 "cannot optimize for size and vectorize at the same time. "
3680 "Enable vectorization of this loop with '#pragma clang loop "
3681 "vectorize(enable)' when compiling with -Os/-Oz",
3682 "NoTailLoopWithOptForSize", ORE, TheLoop);
3684}
3685
3687 ElementCount VF) {
3688 if (ConsiderRegPressure.getNumOccurrences())
3689 return ConsiderRegPressure;
3690
3691 // TODO: We should eventually consider register pressure for all targets. The
3692 // TTI hook is temporary whilst target-specific issues are being fixed.
3693 if (TTI.shouldConsiderVectorizationRegPressure())
3694 return true;
3695
3696 if (!useMaxBandwidth(VF.isScalable()
3699 return false;
3700 // Only calculate register pressure for VFs enabled by MaxBandwidth.
3702 VF, VF.isScalable() ? MaxPermissibleVFWithoutMaxBW.ScalableVF
3704}
3705
3708 return MaximizeBandwidth || (MaximizeBandwidth.getNumOccurrences() == 0 &&
3709 (TTI.shouldMaximizeVectorBandwidth(RegKind) ||
3711 Legal->hasVectorCallVariants())));
3712}
3713
3714ElementCount LoopVectorizationCostModel::clampVFByMaxTripCount(
3715 ElementCount VF, unsigned MaxTripCount, bool FoldTailByMasking) const {
3716 unsigned EstimatedVF = VF.getKnownMinValue();
3717 if (VF.isScalable() && TheFunction->hasFnAttribute(Attribute::VScaleRange)) {
3718 auto Attr = TheFunction->getFnAttribute(Attribute::VScaleRange);
3719 auto Min = Attr.getVScaleRangeMin();
3720 EstimatedVF *= Min;
3721 }
3722
3723 // When a scalar epilogue is required, at least one iteration of the scalar
3724 // loop has to execute. Adjust MaxTripCount accordingly to avoid picking a
3725 // max VF that results in a dead vector loop.
3726 if (MaxTripCount > 0 && requiresScalarEpilogue(true))
3727 MaxTripCount -= 1;
3728
3729 if (MaxTripCount && MaxTripCount <= EstimatedVF &&
3730 (!FoldTailByMasking || isPowerOf2_32(MaxTripCount))) {
3731 // If upper bound loop trip count (TC) is known at compile time there is no
3732 // point in choosing VF greater than TC (as done in the loop below). Select
3733 // maximum power of two which doesn't exceed TC. If VF is
3734 // scalable, we only fall back on a fixed VF when the TC is less than or
3735 // equal to the known number of lanes.
3736 auto ClampedUpperTripCount = llvm::bit_floor(MaxTripCount);
3737 LLVM_DEBUG(dbgs() << "LV: Clamping the MaxVF to maximum power of two not "
3738 "exceeding the constant trip count: "
3739 << ClampedUpperTripCount << "\n");
3740 return ElementCount::get(ClampedUpperTripCount,
3741 FoldTailByMasking ? VF.isScalable() : false);
3742 }
3743 return VF;
3744}
3745
3746ElementCount LoopVectorizationCostModel::getMaximizedVFForTarget(
3747 unsigned MaxTripCount, unsigned SmallestType, unsigned WidestType,
3748 ElementCount MaxSafeVF, bool FoldTailByMasking) {
3749 bool ComputeScalableMaxVF = MaxSafeVF.isScalable();
3750 const TypeSize WidestRegister = TTI.getRegisterBitWidth(
3751 ComputeScalableMaxVF ? TargetTransformInfo::RGK_ScalableVector
3753
3754 // Convenience function to return the minimum of two ElementCounts.
3755 auto MinVF = [](const ElementCount &LHS, const ElementCount &RHS) {
3756 assert((LHS.isScalable() == RHS.isScalable()) &&
3757 "Scalable flags must match");
3758 return ElementCount::isKnownLT(LHS, RHS) ? LHS : RHS;
3759 };
3760
3761 // Ensure MaxVF is a power of 2; the dependence distance bound may not be.
3762 // Note that both WidestRegister and WidestType may not be a powers of 2.
3763 auto MaxVectorElementCount = ElementCount::get(
3764 llvm::bit_floor(WidestRegister.getKnownMinValue() / WidestType),
3765 ComputeScalableMaxVF);
3766 MaxVectorElementCount = MinVF(MaxVectorElementCount, MaxSafeVF);
3767 LLVM_DEBUG(dbgs() << "LV: The Widest register safe to use is: "
3768 << (MaxVectorElementCount * WidestType) << " bits.\n");
3769
3770 if (!MaxVectorElementCount) {
3771 LLVM_DEBUG(dbgs() << "LV: The target has no "
3772 << (ComputeScalableMaxVF ? "scalable" : "fixed")
3773 << " vector registers.\n");
3774 return ElementCount::getFixed(1);
3775 }
3776
3777 ElementCount MaxVF = clampVFByMaxTripCount(MaxVectorElementCount,
3778 MaxTripCount, FoldTailByMasking);
3779 // If the MaxVF was already clamped, there's no point in trying to pick a
3780 // larger one.
3781 if (MaxVF != MaxVectorElementCount)
3782 return MaxVF;
3783
3785 ComputeScalableMaxVF ? TargetTransformInfo::RGK_ScalableVector
3787
3788 if (MaxVF.isScalable())
3789 MaxPermissibleVFWithoutMaxBW.ScalableVF = MaxVF;
3790 else
3791 MaxPermissibleVFWithoutMaxBW.FixedVF = MaxVF;
3792
3793 if (useMaxBandwidth(RegKind)) {
3794 auto MaxVectorElementCountMaxBW = ElementCount::get(
3795 llvm::bit_floor(WidestRegister.getKnownMinValue() / SmallestType),
3796 ComputeScalableMaxVF);
3797 MaxVF = MinVF(MaxVectorElementCountMaxBW, MaxSafeVF);
3798
3799 if (ElementCount MinVF =
3800 TTI.getMinimumVF(SmallestType, ComputeScalableMaxVF)) {
3801 if (ElementCount::isKnownLT(MaxVF, MinVF)) {
3802 LLVM_DEBUG(dbgs() << "LV: Overriding calculated MaxVF(" << MaxVF
3803 << ") with target's minimum: " << MinVF << '\n');
3804 MaxVF = MinVF;
3805 }
3806 }
3807
3808 MaxVF = clampVFByMaxTripCount(MaxVF, MaxTripCount, FoldTailByMasking);
3809
3810 if (MaxVectorElementCount != MaxVF) {
3811 // Invalidate any widening decisions we might have made, in case the loop
3812 // requires prediction (decided later), but we have already made some
3813 // load/store widening decisions.
3814 invalidateCostModelingDecisions();
3815 }
3816 }
3817 return MaxVF;
3818}
3819
3820bool LoopVectorizationPlanner::isMoreProfitable(const VectorizationFactor &A,
3821 const VectorizationFactor &B,
3822 const unsigned MaxTripCount,
3823 bool HasTail,
3824 bool IsEpilogue) const {
3825 InstructionCost CostA = A.Cost;
3826 InstructionCost CostB = B.Cost;
3827
3828 // Improve estimate for the vector width if it is scalable.
3829 unsigned EstimatedWidthA = A.Width.getKnownMinValue();
3830 unsigned EstimatedWidthB = B.Width.getKnownMinValue();
3831 if (std::optional<unsigned> VScale = CM.getVScaleForTuning()) {
3832 if (A.Width.isScalable())
3833 EstimatedWidthA *= *VScale;
3834 if (B.Width.isScalable())
3835 EstimatedWidthB *= *VScale;
3836 }
3837
3838 // When optimizing for size choose whichever is smallest, which will be the
3839 // one with the smallest cost for the whole loop. On a tie pick the larger
3840 // vector width, on the assumption that throughput will be greater.
3841 if (CM.CostKind == TTI::TCK_CodeSize)
3842 return CostA < CostB ||
3843 (CostA == CostB && EstimatedWidthA > EstimatedWidthB);
3844
3845 // Assume vscale may be larger than 1 (or the value being tuned for),
3846 // so that scalable vectorization is slightly favorable over fixed-width
3847 // vectorization.
3848 bool PreferScalable = !TTI.preferFixedOverScalableIfEqualCost(IsEpilogue) &&
3849 A.Width.isScalable() && !B.Width.isScalable();
3850
3851 auto CmpFn = [PreferScalable](const InstructionCost &LHS,
3852 const InstructionCost &RHS) {
3853 return PreferScalable ? LHS <= RHS : LHS < RHS;
3854 };
3855
3856 // To avoid the need for FP division:
3857 // (CostA / EstimatedWidthA) < (CostB / EstimatedWidthB)
3858 // <=> (CostA * EstimatedWidthB) < (CostB * EstimatedWidthA)
3859 if (!MaxTripCount)
3860 return CmpFn(CostA * EstimatedWidthB, CostB * EstimatedWidthA);
3861
3862 auto GetCostForTC = [MaxTripCount, HasTail](unsigned VF,
3863 InstructionCost VectorCost,
3864 InstructionCost ScalarCost) {
3865 // If the trip count is a known (possibly small) constant, the trip count
3866 // will be rounded up to an integer number of iterations under
3867 // FoldTailByMasking. The total cost in that case will be
3868 // VecCost*ceil(TripCount/VF). When not folding the tail, the total
3869 // cost will be VecCost*floor(TC/VF) + ScalarCost*(TC%VF). There will be
3870 // some extra overheads, but for the purpose of comparing the costs of
3871 // different VFs we can use this to compare the total loop-body cost
3872 // expected after vectorization.
3873 if (HasTail)
3874 return VectorCost * (MaxTripCount / VF) +
3875 ScalarCost * (MaxTripCount % VF);
3876 return VectorCost * divideCeil(MaxTripCount, VF);
3877 };
3878
3879 auto RTCostA = GetCostForTC(EstimatedWidthA, CostA, A.ScalarCost);
3880 auto RTCostB = GetCostForTC(EstimatedWidthB, CostB, B.ScalarCost);
3881 return CmpFn(RTCostA, RTCostB);
3882}
3883
3884bool LoopVectorizationPlanner::isMoreProfitable(const VectorizationFactor &A,
3885 const VectorizationFactor &B,
3886 bool HasTail,
3887 bool IsEpilogue) const {
3888 const unsigned MaxTripCount = PSE.getSmallConstantMaxTripCount();
3889 return LoopVectorizationPlanner::isMoreProfitable(A, B, MaxTripCount, HasTail,
3890 IsEpilogue);
3891}
3892
3895 using RecipeVFPair = std::pair<VPRecipeBase *, ElementCount>;
3896 SmallVector<RecipeVFPair> InvalidCosts;
3897 for (const auto &Plan : VPlans) {
3898 for (ElementCount VF : Plan->vectorFactors()) {
3899 // The VPlan-based cost model is designed for computing vector cost.
3900 // Querying VPlan-based cost model with a scarlar VF will cause some
3901 // errors because we expect the VF is vector for most of the widen
3902 // recipes.
3903 if (VF.isScalar())
3904 continue;
3905
3906 VPCostContext CostCtx(CM.TTI, *CM.TLI, *Plan, CM, CM.CostKind);
3907 precomputeCosts(*Plan, VF, CostCtx);
3908 auto Iter = vp_depth_first_deep(Plan->getVectorLoopRegion()->getEntry());
3910 for (auto &R : *VPBB) {
3911 if (!R.cost(VF, CostCtx).isValid())
3912 InvalidCosts.emplace_back(&R, VF);
3913 }
3914 }
3915 }
3916 }
3917 if (InvalidCosts.empty())
3918 return;
3919
3920 // Emit a report of VFs with invalid costs in the loop.
3921
3922 // Group the remarks per recipe, keeping the recipe order from InvalidCosts.
3924 unsigned I = 0;
3925 for (auto &Pair : InvalidCosts)
3926 if (Numbering.try_emplace(Pair.first, I).second)
3927 ++I;
3928
3929 // Sort the list, first on recipe(number) then on VF.
3930 sort(InvalidCosts, [&Numbering](RecipeVFPair &A, RecipeVFPair &B) {
3931 unsigned NA = Numbering[A.first];
3932 unsigned NB = Numbering[B.first];
3933 if (NA != NB)
3934 return NA < NB;
3935 return ElementCount::isKnownLT(A.second, B.second);
3936 });
3937
3938 // For a list of ordered recipe-VF pairs:
3939 // [(load, VF1), (load, VF2), (store, VF1)]
3940 // group the recipes together to emit separate remarks for:
3941 // load (VF1, VF2)
3942 // store (VF1)
3943 auto Tail = ArrayRef<RecipeVFPair>(InvalidCosts);
3944 auto Subset = ArrayRef<RecipeVFPair>();
3945 do {
3946 if (Subset.empty())
3947 Subset = Tail.take_front(1);
3948
3949 VPRecipeBase *R = Subset.front().first;
3950
3951 unsigned Opcode =
3954 [](const auto *R) { return Instruction::PHI; })
3955 .Case<VPWidenSelectRecipe>(
3956 [](const auto *R) { return Instruction::Select; })
3957 .Case<VPWidenStoreRecipe>(
3958 [](const auto *R) { return Instruction::Store; })
3959 .Case<VPWidenLoadRecipe>(
3960 [](const auto *R) { return Instruction::Load; })
3961 .Case<VPWidenCallRecipe, VPWidenIntrinsicRecipe>(
3962 [](const auto *R) { return Instruction::Call; })
3965 [](const auto *R) { return R->getOpcode(); })
3966 .Case<VPInterleaveRecipe>([](const VPInterleaveRecipe *R) {
3967 return R->getStoredValues().empty() ? Instruction::Load
3968 : Instruction::Store;
3969 });
3970
3971 // If the next recipe is different, or if there are no other pairs,
3972 // emit a remark for the collated subset. e.g.
3973 // [(load, VF1), (load, VF2))]
3974 // to emit:
3975 // remark: invalid costs for 'load' at VF=(VF1, VF2)
3976 if (Subset == Tail || Tail[Subset.size()].first != R) {
3977 std::string OutString;
3978 raw_string_ostream OS(OutString);
3979 assert(!Subset.empty() && "Unexpected empty range");
3980 OS << "Recipe with invalid costs prevented vectorization at VF=(";
3981 for (const auto &Pair : Subset)
3982 OS << (Pair.second == Subset.front().second ? "" : ", ") << Pair.second;
3983 OS << "):";
3984 if (Opcode == Instruction::Call) {
3985 StringRef Name = "";
3986 if (auto *Int = dyn_cast<VPWidenIntrinsicRecipe>(R)) {
3987 Name = Int->getIntrinsicName();
3988 } else {
3989 auto *WidenCall = dyn_cast<VPWidenCallRecipe>(R);
3990 Function *CalledFn =
3991 WidenCall ? WidenCall->getCalledScalarFunction()
3992 : cast<Function>(R->getOperand(R->getNumOperands() - 1)
3993 ->getLiveInIRValue());
3994 Name = CalledFn->getName();
3995 }
3996 OS << " call to " << Name;
3997 } else
3998 OS << " " << Instruction::getOpcodeName(Opcode);
3999 reportVectorizationInfo(OutString, "InvalidCost", ORE, OrigLoop, nullptr,
4000 R->getDebugLoc());
4001 Tail = Tail.drop_front(Subset.size());
4002 Subset = {};
4003 } else
4004 // Grow the subset by one element
4005 Subset = Tail.take_front(Subset.size() + 1);
4006 } while (!Tail.empty());
4007}
4008
4009/// Check if any recipe of \p Plan will generate a vector value, which will be
4010/// assigned a vector register.
4012 const TargetTransformInfo &TTI) {
4013 assert(VF.isVector() && "Checking a scalar VF?");
4014 VPTypeAnalysis TypeInfo(Plan);
4015 DenseSet<VPRecipeBase *> EphemeralRecipes;
4016 collectEphemeralRecipesForVPlan(Plan, EphemeralRecipes);
4017 // Set of already visited types.
4018 DenseSet<Type *> Visited;
4021 for (VPRecipeBase &R : *VPBB) {
4022 if (EphemeralRecipes.contains(&R))
4023 continue;
4024 // Continue early if the recipe is considered to not produce a vector
4025 // result. Note that this includes VPInstruction where some opcodes may
4026 // produce a vector, to preserve existing behavior as VPInstructions model
4027 // aspects not directly mapped to existing IR instructions.
4028 switch (R.getVPDefID()) {
4029 case VPDef::VPDerivedIVSC:
4030 case VPDef::VPScalarIVStepsSC:
4031 case VPDef::VPReplicateSC:
4032 case VPDef::VPInstructionSC:
4033 case VPDef::VPCanonicalIVPHISC:
4034 case VPDef::VPVectorPointerSC:
4035 case VPDef::VPVectorEndPointerSC:
4036 case VPDef::VPExpandSCEVSC:
4037 case VPDef::VPEVLBasedIVPHISC:
4038 case VPDef::VPPredInstPHISC:
4039 case VPDef::VPBranchOnMaskSC:
4040 continue;
4041 case VPDef::VPReductionSC:
4042 case VPDef::VPActiveLaneMaskPHISC:
4043 case VPDef::VPWidenCallSC:
4044 case VPDef::VPWidenCanonicalIVSC:
4045 case VPDef::VPWidenCastSC:
4046 case VPDef::VPWidenGEPSC:
4047 case VPDef::VPWidenIntrinsicSC:
4048 case VPDef::VPWidenSC:
4049 case VPDef::VPWidenSelectSC:
4050 case VPDef::VPBlendSC:
4051 case VPDef::VPFirstOrderRecurrencePHISC:
4052 case VPDef::VPHistogramSC:
4053 case VPDef::VPWidenPHISC:
4054 case VPDef::VPWidenIntOrFpInductionSC:
4055 case VPDef::VPWidenPointerInductionSC:
4056 case VPDef::VPReductionPHISC:
4057 case VPDef::VPInterleaveEVLSC:
4058 case VPDef::VPInterleaveSC:
4059 case VPDef::VPWidenLoadEVLSC:
4060 case VPDef::VPWidenLoadSC:
4061 case VPDef::VPWidenStoreEVLSC:
4062 case VPDef::VPWidenStoreSC:
4063 break;
4064 default:
4065 llvm_unreachable("unhandled recipe");
4066 }
4067
4068 auto WillGenerateTargetVectors = [&TTI, VF](Type *VectorTy) {
4069 unsigned NumLegalParts = TTI.getNumberOfParts(VectorTy);
4070 if (!NumLegalParts)
4071 return false;
4072 if (VF.isScalable()) {
4073 // <vscale x 1 x iN> is assumed to be profitable over iN because
4074 // scalable registers are a distinct register class from scalar
4075 // ones. If we ever find a target which wants to lower scalable
4076 // vectors back to scalars, we'll need to update this code to
4077 // explicitly ask TTI about the register class uses for each part.
4078 return NumLegalParts <= VF.getKnownMinValue();
4079 }
4080 // Two or more elements that share a register - are vectorized.
4081 return NumLegalParts < VF.getFixedValue();
4082 };
4083
4084 // If no def nor is a store, e.g., branches, continue - no value to check.
4085 if (R.getNumDefinedValues() == 0 &&
4087 continue;
4088 // For multi-def recipes, currently only interleaved loads, suffice to
4089 // check first def only.
4090 // For stores check their stored value; for interleaved stores suffice
4091 // the check first stored value only. In all cases this is the second
4092 // operand.
4093 VPValue *ToCheck =
4094 R.getNumDefinedValues() >= 1 ? R.getVPValue(0) : R.getOperand(1);
4095 Type *ScalarTy = TypeInfo.inferScalarType(ToCheck);
4096 if (!Visited.insert({ScalarTy}).second)
4097 continue;
4098 Type *WideTy = toVectorizedTy(ScalarTy, VF);
4099 if (any_of(getContainedTypes(WideTy), WillGenerateTargetVectors))
4100 return true;
4101 }
4102 }
4103
4104 return false;
4105}
4106
4107static bool hasReplicatorRegion(VPlan &Plan) {
4109 Plan.getVectorLoopRegion()->getEntry())),
4110 [](auto *VPRB) { return VPRB->isReplicator(); });
4111}
4112
4113#ifndef NDEBUG
4114VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() {
4115 InstructionCost ExpectedCost = CM.expectedCost(ElementCount::getFixed(1));
4116 LLVM_DEBUG(dbgs() << "LV: Scalar loop costs: " << ExpectedCost << ".\n");
4117 assert(ExpectedCost.isValid() && "Unexpected invalid cost for scalar loop");
4118 assert(
4119 any_of(VPlans,
4120 [](std::unique_ptr<VPlan> &P) { return P->hasScalarVFOnly(); }) &&
4121 "Expected Scalar VF to be a candidate");
4122
4123 const VectorizationFactor ScalarCost(ElementCount::getFixed(1), ExpectedCost,
4124 ExpectedCost);
4125 VectorizationFactor ChosenFactor = ScalarCost;
4126
4127 bool ForceVectorization = Hints.getForce() == LoopVectorizeHints::FK_Enabled;
4128 if (ForceVectorization &&
4129 (VPlans.size() > 1 || !VPlans[0]->hasScalarVFOnly())) {
4130 // Ignore scalar width, because the user explicitly wants vectorization.
4131 // Initialize cost to max so that VF = 2 is, at least, chosen during cost
4132 // evaluation.
4133 ChosenFactor.Cost = InstructionCost::getMax();
4134 }
4135
4136 for (auto &P : VPlans) {
4137 ArrayRef<ElementCount> VFs(P->vectorFactors().begin(),
4138 P->vectorFactors().end());
4139
4141 if (any_of(VFs, [this](ElementCount VF) {
4142 return CM.shouldConsiderRegPressureForVF(VF);
4143 }))
4144 RUs = calculateRegisterUsageForPlan(*P, VFs, TTI, CM.ValuesToIgnore);
4145
4146 for (unsigned I = 0; I < VFs.size(); I++) {
4147 ElementCount VF = VFs[I];
4148 // The cost for scalar VF=1 is already calculated, so ignore it.
4149 if (VF.isScalar())
4150 continue;
4151
4152 /// If the register pressure needs to be considered for VF,
4153 /// don't consider the VF as valid if it exceeds the number
4154 /// of registers for the target.
4155 if (CM.shouldConsiderRegPressureForVF(VF) &&
4156 RUs[I].exceedsMaxNumRegs(TTI, ForceTargetNumVectorRegs))
4157 continue;
4158
4159 InstructionCost C = CM.expectedCost(VF);
4160
4161 // Add on other costs that are modelled in VPlan, but not in the legacy
4162 // cost model.
4163 VPCostContext CostCtx(CM.TTI, *CM.TLI, *P, CM, CM.CostKind);
4164 VPRegionBlock *VectorRegion = P->getVectorLoopRegion();
4165 assert(VectorRegion && "Expected to have a vector region!");
4166 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
4167 vp_depth_first_shallow(VectorRegion->getEntry()))) {
4168 for (VPRecipeBase &R : *VPBB) {
4169 auto *VPI = dyn_cast<VPInstruction>(&R);
4170 if (!VPI)
4171 continue;
4172 switch (VPI->getOpcode()) {
4173 // Selects are only modelled in the legacy cost model for safe
4174 // divisors.
4175 case Instruction::Select: {
4176 VPValue *VPV = VPI->getVPSingleValue();
4177 if (VPV->getNumUsers() == 1) {
4178 if (auto *WR = dyn_cast<VPWidenRecipe>(*VPV->user_begin())) {
4179 switch (WR->getOpcode()) {
4180 case Instruction::UDiv:
4181 case Instruction::SDiv:
4182 case Instruction::URem:
4183 case Instruction::SRem:
4184 continue;
4185 default:
4186 break;
4187 }
4188 }
4189 }
4190 C += VPI->cost(VF, CostCtx);
4191 break;
4192 }
4194 unsigned Multiplier =
4195 cast<ConstantInt>(VPI->getOperand(2)->getLiveInIRValue())
4196 ->getZExtValue();
4197 C += VPI->cost(VF * Multiplier, CostCtx);
4198 break;
4199 }
4201 C += VPI->cost(VF, CostCtx);
4202 break;
4203 default:
4204 break;
4205 }
4206 }
4207 }
4208
4209 VectorizationFactor Candidate(VF, C, ScalarCost.ScalarCost);
4210 unsigned Width =
4211 estimateElementCount(Candidate.Width, CM.getVScaleForTuning());
4212 LLVM_DEBUG(dbgs() << "LV: Vector loop of width " << VF
4213 << " costs: " << (Candidate.Cost / Width));
4214 if (VF.isScalable())
4215 LLVM_DEBUG(dbgs() << " (assuming a minimum vscale of "
4216 << CM.getVScaleForTuning().value_or(1) << ")");
4217 LLVM_DEBUG(dbgs() << ".\n");
4218
4219 if (!ForceVectorization && !willGenerateVectors(*P, VF, TTI)) {
4220 LLVM_DEBUG(
4221 dbgs()
4222 << "LV: Not considering vector loop of width " << VF
4223 << " because it will not generate any vector instructions.\n");
4224 continue;
4225 }
4226
4227 if (CM.OptForSize && !ForceVectorization && hasReplicatorRegion(*P)) {
4228 LLVM_DEBUG(
4229 dbgs()
4230 << "LV: Not considering vector loop of width " << VF
4231 << " because it would cause replicated blocks to be generated,"
4232 << " which isn't allowed when optimizing for size.\n");
4233 continue;
4234 }
4235
4236 if (isMoreProfitable(Candidate, ChosenFactor, P->hasScalarTail()))
4237 ChosenFactor = Candidate;
4238 }
4239 }
4240
4241 if (!EnableCondStoresVectorization && CM.hasPredStores()) {
4243 "There are conditional stores.",
4244 "store that is conditionally executed prevents vectorization",
4245 "ConditionalStore", ORE, OrigLoop);
4246 ChosenFactor = ScalarCost;
4247 }
4248
4249 LLVM_DEBUG(if (ForceVectorization && !ChosenFactor.Width.isScalar() &&
4250 !isMoreProfitable(ChosenFactor, ScalarCost,
4251 !CM.foldTailByMasking())) dbgs()
4252 << "LV: Vectorization seems to be not beneficial, "
4253 << "but was forced by a user.\n");
4254 return ChosenFactor;
4255}
4256#endif
4257
4258bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
4259 ElementCount VF) const {
4260 // Cross iteration phis such as fixed-order recurrences and FMaxNum/FMinNum
4261 // reductions need special handling and are currently unsupported.
4262 if (any_of(OrigLoop->getHeader()->phis(), [&](PHINode &Phi) {
4263 if (!Legal->isReductionVariable(&Phi))
4264 return Legal->isFixedOrderRecurrence(&Phi);
4265 RecurKind RK = Legal->getRecurrenceDescriptor(&Phi).getRecurrenceKind();
4266 return RK == RecurKind::FMinNum || RK == RecurKind::FMaxNum;
4267 }))
4268 return false;
4269
4270 // Phis with uses outside of the loop require special handling and are
4271 // currently unsupported.
4272 for (const auto &Entry : Legal->getInductionVars()) {
4273 // Look for uses of the value of the induction at the last iteration.
4274 Value *PostInc =
4275 Entry.first->getIncomingValueForBlock(OrigLoop->getLoopLatch());
4276 for (User *U : PostInc->users())
4277 if (!OrigLoop->contains(cast<Instruction>(U)))
4278 return false;
4279 // Look for uses of penultimate value of the induction.
4280 for (User *U : Entry.first->users())
4281 if (!OrigLoop->contains(cast<Instruction>(U)))
4282 return false;
4283 }
4284
4285 // Epilogue vectorization code has not been auditted to ensure it handles
4286 // non-latch exits properly. It may be fine, but it needs auditted and
4287 // tested.
4288 // TODO: Add support for loops with an early exit.
4289 if (OrigLoop->getExitingBlock() != OrigLoop->getLoopLatch())
4290 return false;
4291
4292 return true;
4293}
4294
4296 const ElementCount VF, const unsigned IC) const {
4297 // FIXME: We need a much better cost-model to take different parameters such
4298 // as register pressure, code size increase and cost of extra branches into
4299 // account. For now we apply a very crude heuristic and only consider loops
4300 // with vectorization factors larger than a certain value.
4301
4302 // Allow the target to opt out entirely.
4303 if (!TTI.preferEpilogueVectorization())
4304 return false;
4305
4306 // We also consider epilogue vectorization unprofitable for targets that don't
4307 // consider interleaving beneficial (eg. MVE).
4308 if (TTI.getMaxInterleaveFactor(VF) <= 1)
4309 return false;
4310
4311 unsigned MinVFThreshold = EpilogueVectorizationMinVF.getNumOccurrences() > 0
4313 : TTI.getEpilogueVectorizationMinVF();
4314 return estimateElementCount(VF * IC, VScaleForTuning) >= MinVFThreshold;
4315}
4316
4318 const ElementCount MainLoopVF, unsigned IC) {
4321 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is disabled.\n");
4322 return Result;
4323 }
4324
4325 if (!CM.isScalarEpilogueAllowed()) {
4326 LLVM_DEBUG(dbgs() << "LEV: Unable to vectorize epilogue because no "
4327 "epilogue is allowed.\n");
4328 return Result;
4329 }
4330
4331 // Not really a cost consideration, but check for unsupported cases here to
4332 // simplify the logic.
4333 if (!isCandidateForEpilogueVectorization(MainLoopVF)) {
4334 LLVM_DEBUG(dbgs() << "LEV: Unable to vectorize epilogue because the loop "
4335 "is not a supported candidate.\n");
4336 return Result;
4337 }
4338
4340 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization factor is forced.\n");
4342 if (hasPlanWithVF(ForcedEC))
4343 return {ForcedEC, 0, 0};
4344
4345 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization forced factor is not "
4346 "viable.\n");
4347 return Result;
4348 }
4349
4350 if (OrigLoop->getHeader()->getParent()->hasOptSize()) {
4351 LLVM_DEBUG(
4352 dbgs() << "LEV: Epilogue vectorization skipped due to opt for size.\n");
4353 return Result;
4354 }
4355
4356 if (!CM.isEpilogueVectorizationProfitable(MainLoopVF, IC)) {
4357 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is not profitable for "
4358 "this loop\n");
4359 return Result;
4360 }
4361
4362 // If MainLoopVF = vscale x 2, and vscale is expected to be 4, then we know
4363 // the main loop handles 8 lanes per iteration. We could still benefit from
4364 // vectorizing the epilogue loop with VF=4.
4365 ElementCount EstimatedRuntimeVF = ElementCount::getFixed(
4366 estimateElementCount(MainLoopVF, CM.getVScaleForTuning()));
4367
4368 ScalarEvolution &SE = *PSE.getSE();
4369 Type *TCType = Legal->getWidestInductionType();
4370 const SCEV *RemainingIterations = nullptr;
4371 unsigned MaxTripCount = 0;
4372 const SCEV *TC =
4373 vputils::getSCEVExprForVPValue(getPlanFor(MainLoopVF).getTripCount(), SE);
4374 assert(!isa<SCEVCouldNotCompute>(TC) && "Trip count SCEV must be computable");
4375 RemainingIterations =
4376 SE.getURemExpr(TC, SE.getElementCount(TCType, MainLoopVF * IC));
4377
4378 // No iterations left to process in the epilogue.
4379 if (RemainingIterations->isZero())
4380 return Result;
4381
4382 if (MainLoopVF.isFixed()) {
4383 MaxTripCount = MainLoopVF.getFixedValue() * IC - 1;
4384 if (SE.isKnownPredicate(CmpInst::ICMP_ULT, RemainingIterations,
4385 SE.getConstant(TCType, MaxTripCount))) {
4386 MaxTripCount = SE.getUnsignedRangeMax(RemainingIterations).getZExtValue();
4387 }
4388 LLVM_DEBUG(dbgs() << "LEV: Maximum Trip Count for Epilogue: "
4389 << MaxTripCount << "\n");
4390 }
4391
4392 for (auto &NextVF : ProfitableVFs) {
4393 // Skip candidate VFs without a corresponding VPlan.
4394 if (!hasPlanWithVF(NextVF.Width))
4395 continue;
4396
4397 // Skip candidate VFs with widths >= the (estimated) runtime VF (scalable
4398 // vectors) or > the VF of the main loop (fixed vectors).
4399 if ((!NextVF.Width.isScalable() && MainLoopVF.isScalable() &&
4400 ElementCount::isKnownGE(NextVF.Width, EstimatedRuntimeVF)) ||
4401 (NextVF.Width.isScalable() &&
4402 ElementCount::isKnownGE(NextVF.Width, MainLoopVF)) ||
4403 (!NextVF.Width.isScalable() && !MainLoopVF.isScalable() &&
4404 ElementCount::isKnownGT(NextVF.Width, MainLoopVF)))
4405 continue;
4406
4407 // If NextVF is greater than the number of remaining iterations, the
4408 // epilogue loop would be dead. Skip such factors.
4409 if (RemainingIterations && !NextVF.Width.isScalable()) {
4410 if (SE.isKnownPredicate(
4412 SE.getConstant(TCType, NextVF.Width.getFixedValue()),
4413 RemainingIterations))
4414 continue;
4415 }
4416
4417 if (Result.Width.isScalar() ||
4418 isMoreProfitable(NextVF, Result, MaxTripCount, !CM.foldTailByMasking(),
4419 /*IsEpilogue*/ true))
4420 Result = NextVF;
4421 }
4422
4423 if (Result != VectorizationFactor::Disabled())
4424 LLVM_DEBUG(dbgs() << "LEV: Vectorizing epilogue loop with VF = "
4425 << Result.Width << "\n");
4426 return Result;
4427}
4428
4429std::pair<unsigned, unsigned>
4431 unsigned MinWidth = -1U;
4432 unsigned MaxWidth = 8;
4433 const DataLayout &DL = TheFunction->getDataLayout();
4434 // For in-loop reductions, no element types are added to ElementTypesInLoop
4435 // if there are no loads/stores in the loop. In this case, check through the
4436 // reduction variables to determine the maximum width.
4437 if (ElementTypesInLoop.empty() && !Legal->getReductionVars().empty()) {
4438 for (const auto &PhiDescriptorPair : Legal->getReductionVars()) {
4439 const RecurrenceDescriptor &RdxDesc = PhiDescriptorPair.second;
4440 // When finding the min width used by the recurrence we need to account
4441 // for casts on the input operands of the recurrence.
4442 MinWidth = std::min(
4443 MinWidth,
4444 std::min(RdxDesc.getMinWidthCastToRecurrenceTypeInBits(),
4446 MaxWidth = std::max(MaxWidth,
4448 }
4449 } else {
4450 for (Type *T : ElementTypesInLoop) {
4451 MinWidth = std::min<unsigned>(
4452 MinWidth, DL.getTypeSizeInBits(T->getScalarType()).getFixedValue());
4453 MaxWidth = std::max<unsigned>(
4454 MaxWidth, DL.getTypeSizeInBits(T->getScalarType()).getFixedValue());
4455 }
4456 }
4457 return {MinWidth, MaxWidth};
4458}
4459
4461 ElementTypesInLoop.clear();
4462 // For each block.
4463 for (BasicBlock *BB : TheLoop->blocks()) {
4464 // For each instruction in the loop.
4465 for (Instruction &I : BB->instructionsWithoutDebug()) {
4466 Type *T = I.getType();
4467
4468 // Skip ignored values.
4469 if (ValuesToIgnore.count(&I))
4470 continue;
4471
4472 // Only examine Loads, Stores and PHINodes.
4473 if (!isa<LoadInst>(I) && !isa<StoreInst>(I) && !isa<PHINode>(I))
4474 continue;
4475
4476 // Examine PHI nodes that are reduction variables. Update the type to
4477 // account for the recurrence type.
4478 if (auto *PN = dyn_cast<PHINode>(&I)) {
4479 if (!Legal->isReductionVariable(PN))
4480 continue;
4481 const RecurrenceDescriptor &RdxDesc =
4482 Legal->getRecurrenceDescriptor(PN);
4484 TTI.preferInLoopReduction(RdxDesc.getRecurrenceKind(),
4485 RdxDesc.getRecurrenceType()))
4486 continue;
4487 T = RdxDesc.getRecurrenceType();
4488 }
4489
4490 // Examine the stored values.
4491 if (auto *ST = dyn_cast<StoreInst>(&I))
4492 T = ST->getValueOperand()->getType();
4493
4494 assert(T->isSized() &&
4495 "Expected the load/store/recurrence type to be sized");
4496
4497 ElementTypesInLoop.insert(T);
4498 }
4499 }
4500}
4501
4502unsigned
4504 InstructionCost LoopCost) {
4505 // -- The interleave heuristics --
4506 // We interleave the loop in order to expose ILP and reduce the loop overhead.
4507 // There are many micro-architectural considerations that we can't predict
4508 // at this level. For example, frontend pressure (on decode or fetch) due to
4509 // code size, or the number and capabilities of the execution ports.
4510 //
4511 // We use the following heuristics to select the interleave count:
4512 // 1. If the code has reductions, then we interleave to break the cross
4513 // iteration dependency.
4514 // 2. If the loop is really small, then we interleave to reduce the loop
4515 // overhead.
4516 // 3. We don't interleave if we think that we will spill registers to memory
4517 // due to the increased register pressure.
4518
4519 if (!CM.isScalarEpilogueAllowed())
4520 return 1;
4521
4524 LLVM_DEBUG(dbgs() << "LV: Preference for VP intrinsics indicated. "
4525 "Unroll factor forced to be 1.\n");
4526 return 1;
4527 }
4528
4529 // We used the distance for the interleave count.
4530 if (!Legal->isSafeForAnyVectorWidth())
4531 return 1;
4532
4533 // We don't attempt to perform interleaving for loops with uncountable early
4534 // exits because the VPInstruction::AnyOf code cannot currently handle
4535 // multiple parts.
4536 if (Plan.hasEarlyExit())
4537 return 1;
4538
4539 const bool HasReductions =
4542
4543 // If we did not calculate the cost for VF (because the user selected the VF)
4544 // then we calculate the cost of VF here.
4545 if (LoopCost == 0) {
4546 if (VF.isScalar())
4547 LoopCost = CM.expectedCost(VF);
4548 else
4549 LoopCost = cost(Plan, VF);
4550 assert(LoopCost.isValid() && "Expected to have chosen a VF with valid cost");
4551
4552 // Loop body is free and there is no need for interleaving.
4553 if (LoopCost == 0)
4554 return 1;
4555 }
4556
4557 VPRegisterUsage R =
4558 calculateRegisterUsageForPlan(Plan, {VF}, TTI, CM.ValuesToIgnore)[0];
4559 // We divide by these constants so assume that we have at least one
4560 // instruction that uses at least one register.
4561 for (auto &Pair : R.MaxLocalUsers) {
4562 Pair.second = std::max(Pair.second, 1U);
4563 }
4564
4565 // We calculate the interleave count using the following formula.
4566 // Subtract the number of loop invariants from the number of available
4567 // registers. These registers are used by all of the interleaved instances.
4568 // Next, divide the remaining registers by the number of registers that is
4569 // required by the loop, in order to estimate how many parallel instances
4570 // fit without causing spills. All of this is rounded down if necessary to be
4571 // a power of two. We want power of two interleave count to simplify any
4572 // addressing operations or alignment considerations.
4573 // We also want power of two interleave counts to ensure that the induction
4574 // variable of the vector loop wraps to zero, when tail is folded by masking;
4575 // this currently happens when OptForSize, in which case IC is set to 1 above.
4576 unsigned IC = UINT_MAX;
4577
4578 for (const auto &Pair : R.MaxLocalUsers) {
4579 unsigned TargetNumRegisters = TTI.getNumberOfRegisters(Pair.first);
4580 LLVM_DEBUG(dbgs() << "LV: The target has " << TargetNumRegisters
4581 << " registers of "
4582 << TTI.getRegisterClassName(Pair.first)
4583 << " register class\n");
4584 if (VF.isScalar()) {
4585 if (ForceTargetNumScalarRegs.getNumOccurrences() > 0)
4586 TargetNumRegisters = ForceTargetNumScalarRegs;
4587 } else {
4588 if (ForceTargetNumVectorRegs.getNumOccurrences() > 0)
4589 TargetNumRegisters = ForceTargetNumVectorRegs;
4590 }
4591 unsigned MaxLocalUsers = Pair.second;
4592 unsigned LoopInvariantRegs = 0;
4593 if (R.LoopInvariantRegs.contains(Pair.first))
4594 LoopInvariantRegs = R.LoopInvariantRegs[Pair.first];
4595
4596 unsigned TmpIC = llvm::bit_floor((TargetNumRegisters - LoopInvariantRegs) /
4597 MaxLocalUsers);
4598 // Don't count the induction variable as interleaved.
4600 TmpIC = llvm::bit_floor((TargetNumRegisters - LoopInvariantRegs - 1) /
4601 std::max(1U, (MaxLocalUsers - 1)));
4602 }
4603
4604 IC = std::min(IC, TmpIC);
4605 }
4606
4607 // Clamp the interleave ranges to reasonable counts.
4608 unsigned MaxInterleaveCount = TTI.getMaxInterleaveFactor(VF);
4609
4610 // Check if the user has overridden the max.
4611 if (VF.isScalar()) {
4612 if (ForceTargetMaxScalarInterleaveFactor.getNumOccurrences() > 0)
4613 MaxInterleaveCount = ForceTargetMaxScalarInterleaveFactor;
4614 } else {
4615 if (ForceTargetMaxVectorInterleaveFactor.getNumOccurrences() > 0)
4616 MaxInterleaveCount = ForceTargetMaxVectorInterleaveFactor;
4617 }
4618
4619 // Try to get the exact trip count, or an estimate based on profiling data or
4620 // ConstantMax from PSE, failing that.
4621 auto BestKnownTC = getSmallBestKnownTC(PSE, OrigLoop);
4622
4623 // For fixed length VFs treat a scalable trip count as unknown.
4624 if (BestKnownTC && (BestKnownTC->isFixed() || VF.isScalable())) {
4625 // Re-evaluate trip counts and VFs to be in the same numerical space.
4626 unsigned AvailableTC =
4627 estimateElementCount(*BestKnownTC, CM.getVScaleForTuning());
4628 unsigned EstimatedVF = estimateElementCount(VF, CM.getVScaleForTuning());
4629
4630 // At least one iteration must be scalar when this constraint holds. So the
4631 // maximum available iterations for interleaving is one less.
4632 if (CM.requiresScalarEpilogue(VF.isVector()))
4633 --AvailableTC;
4634
4635 unsigned InterleaveCountLB = bit_floor(std::max(
4636 1u, std::min(AvailableTC / (EstimatedVF * 2), MaxInterleaveCount)));
4637
4638 if (getSmallConstantTripCount(PSE.getSE(), OrigLoop).isNonZero()) {
4639 // If the best known trip count is exact, we select between two
4640 // prospective ICs, where
4641 //
4642 // 1) the aggressive IC is capped by the trip count divided by VF
4643 // 2) the conservative IC is capped by the trip count divided by (VF * 2)
4644 //
4645 // The final IC is selected in a way that the epilogue loop trip count is
4646 // minimized while maximizing the IC itself, so that we either run the
4647 // vector loop at least once if it generates a small epilogue loop, or
4648 // else we run the vector loop at least twice.
4649
4650 unsigned InterleaveCountUB = bit_floor(std::max(
4651 1u, std::min(AvailableTC / EstimatedVF, MaxInterleaveCount)));
4652 MaxInterleaveCount = InterleaveCountLB;
4653
4654 if (InterleaveCountUB != InterleaveCountLB) {
4655 unsigned TailTripCountUB =
4656 (AvailableTC % (EstimatedVF * InterleaveCountUB));
4657 unsigned TailTripCountLB =
4658 (AvailableTC % (EstimatedVF * InterleaveCountLB));
4659 // If both produce same scalar tail, maximize the IC to do the same work
4660 // in fewer vector loop iterations
4661 if (TailTripCountUB == TailTripCountLB)
4662 MaxInterleaveCount = InterleaveCountUB;
4663 }
4664 } else {
4665 // If trip count is an estimated compile time constant, limit the
4666 // IC to be capped by the trip count divided by VF * 2, such that the
4667 // vector loop runs at least twice to make interleaving seem profitable
4668 // when there is an epilogue loop present. Since exact Trip count is not
4669 // known we choose to be conservative in our IC estimate.
4670 MaxInterleaveCount = InterleaveCountLB;
4671 }
4672 }
4673
4674 assert(MaxInterleaveCount > 0 &&
4675 "Maximum interleave count must be greater than 0");
4676
4677 // Clamp the calculated IC to be between the 1 and the max interleave count
4678 // that the target and trip count allows.
4679 if (IC > MaxInterleaveCount)
4680 IC = MaxInterleaveCount;
4681 else
4682 // Make sure IC is greater than 0.
4683 IC = std::max(1u, IC);
4684
4685 assert(IC > 0 && "Interleave count must be greater than 0.");
4686
4687 // Interleave if we vectorized this loop and there is a reduction that could
4688 // benefit from interleaving.
4689 if (VF.isVector() && HasReductions) {
4690 LLVM_DEBUG(dbgs() << "LV: Interleaving because of reductions.\n");
4691 return IC;
4692 }
4693
4694 // For any scalar loop that either requires runtime checks or predication we
4695 // are better off leaving this to the unroller. Note that if we've already
4696 // vectorized the loop we will have done the runtime check and so interleaving
4697 // won't require further checks.
4698 bool ScalarInterleavingRequiresPredication =
4699 (VF.isScalar() && any_of(OrigLoop->blocks(), [this](BasicBlock *BB) {
4700 return Legal->blockNeedsPredication(BB);
4701 }));
4702 bool ScalarInterleavingRequiresRuntimePointerCheck =
4703 (VF.isScalar() && Legal->getRuntimePointerChecking()->Need);
4704
4705 // We want to interleave small loops in order to reduce the loop overhead and
4706 // potentially expose ILP opportunities.
4707 LLVM_DEBUG(dbgs() << "LV: Loop cost is " << LoopCost << '\n'
4708 << "LV: IC is " << IC << '\n'
4709 << "LV: VF is " << VF << '\n');
4710 const bool AggressivelyInterleaveReductions =
4711 TTI.enableAggressiveInterleaving(HasReductions);
4712 if (!ScalarInterleavingRequiresRuntimePointerCheck &&
4713 !ScalarInterleavingRequiresPredication && LoopCost < SmallLoopCost) {
4714 // We assume that the cost overhead is 1 and we use the cost model
4715 // to estimate the cost of the loop and interleave until the cost of the
4716 // loop overhead is about 5% of the cost of the loop.
4717 unsigned SmallIC = std::min(IC, (unsigned)llvm::bit_floor<uint64_t>(
4718 SmallLoopCost / LoopCost.getValue()));
4719
4720 // Interleave until store/load ports (estimated by max interleave count) are
4721 // saturated.
4722 unsigned NumStores = 0;
4723 unsigned NumLoads = 0;
4726 for (VPRecipeBase &R : *VPBB) {
4728 NumLoads++;
4729 continue;
4730 }
4732 NumStores++;
4733 continue;
4734 }
4735
4736 if (auto *InterleaveR = dyn_cast<VPInterleaveRecipe>(&R)) {
4737 if (unsigned StoreOps = InterleaveR->getNumStoreOperands())
4738 NumStores += StoreOps;
4739 else
4740 NumLoads += InterleaveR->getNumDefinedValues();
4741 continue;
4742 }
4743 if (auto *RepR = dyn_cast<VPReplicateRecipe>(&R)) {
4744 NumLoads += isa<LoadInst>(RepR->getUnderlyingInstr());
4745 NumStores += isa<StoreInst>(RepR->getUnderlyingInstr());
4746 continue;
4747 }
4748 if (isa<VPHistogramRecipe>(&R)) {
4749 NumLoads++;
4750 NumStores++;
4751 continue;
4752 }
4753 }
4754 }
4755 unsigned StoresIC = IC / (NumStores ? NumStores : 1);
4756 unsigned LoadsIC = IC / (NumLoads ? NumLoads : 1);
4757
4758 // There is little point in interleaving for reductions containing selects
4759 // and compares when VF=1 since it may just create more overhead than it's
4760 // worth for loops with small trip counts. This is because we still have to
4761 // do the final reduction after the loop.
4762 bool HasSelectCmpReductions =
4763 HasReductions &&
4765 [](VPRecipeBase &R) {
4766 auto *RedR = dyn_cast<VPReductionPHIRecipe>(&R);
4767 return RedR && (RecurrenceDescriptor::isAnyOfRecurrenceKind(
4768 RedR->getRecurrenceKind()) ||
4769 RecurrenceDescriptor::isFindIVRecurrenceKind(
4770 RedR->getRecurrenceKind()));
4771 });
4772 if (HasSelectCmpReductions) {
4773 LLVM_DEBUG(dbgs() << "LV: Not interleaving select-cmp reductions.\n");
4774 return 1;
4775 }
4776
4777 // If we have a scalar reduction (vector reductions are already dealt with
4778 // by this point), we can increase the critical path length if the loop
4779 // we're interleaving is inside another loop. For tree-wise reductions
4780 // set the limit to 2, and for ordered reductions it's best to disable
4781 // interleaving entirely.
4782 if (HasReductions && OrigLoop->getLoopDepth() > 1) {
4783 bool HasOrderedReductions =
4785 [](VPRecipeBase &R) {
4786 auto *RedR = dyn_cast<VPReductionPHIRecipe>(&R);
4787
4788 return RedR && RedR->isOrdered();
4789 });
4790 if (HasOrderedReductions) {
4791 LLVM_DEBUG(
4792 dbgs() << "LV: Not interleaving scalar ordered reductions.\n");
4793 return 1;
4794 }
4795
4796 unsigned F = MaxNestedScalarReductionIC;
4797 SmallIC = std::min(SmallIC, F);
4798 StoresIC = std::min(StoresIC, F);
4799 LoadsIC = std::min(LoadsIC, F);
4800 }
4801
4803 std::max(StoresIC, LoadsIC) > SmallIC) {
4804 LLVM_DEBUG(
4805 dbgs() << "LV: Interleaving to saturate store or load ports.\n");
4806 return std::max(StoresIC, LoadsIC);
4807 }
4808
4809 // If there are scalar reductions and TTI has enabled aggressive
4810 // interleaving for reductions, we will interleave to expose ILP.
4811 if (VF.isScalar() && AggressivelyInterleaveReductions) {
4812 LLVM_DEBUG(dbgs() << "LV: Interleaving to expose ILP.\n");
4813 // Interleave no less than SmallIC but not as aggressive as the normal IC
4814 // to satisfy the rare situation when resources are too limited.
4815 return std::max(IC / 2, SmallIC);
4816 }
4817
4818 LLVM_DEBUG(dbgs() << "LV: Interleaving to reduce branch cost.\n");
4819 return SmallIC;
4820 }
4821
4822 // Interleave if this is a large loop (small loops are already dealt with by
4823 // this point) that could benefit from interleaving.
4824 if (AggressivelyInterleaveReductions) {
4825 LLVM_DEBUG(dbgs() << "LV: Interleaving to expose ILP.\n");
4826 return IC;
4827 }
4828
4829 LLVM_DEBUG(dbgs() << "LV: Not Interleaving.\n");
4830 return 1;
4831}
4832
4833bool LoopVectorizationCostModel::useEmulatedMaskMemRefHack(Instruction *I,
4834 ElementCount VF) {
4835 // TODO: Cost model for emulated masked load/store is completely
4836 // broken. This hack guides the cost model to use an artificially
4837 // high enough value to practically disable vectorization with such
4838 // operations, except where previously deployed legality hack allowed
4839 // using very low cost values. This is to avoid regressions coming simply
4840 // from moving "masked load/store" check from legality to cost model.
4841 // Masked Load/Gather emulation was previously never allowed.
4842 // Limited number of Masked Store/Scatter emulation was allowed.
4843 assert((isPredicatedInst(I)) &&
4844 "Expecting a scalar emulated instruction");
4845 return isa<LoadInst>(I) ||
4846 (isa<StoreInst>(I) &&
4847 NumPredStores > NumberOfStoresToPredicate);
4848}
4849
4851 assert(VF.isVector() && "Expected VF >= 2");
4852
4853 // If we've already collected the instructions to scalarize or the predicated
4854 // BBs after vectorization, there's nothing to do. Collection may already have
4855 // occurred if we have a user-selected VF and are now computing the expected
4856 // cost for interleaving.
4857 if (InstsToScalarize.contains(VF) ||
4858 PredicatedBBsAfterVectorization.contains(VF))
4859 return;
4860
4861 // Initialize a mapping for VF in InstsToScalalarize. If we find that it's
4862 // not profitable to scalarize any instructions, the presence of VF in the
4863 // map will indicate that we've analyzed it already.
4864 ScalarCostsTy &ScalarCostsVF = InstsToScalarize[VF];
4865
4866 // Find all the instructions that are scalar with predication in the loop and
4867 // determine if it would be better to not if-convert the blocks they are in.
4868 // If so, we also record the instructions to scalarize.
4869 for (BasicBlock *BB : TheLoop->blocks()) {
4871 continue;
4872 for (Instruction &I : *BB)
4873 if (isScalarWithPredication(&I, VF)) {
4874 ScalarCostsTy ScalarCosts;
4875 // Do not apply discount logic for:
4876 // 1. Scalars after vectorization, as there will only be a single copy
4877 // of the instruction.
4878 // 2. Scalable VF, as that would lead to invalid scalarization costs.
4879 // 3. Emulated masked memrefs, if a hacked cost is needed.
4880 if (!isScalarAfterVectorization(&I, VF) && !VF.isScalable() &&
4881 !useEmulatedMaskMemRefHack(&I, VF) &&
4882 computePredInstDiscount(&I, ScalarCosts, VF) >= 0) {
4883 for (const auto &[I, IC] : ScalarCosts)
4884 ScalarCostsVF.insert({I, IC});
4885 // Check if we decided to scalarize a call. If so, update the widening
4886 // decision of the call to CM_Scalarize with the computed scalar cost.
4887 for (const auto &[I, Cost] : ScalarCosts) {
4888 auto *CI = dyn_cast<CallInst>(I);
4889 if (!CI || !CallWideningDecisions.contains({CI, VF}))
4890 continue;
4891 CallWideningDecisions[{CI, VF}].Kind = CM_Scalarize;
4892 CallWideningDecisions[{CI, VF}].Cost = Cost;
4893 }
4894 }
4895 // Remember that BB will remain after vectorization.
4896 PredicatedBBsAfterVectorization[VF].insert(BB);
4897 for (auto *Pred : predecessors(BB)) {
4898 if (Pred->getSingleSuccessor() == BB)
4899 PredicatedBBsAfterVectorization[VF].insert(Pred);
4900 }
4901 }
4902 }
4903}
4904
4905InstructionCost LoopVectorizationCostModel::computePredInstDiscount(
4906 Instruction *PredInst, ScalarCostsTy &ScalarCosts, ElementCount VF) {
4907 assert(!isUniformAfterVectorization(PredInst, VF) &&
4908 "Instruction marked uniform-after-vectorization will be predicated");
4909
4910 // Initialize the discount to zero, meaning that the scalar version and the
4911 // vector version cost the same.
4912 InstructionCost Discount = 0;
4913
4914 // Holds instructions to analyze. The instructions we visit are mapped in
4915 // ScalarCosts. Those instructions are the ones that would be scalarized if
4916 // we find that the scalar version costs less.
4918
4919 // Returns true if the given instruction can be scalarized.
4920 auto CanBeScalarized = [&](Instruction *I) -> bool {
4921 // We only attempt to scalarize instructions forming a single-use chain
4922 // from the original predicated block that would otherwise be vectorized.
4923 // Although not strictly necessary, we give up on instructions we know will
4924 // already be scalar to avoid traversing chains that are unlikely to be
4925 // beneficial.
4926 if (!I->hasOneUse() || PredInst->getParent() != I->getParent() ||
4927 isScalarAfterVectorization(I, VF))
4928 return false;
4929
4930 // If the instruction is scalar with predication, it will be analyzed
4931 // separately. We ignore it within the context of PredInst.
4932 if (isScalarWithPredication(I, VF))
4933 return false;
4934
4935 // If any of the instruction's operands are uniform after vectorization,
4936 // the instruction cannot be scalarized. This prevents, for example, a
4937 // masked load from being scalarized.
4938 //
4939 // We assume we will only emit a value for lane zero of an instruction
4940 // marked uniform after vectorization, rather than VF identical values.
4941 // Thus, if we scalarize an instruction that uses a uniform, we would
4942 // create uses of values corresponding to the lanes we aren't emitting code
4943 // for. This behavior can be changed by allowing getScalarValue to clone
4944 // the lane zero values for uniforms rather than asserting.
4945 for (Use &U : I->operands())
4946 if (auto *J = dyn_cast<Instruction>(U.get()))
4947 if (isUniformAfterVectorization(J, VF))
4948 return false;
4949
4950 // Otherwise, we can scalarize the instruction.
4951 return true;
4952 };
4953
4954 // Compute the expected cost discount from scalarizing the entire expression
4955 // feeding the predicated instruction. We currently only consider expressions
4956 // that are single-use instruction chains.
4957 Worklist.push_back(PredInst);
4958 while (!Worklist.empty()) {
4959 Instruction *I = Worklist.pop_back_val();
4960
4961 // If we've already analyzed the instruction, there's nothing to do.
4962 if (ScalarCosts.contains(I))
4963 continue;
4964
4965 // Cannot scalarize fixed-order recurrence phis at the moment.
4966 if (isa<PHINode>(I) && Legal->isFixedOrderRecurrence(cast<PHINode>(I)))
4967 continue;
4968
4969 // Compute the cost of the vector instruction. Note that this cost already
4970 // includes the scalarization overhead of the predicated instruction.
4971 InstructionCost VectorCost = getInstructionCost(I, VF);
4972
4973 // Compute the cost of the scalarized instruction. This cost is the cost of
4974 // the instruction as if it wasn't if-converted and instead remained in the
4975 // predicated block. We will scale this cost by block probability after
4976 // computing the scalarization overhead.
4977 InstructionCost ScalarCost =
4978 VF.getFixedValue() * getInstructionCost(I, ElementCount::getFixed(1));
4979
4980 // Compute the scalarization overhead of needed insertelement instructions
4981 // and phi nodes.
4982 if (isScalarWithPredication(I, VF) && !I->getType()->isVoidTy()) {
4983 Type *WideTy = toVectorizedTy(I->getType(), VF);
4984 for (Type *VectorTy : getContainedTypes(WideTy)) {
4985 ScalarCost += TTI.getScalarizationOverhead(
4987 /*Insert=*/true,
4988 /*Extract=*/false, CostKind);
4989 }
4990 ScalarCost +=
4991 VF.getFixedValue() * TTI.getCFInstrCost(Instruction::PHI, CostKind);
4992 }
4993
4994 // Compute the scalarization overhead of needed extractelement
4995 // instructions. For each of the instruction's operands, if the operand can
4996 // be scalarized, add it to the worklist; otherwise, account for the
4997 // overhead.
4998 for (Use &U : I->operands())
4999 if (auto *J = dyn_cast<Instruction>(U.get())) {
5000 assert(canVectorizeTy(J->getType()) &&
5001 "Instruction has non-scalar type");
5002 if (CanBeScalarized(J))
5003 Worklist.push_back(J);
5004 else if (needsExtract(J, VF)) {
5005 Type *WideTy = toVectorizedTy(J->getType(), VF);
5006 for (Type *VectorTy : getContainedTypes(WideTy)) {
5007 ScalarCost += TTI.getScalarizationOverhead(
5008 cast<VectorType>(VectorTy),
5009 APInt::getAllOnes(VF.getFixedValue()), /*Insert*/ false,
5010 /*Extract*/ true, CostKind);
5011 }
5012 }
5013 }
5014
5015 // Scale the total scalar cost by block probability.
5016 ScalarCost /= getPredBlockCostDivisor(CostKind);
5017
5018 // Compute the discount. A non-negative discount means the vector version
5019 // of the instruction costs more, and scalarizing would be beneficial.
5020 Discount += VectorCost - ScalarCost;
5021 ScalarCosts[I] = ScalarCost;
5022 }
5023
5024 return Discount;
5025}
5026
5029
5030 // If the vector loop gets executed exactly once with the given VF, ignore the
5031 // costs of comparison and induction instructions, as they'll get simplified
5032 // away.
5033 SmallPtrSet<Instruction *, 2> ValuesToIgnoreForVF;
5034 auto TC = getSmallConstantTripCount(PSE.getSE(), TheLoop);
5035 if (TC == VF && !foldTailByMasking())
5037 ValuesToIgnoreForVF);
5038
5039 // For each block.
5040 for (BasicBlock *BB : TheLoop->blocks()) {
5041 InstructionCost BlockCost;
5042
5043 // For each instruction in the old loop.
5044 for (Instruction &I : BB->instructionsWithoutDebug()) {
5045 // Skip ignored values.
5046 if (ValuesToIgnore.count(&I) || ValuesToIgnoreForVF.count(&I) ||
5047 (VF.isVector() && VecValuesToIgnore.count(&I)))
5048 continue;
5049
5051
5052 // Check if we should override the cost.
5053 if (C.isValid() && ForceTargetInstructionCost.getNumOccurrences() > 0)
5055
5056 BlockCost += C;
5057 LLVM_DEBUG(dbgs() << "LV: Found an estimated cost of " << C << " for VF "
5058 << VF << " For instruction: " << I << '\n');
5059 }
5060
5061 // If we are vectorizing a predicated block, it will have been
5062 // if-converted. This means that the block's instructions (aside from
5063 // stores and instructions that may divide by zero) will now be
5064 // unconditionally executed. For the scalar case, we may not always execute
5065 // the predicated block, if it is an if-else block. Thus, scale the block's
5066 // cost by the probability of executing it. blockNeedsPredication from
5067 // Legal is used so as to not include all blocks in tail folded loops.
5068 if (VF.isScalar() && Legal->blockNeedsPredication(BB))
5069 BlockCost /= getPredBlockCostDivisor(CostKind);
5070
5071 Cost += BlockCost;
5072 }
5073
5074 return Cost;
5075}
5076
5077/// Gets Address Access SCEV after verifying that the access pattern
5078/// is loop invariant except the induction variable dependence.
5079///
5080/// This SCEV can be sent to the Target in order to estimate the address
5081/// calculation cost.
5083 Value *Ptr,
5086 const Loop *TheLoop) {
5087
5088 auto *Gep = dyn_cast<GetElementPtrInst>(Ptr);
5089 if (!Gep)
5090 return nullptr;
5091
5092 // We are looking for a gep with all loop invariant indices except for one
5093 // which should be an induction variable.
5094 auto *SE = PSE.getSE();
5095 unsigned NumOperands = Gep->getNumOperands();
5096 for (unsigned Idx = 1; Idx < NumOperands; ++Idx) {
5097 Value *Opd = Gep->getOperand(Idx);
5098 if (!SE->isLoopInvariant(SE->getSCEV(Opd), TheLoop) &&
5099 !Legal->isInductionVariable(Opd))
5100 return nullptr;
5101 }
5102
5103 // Now we know we have a GEP ptr, %inv, %ind, %inv. return the Ptr SCEV.
5104 return PSE.getSCEV(Ptr);
5105}
5106
5108LoopVectorizationCostModel::getMemInstScalarizationCost(Instruction *I,
5109 ElementCount VF) {
5110 assert(VF.isVector() &&
5111 "Scalarization cost of instruction implies vectorization.");
5112 if (VF.isScalable())
5113 return InstructionCost::getInvalid();
5114
5115 Type *ValTy = getLoadStoreType(I);
5116 auto *SE = PSE.getSE();
5117
5118 unsigned AS = getLoadStoreAddressSpace(I);
5120 Type *PtrTy = toVectorTy(Ptr->getType(), VF);
5121 // NOTE: PtrTy is a vector to signal `TTI::getAddressComputationCost`
5122 // that it is being called from this specific place.
5123
5124 // Figure out whether the access is strided and get the stride value
5125 // if it's known in compile time
5126 const SCEV *PtrSCEV = getAddressAccessSCEV(Ptr, Legal, PSE, TheLoop);
5127
5128 // Get the cost of the scalar memory instruction and address computation.
5130 PtrTy, SE, PtrSCEV, CostKind);
5131
5132 // Don't pass *I here, since it is scalar but will actually be part of a
5133 // vectorized loop where the user of it is a vectorized instruction.
5134 const Align Alignment = getLoadStoreAlignment(I);
5135 TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(I->getOperand(0));
5136 Cost += VF.getFixedValue() *
5137 TTI.getMemoryOpCost(I->getOpcode(), ValTy->getScalarType(), Alignment,
5138 AS, CostKind, OpInfo);
5139
5140 // Get the overhead of the extractelement and insertelement instructions
5141 // we might create due to scalarization.
5143
5144 // If we have a predicated load/store, it will need extra i1 extracts and
5145 // conditional branches, but may not be executed for each vector lane. Scale
5146 // the cost by the probability of executing the predicated block.
5147 if (isPredicatedInst(I)) {
5149
5150 // Add the cost of an i1 extract and a branch
5151 auto *VecI1Ty =
5152 VectorType::get(IntegerType::getInt1Ty(ValTy->getContext()), VF);
5154 VecI1Ty, APInt::getAllOnes(VF.getFixedValue()),
5155 /*Insert=*/false, /*Extract=*/true, CostKind);
5156 Cost += TTI.getCFInstrCost(Instruction::Br, CostKind);
5157
5158 if (useEmulatedMaskMemRefHack(I, VF))
5159 // Artificially setting to a high enough value to practically disable
5160 // vectorization with such operations.
5161 Cost = 3000000;
5162 }
5163
5164 return Cost;
5165}
5166
5168LoopVectorizationCostModel::getConsecutiveMemOpCost(Instruction *I,
5169 ElementCount VF) {
5170 Type *ValTy = getLoadStoreType(I);
5171 auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
5173 unsigned AS = getLoadStoreAddressSpace(I);
5174 int ConsecutiveStride = Legal->isConsecutivePtr(ValTy, Ptr);
5175
5176 assert((ConsecutiveStride == 1 || ConsecutiveStride == -1) &&
5177 "Stride should be 1 or -1 for consecutive memory access");
5178 const Align Alignment = getLoadStoreAlignment(I);
5180 if (Legal->isMaskRequired(I)) {
5181 Cost += TTI.getMaskedMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS,
5182 CostKind);
5183 } else {
5184 TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(I->getOperand(0));
5185 Cost += TTI.getMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS,
5186 CostKind, OpInfo, I);
5187 }
5188
5189 bool Reverse = ConsecutiveStride < 0;
5190 if (Reverse)
5192 VectorTy, {}, CostKind, 0);
5193 return Cost;
5194}
5195
5197LoopVectorizationCostModel::getUniformMemOpCost(Instruction *I,
5198 ElementCount VF) {
5199 assert(Legal->isUniformMemOp(*I, VF));
5200
5201 Type *ValTy = getLoadStoreType(I);
5203 auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
5204 const Align Alignment = getLoadStoreAlignment(I);
5205 unsigned AS = getLoadStoreAddressSpace(I);
5206 if (isa<LoadInst>(I)) {
5207 return TTI.getAddressComputationCost(PtrTy, nullptr, nullptr, CostKind) +
5208 TTI.getMemoryOpCost(Instruction::Load, ValTy, Alignment, AS,
5209 CostKind) +
5211 VectorTy, {}, CostKind);
5212 }
5213 StoreInst *SI = cast<StoreInst>(I);
5214
5215 bool IsLoopInvariantStoreValue = Legal->isInvariant(SI->getValueOperand());
5216 // TODO: We have existing tests that request the cost of extracting element
5217 // VF.getKnownMinValue() - 1 from a scalable vector. This does not represent
5218 // the actual generated code, which involves extracting the last element of
5219 // a scalable vector where the lane to extract is unknown at compile time.
5221 TTI.getAddressComputationCost(PtrTy, nullptr, nullptr, CostKind) +
5222 TTI.getMemoryOpCost(Instruction::Store, ValTy, Alignment, AS, CostKind);
5223 if (!IsLoopInvariantStoreValue)
5224 Cost += TTI.getIndexedVectorInstrCostFromEnd(Instruction::ExtractElement,
5225 VectorTy, CostKind, 0);
5226 return Cost;
5227}
5228
5230LoopVectorizationCostModel::getGatherScatterCost(Instruction *I,
5231 ElementCount VF) {
5232 Type *ValTy = getLoadStoreType(I);
5233 auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
5234 const Align Alignment = getLoadStoreAlignment(I);
5236 Type *PtrTy = Ptr->getType();
5237
5238 if (!Legal->isUniform(Ptr, VF))
5239 PtrTy = toVectorTy(PtrTy, VF);
5240
5241 return TTI.getAddressComputationCost(PtrTy, nullptr, nullptr, CostKind) +
5242 TTI.getGatherScatterOpCost(I->getOpcode(), VectorTy, Ptr,
5243 Legal->isMaskRequired(I), Alignment,
5244 CostKind, I);
5245}
5246
5248LoopVectorizationCostModel::getInterleaveGroupCost(Instruction *I,
5249 ElementCount VF) {
5250 const auto *Group = getInterleavedAccessGroup(I);
5251 assert(Group && "Fail to get an interleaved access group.");
5252
5253 Instruction *InsertPos = Group->getInsertPos();
5254 Type *ValTy = getLoadStoreType(InsertPos);
5255 auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
5256 unsigned AS = getLoadStoreAddressSpace(InsertPos);
5257
5258 unsigned InterleaveFactor = Group->getFactor();
5259 auto *WideVecTy = VectorType::get(ValTy, VF * InterleaveFactor);
5260
5261 // Holds the indices of existing members in the interleaved group.
5262 SmallVector<unsigned, 4> Indices;
5263 for (unsigned IF = 0; IF < InterleaveFactor; IF++)
5264 if (Group->getMember(IF))
5265 Indices.push_back(IF);
5266
5267 // Calculate the cost of the whole interleaved group.
5268 bool UseMaskForGaps =
5269 (Group->requiresScalarEpilogue() && !isScalarEpilogueAllowed()) ||
5270 (isa<StoreInst>(I) && !Group->isFull());
5272 InsertPos->getOpcode(), WideVecTy, Group->getFactor(), Indices,
5273 Group->getAlign(), AS, CostKind, Legal->isMaskRequired(I),
5274 UseMaskForGaps);
5275
5276 if (Group->isReverse()) {
5277 // TODO: Add support for reversed masked interleaved access.
5278 assert(!Legal->isMaskRequired(I) &&
5279 "Reverse masked interleaved access not supported.");
5280 Cost += Group->getNumMembers() *
5282 VectorTy, {}, CostKind, 0);
5283 }
5284 return Cost;
5285}
5286
5287std::optional<InstructionCost>
5289 ElementCount VF,
5290 Type *Ty) const {
5291 using namespace llvm::PatternMatch;
5292 // Early exit for no inloop reductions
5293 if (InLoopReductions.empty() || VF.isScalar() || !isa<VectorType>(Ty))
5294 return std::nullopt;
5295 auto *VectorTy = cast<VectorType>(Ty);
5296
5297 // We are looking for a pattern of, and finding the minimal acceptable cost:
5298 // reduce(mul(ext(A), ext(B))) or
5299 // reduce(mul(A, B)) or
5300 // reduce(ext(A)) or
5301 // reduce(A).
5302 // The basic idea is that we walk down the tree to do that, finding the root
5303 // reduction instruction in InLoopReductionImmediateChains. From there we find
5304 // the pattern of mul/ext and test the cost of the entire pattern vs the cost
5305 // of the components. If the reduction cost is lower then we return it for the
5306 // reduction instruction and 0 for the other instructions in the pattern. If
5307 // it is not we return an invalid cost specifying the orignal cost method
5308 // should be used.
5309 Instruction *RetI = I;
5310 if (match(RetI, m_ZExtOrSExt(m_Value()))) {
5311 if (!RetI->hasOneUser())
5312 return std::nullopt;
5313 RetI = RetI->user_back();
5314 }
5315
5316 if (match(RetI, m_OneUse(m_Mul(m_Value(), m_Value()))) &&
5317 RetI->user_back()->getOpcode() == Instruction::Add) {
5318 RetI = RetI->user_back();
5319 }
5320
5321 // Test if the found instruction is a reduction, and if not return an invalid
5322 // cost specifying the parent to use the original cost modelling.
5323 Instruction *LastChain = InLoopReductionImmediateChains.lookup(RetI);
5324 if (!LastChain)
5325 return std::nullopt;
5326
5327 // Find the reduction this chain is a part of and calculate the basic cost of
5328 // the reduction on its own.
5329 Instruction *ReductionPhi = LastChain;
5330 while (!isa<PHINode>(ReductionPhi))
5331 ReductionPhi = InLoopReductionImmediateChains.at(ReductionPhi);
5332
5333 const RecurrenceDescriptor &RdxDesc =
5334 Legal->getRecurrenceDescriptor(cast<PHINode>(ReductionPhi));
5335
5336 InstructionCost BaseCost;
5337 RecurKind RK = RdxDesc.getRecurrenceKind();
5340 BaseCost = TTI.getMinMaxReductionCost(MinMaxID, VectorTy,
5341 RdxDesc.getFastMathFlags(), CostKind);
5342 } else {
5343 BaseCost = TTI.getArithmeticReductionCost(
5344 RdxDesc.getOpcode(), VectorTy, RdxDesc.getFastMathFlags(), CostKind);
5345 }
5346
5347 // For a call to the llvm.fmuladd intrinsic we need to add the cost of a
5348 // normal fmul instruction to the cost of the fadd reduction.
5349 if (RK == RecurKind::FMulAdd)
5350 BaseCost +=
5351 TTI.getArithmeticInstrCost(Instruction::FMul, VectorTy, CostKind);
5352
5353 // If we're using ordered reductions then we can just return the base cost
5354 // here, since getArithmeticReductionCost calculates the full ordered
5355 // reduction cost when FP reassociation is not allowed.
5356 if (useOrderedReductions(RdxDesc))
5357 return BaseCost;
5358
5359 // Get the operand that was not the reduction chain and match it to one of the
5360 // patterns, returning the better cost if it is found.
5361 Instruction *RedOp = RetI->getOperand(1) == LastChain
5364
5365 VectorTy = VectorType::get(I->getOperand(0)->getType(), VectorTy);
5366
5367 Instruction *Op0, *Op1;
5368 if (RedOp && RdxDesc.getOpcode() == Instruction::Add &&
5369 match(RedOp,
5371 match(Op0, m_ZExtOrSExt(m_Value())) &&
5372 Op0->getOpcode() == Op1->getOpcode() &&
5373 Op0->getOperand(0)->getType() == Op1->getOperand(0)->getType() &&
5374 !TheLoop->isLoopInvariant(Op0) && !TheLoop->isLoopInvariant(Op1) &&
5375 (Op0->getOpcode() == RedOp->getOpcode() || Op0 == Op1)) {
5376
5377 // Matched reduce.add(ext(mul(ext(A), ext(B)))
5378 // Note that the extend opcodes need to all match, or if A==B they will have
5379 // been converted to zext(mul(sext(A), sext(A))) as it is known positive,
5380 // which is equally fine.
5381 bool IsUnsigned = isa<ZExtInst>(Op0);
5382 auto *ExtType = VectorType::get(Op0->getOperand(0)->getType(), VectorTy);
5383 auto *MulType = VectorType::get(Op0->getType(), VectorTy);
5384
5385 InstructionCost ExtCost =
5386 TTI.getCastInstrCost(Op0->getOpcode(), MulType, ExtType,
5388 InstructionCost MulCost =
5389 TTI.getArithmeticInstrCost(Instruction::Mul, MulType, CostKind);
5390 InstructionCost Ext2Cost =
5391 TTI.getCastInstrCost(RedOp->getOpcode(), VectorTy, MulType,
5393
5394 InstructionCost RedCost = TTI.getMulAccReductionCost(
5395 IsUnsigned, RdxDesc.getOpcode(), RdxDesc.getRecurrenceType(), ExtType,
5396 CostKind);
5397
5398 if (RedCost.isValid() &&
5399 RedCost < ExtCost * 2 + MulCost + Ext2Cost + BaseCost)
5400 return I == RetI ? RedCost : 0;
5401 } else if (RedOp && match(RedOp, m_ZExtOrSExt(m_Value())) &&
5402 !TheLoop->isLoopInvariant(RedOp)) {
5403 // Matched reduce(ext(A))
5404 bool IsUnsigned = isa<ZExtInst>(RedOp);
5405 auto *ExtType = VectorType::get(RedOp->getOperand(0)->getType(), VectorTy);
5406 InstructionCost RedCost = TTI.getExtendedReductionCost(
5407 RdxDesc.getOpcode(), IsUnsigned, RdxDesc.getRecurrenceType(), ExtType,
5408 RdxDesc.getFastMathFlags(), CostKind);
5409
5410 InstructionCost ExtCost =
5411 TTI.getCastInstrCost(RedOp->getOpcode(), VectorTy, ExtType,
5413 if (RedCost.isValid() && RedCost < BaseCost + ExtCost)
5414 return I == RetI ? RedCost : 0;
5415 } else if (RedOp && RdxDesc.getOpcode() == Instruction::Add &&
5416 match(RedOp, m_Mul(m_Instruction(Op0), m_Instruction(Op1)))) {
5417 if (match(Op0, m_ZExtOrSExt(m_Value())) &&
5418 Op0->getOpcode() == Op1->getOpcode() &&
5419 !TheLoop->isLoopInvariant(Op0) && !TheLoop->isLoopInvariant(Op1)) {
5420 bool IsUnsigned = isa<ZExtInst>(Op0);
5421 Type *Op0Ty = Op0->getOperand(0)->getType();
5422 Type *Op1Ty = Op1->getOperand(0)->getType();
5423 Type *LargestOpTy =
5424 Op0Ty->getIntegerBitWidth() < Op1Ty->getIntegerBitWidth() ? Op1Ty
5425 : Op0Ty;
5426 auto *ExtType = VectorType::get(LargestOpTy, VectorTy);
5427
5428 // Matched reduce.add(mul(ext(A), ext(B))), where the two ext may be of
5429 // different sizes. We take the largest type as the ext to reduce, and add
5430 // the remaining cost as, for example reduce(mul(ext(ext(A)), ext(B))).
5431 InstructionCost ExtCost0 = TTI.getCastInstrCost(
5432 Op0->getOpcode(), VectorTy, VectorType::get(Op0Ty, VectorTy),
5434 InstructionCost ExtCost1 = TTI.getCastInstrCost(
5435 Op1->getOpcode(), VectorTy, VectorType::get(Op1Ty, VectorTy),
5437 InstructionCost MulCost =
5438 TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy, CostKind);
5439
5440 InstructionCost RedCost = TTI.getMulAccReductionCost(
5441 IsUnsigned, RdxDesc.getOpcode(), RdxDesc.getRecurrenceType(), ExtType,
5442 CostKind);
5443 InstructionCost ExtraExtCost = 0;
5444 if (Op0Ty != LargestOpTy || Op1Ty != LargestOpTy) {
5445 Instruction *ExtraExtOp = (Op0Ty != LargestOpTy) ? Op0 : Op1;
5446 ExtraExtCost = TTI.getCastInstrCost(
5447 ExtraExtOp->getOpcode(), ExtType,
5448 VectorType::get(ExtraExtOp->getOperand(0)->getType(), VectorTy),
5450 }
5451
5452 if (RedCost.isValid() &&
5453 (RedCost + ExtraExtCost) < (ExtCost0 + ExtCost1 + MulCost + BaseCost))
5454 return I == RetI ? RedCost : 0;
5455 } else if (!match(I, m_ZExtOrSExt(m_Value()))) {
5456 // Matched reduce.add(mul())
5457 InstructionCost MulCost =
5458 TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy, CostKind);
5459
5460 InstructionCost RedCost = TTI.getMulAccReductionCost(
5461 true, RdxDesc.getOpcode(), RdxDesc.getRecurrenceType(), VectorTy,
5462 CostKind);
5463
5464 if (RedCost.isValid() && RedCost < MulCost + BaseCost)
5465 return I == RetI ? RedCost : 0;
5466 }
5467 }
5468
5469 return I == RetI ? std::optional<InstructionCost>(BaseCost) : std::nullopt;
5470}
5471
5473LoopVectorizationCostModel::getMemoryInstructionCost(Instruction *I,
5474 ElementCount VF) {
5475 // Calculate scalar cost only. Vectorization cost should be ready at this
5476 // moment.
5477 if (VF.isScalar()) {
5478 Type *ValTy = getLoadStoreType(I);
5480 const Align Alignment = getLoadStoreAlignment(I);
5481 unsigned AS = getLoadStoreAddressSpace(I);
5482
5483 TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(I->getOperand(0));
5484 return TTI.getAddressComputationCost(PtrTy, nullptr, nullptr, CostKind) +
5485 TTI.getMemoryOpCost(I->getOpcode(), ValTy, Alignment, AS, CostKind,
5486 OpInfo, I);
5487 }
5488 return getWideningCost(I, VF);
5489}
5490
5492LoopVectorizationCostModel::getScalarizationOverhead(Instruction *I,
5493 ElementCount VF) const {
5494
5495 // There is no mechanism yet to create a scalable scalarization loop,
5496 // so this is currently Invalid.
5497 if (VF.isScalable())
5498 return InstructionCost::getInvalid();
5499
5500 if (VF.isScalar())
5501 return 0;
5502
5504 Type *RetTy = toVectorizedTy(I->getType(), VF);
5505 if (!RetTy->isVoidTy() &&
5507
5508 for (Type *VectorTy : getContainedTypes(RetTy)) {
5511 /*Insert=*/true,
5512 /*Extract=*/false, CostKind);
5513 }
5514 }
5515
5516 // Some targets keep addresses scalar.
5518 return Cost;
5519
5520 // Some targets support efficient element stores.
5522 return Cost;
5523
5524 // Collect operands to consider.
5525 CallInst *CI = dyn_cast<CallInst>(I);
5526 Instruction::op_range Ops = CI ? CI->args() : I->operands();
5527
5528 // Skip operands that do not require extraction/scalarization and do not incur
5529 // any overhead.
5531 for (auto *V : filterExtractingOperands(Ops, VF))
5532 Tys.push_back(maybeVectorizeType(V->getType(), VF));
5534}
5535
5537 if (VF.isScalar())
5538 return;
5539 NumPredStores = 0;
5540 for (BasicBlock *BB : TheLoop->blocks()) {
5541 // For each instruction in the old loop.
5542 for (Instruction &I : *BB) {
5544 if (!Ptr)
5545 continue;
5546
5547 // TODO: We should generate better code and update the cost model for
5548 // predicated uniform stores. Today they are treated as any other
5549 // predicated store (see added test cases in
5550 // invariant-store-vectorization.ll).
5552 NumPredStores++;
5553
5554 if (Legal->isUniformMemOp(I, VF)) {
5555 auto IsLegalToScalarize = [&]() {
5556 if (!VF.isScalable())
5557 // Scalarization of fixed length vectors "just works".
5558 return true;
5559
5560 // We have dedicated lowering for unpredicated uniform loads and
5561 // stores. Note that even with tail folding we know that at least
5562 // one lane is active (i.e. generalized predication is not possible
5563 // here), and the logic below depends on this fact.
5564 if (!foldTailByMasking())
5565 return true;
5566
5567 // For scalable vectors, a uniform memop load is always
5568 // uniform-by-parts and we know how to scalarize that.
5569 if (isa<LoadInst>(I))
5570 return true;
5571
5572 // A uniform store isn't neccessarily uniform-by-part
5573 // and we can't assume scalarization.
5574 auto &SI = cast<StoreInst>(I);
5575 return TheLoop->isLoopInvariant(SI.getValueOperand());
5576 };
5577
5578 const InstructionCost GatherScatterCost =
5580 getGatherScatterCost(&I, VF) : InstructionCost::getInvalid();
5581
5582 // Load: Scalar load + broadcast
5583 // Store: Scalar store + isLoopInvariantStoreValue ? 0 : extract
5584 // FIXME: This cost is a significant under-estimate for tail folded
5585 // memory ops.
5586 const InstructionCost ScalarizationCost =
5587 IsLegalToScalarize() ? getUniformMemOpCost(&I, VF)
5589
5590 // Choose better solution for the current VF, Note that Invalid
5591 // costs compare as maximumal large. If both are invalid, we get
5592 // scalable invalid which signals a failure and a vectorization abort.
5593 if (GatherScatterCost < ScalarizationCost)
5594 setWideningDecision(&I, VF, CM_GatherScatter, GatherScatterCost);
5595 else
5596 setWideningDecision(&I, VF, CM_Scalarize, ScalarizationCost);
5597 continue;
5598 }
5599
5600 // We assume that widening is the best solution when possible.
5601 if (memoryInstructionCanBeWidened(&I, VF)) {
5602 InstructionCost Cost = getConsecutiveMemOpCost(&I, VF);
5603 int ConsecutiveStride = Legal->isConsecutivePtr(
5605 assert((ConsecutiveStride == 1 || ConsecutiveStride == -1) &&
5606 "Expected consecutive stride.");
5607 InstWidening Decision =
5608 ConsecutiveStride == 1 ? CM_Widen : CM_Widen_Reverse;
5609 setWideningDecision(&I, VF, Decision, Cost);
5610 continue;
5611 }
5612
5613 // Choose between Interleaving, Gather/Scatter or Scalarization.
5615 unsigned NumAccesses = 1;
5616 if (isAccessInterleaved(&I)) {
5617 const auto *Group = getInterleavedAccessGroup(&I);
5618 assert(Group && "Fail to get an interleaved access group.");
5619
5620 // Make one decision for the whole group.
5621 if (getWideningDecision(&I, VF) != CM_Unknown)
5622 continue;
5623
5624 NumAccesses = Group->getNumMembers();
5626 InterleaveCost = getInterleaveGroupCost(&I, VF);
5627 }
5628
5629 InstructionCost GatherScatterCost =
5631 ? getGatherScatterCost(&I, VF) * NumAccesses
5633
5634 InstructionCost ScalarizationCost =
5635 getMemInstScalarizationCost(&I, VF) * NumAccesses;
5636
5637 // Choose better solution for the current VF,
5638 // write down this decision and use it during vectorization.
5640 InstWidening Decision;
5641 if (InterleaveCost <= GatherScatterCost &&
5642 InterleaveCost < ScalarizationCost) {
5643 Decision = CM_Interleave;
5644 Cost = InterleaveCost;
5645 } else if (GatherScatterCost < ScalarizationCost) {
5646 Decision = CM_GatherScatter;
5647 Cost = GatherScatterCost;
5648 } else {
5649 Decision = CM_Scalarize;
5650 Cost = ScalarizationCost;
5651 }
5652 // If the instructions belongs to an interleave group, the whole group
5653 // receives the same decision. The whole group receives the cost, but
5654 // the cost will actually be assigned to one instruction.
5655 if (const auto *Group = getInterleavedAccessGroup(&I)) {
5656 if (Decision == CM_Scalarize) {
5657 for (unsigned Idx = 0; Idx < Group->getFactor(); ++Idx) {
5658 if (auto *I = Group->getMember(Idx)) {
5659 setWideningDecision(I, VF, Decision,
5660 getMemInstScalarizationCost(I, VF));
5661 }
5662 }
5663 } else {
5664 setWideningDecision(Group, VF, Decision, Cost);
5665 }
5666 } else
5667 setWideningDecision(&I, VF, Decision, Cost);
5668 }
5669 }
5670
5671 // Make sure that any load of address and any other address computation
5672 // remains scalar unless there is gather/scatter support. This avoids
5673 // inevitable extracts into address registers, and also has the benefit of
5674 // activating LSR more, since that pass can't optimize vectorized
5675 // addresses.
5676 if (TTI.prefersVectorizedAddressing())
5677 return;
5678
5679 // Start with all scalar pointer uses.
5681 for (BasicBlock *BB : TheLoop->blocks())
5682 for (Instruction &I : *BB) {
5683 Instruction *PtrDef =
5685 if (PtrDef && TheLoop->contains(PtrDef) &&
5687 AddrDefs.insert(PtrDef);
5688 }
5689
5690 // Add all instructions used to generate the addresses.
5692 append_range(Worklist, AddrDefs);
5693 while (!Worklist.empty()) {
5694 Instruction *I = Worklist.pop_back_val();
5695 for (auto &Op : I->operands())
5696 if (auto *InstOp = dyn_cast<Instruction>(Op))
5697 if ((InstOp->getParent() == I->getParent()) && !isa<PHINode>(InstOp) &&
5698 AddrDefs.insert(InstOp).second)
5699 Worklist.push_back(InstOp);
5700 }
5701
5702 for (auto *I : AddrDefs) {
5703 if (isa<LoadInst>(I)) {
5704 // Setting the desired widening decision should ideally be handled in
5705 // by cost functions, but since this involves the task of finding out
5706 // if the loaded register is involved in an address computation, it is
5707 // instead changed here when we know this is the case.
5708 InstWidening Decision = getWideningDecision(I, VF);
5709 if (Decision == CM_Widen || Decision == CM_Widen_Reverse ||
5710 (!isPredicatedInst(I) && !Legal->isUniformMemOp(*I, VF) &&
5711 Decision == CM_Scalarize))
5712 // Scalarize a widened load of address or update the cost of a scalar
5713 // load of an address.
5715 I, VF, CM_Scalarize,
5716 (VF.getKnownMinValue() *
5717 getMemoryInstructionCost(I, ElementCount::getFixed(1))));
5718 else if (const auto *Group = getInterleavedAccessGroup(I)) {
5719 // Scalarize an interleave group of address loads.
5720 for (unsigned I = 0; I < Group->getFactor(); ++I) {
5721 if (Instruction *Member = Group->getMember(I))
5723 Member, VF, CM_Scalarize,
5724 (VF.getKnownMinValue() *
5725 getMemoryInstructionCost(Member, ElementCount::getFixed(1))));
5726 }
5727 }
5728 } else {
5729 // Cannot scalarize fixed-order recurrence phis at the moment.
5730 if (isa<PHINode>(I) && Legal->isFixedOrderRecurrence(cast<PHINode>(I)))
5731 continue;
5732
5733 // Make sure I gets scalarized and a cost estimate without
5734 // scalarization overhead.
5735 ForcedScalars[VF].insert(I);
5736 }
5737 }
5738}
5739
5741 assert(!VF.isScalar() &&
5742 "Trying to set a vectorization decision for a scalar VF");
5743
5744 auto ForcedScalar = ForcedScalars.find(VF);
5745 for (BasicBlock *BB : TheLoop->blocks()) {
5746 // For each instruction in the old loop.
5747 for (Instruction &I : *BB) {
5749
5750 if (!CI)
5751 continue;
5752
5756 Function *ScalarFunc = CI->getCalledFunction();
5757 Type *ScalarRetTy = CI->getType();
5758 SmallVector<Type *, 4> Tys, ScalarTys;
5759 for (auto &ArgOp : CI->args())
5760 ScalarTys.push_back(ArgOp->getType());
5761
5762 // Estimate cost of scalarized vector call. The source operands are
5763 // assumed to be vectors, so we need to extract individual elements from
5764 // there, execute VF scalar calls, and then gather the result into the
5765 // vector return value.
5766 if (VF.isFixed()) {
5767 InstructionCost ScalarCallCost =
5768 TTI.getCallInstrCost(ScalarFunc, ScalarRetTy, ScalarTys, CostKind);
5769
5770 // Compute costs of unpacking argument values for the scalar calls and
5771 // packing the return values to a vector.
5772 InstructionCost ScalarizationCost = getScalarizationOverhead(CI, VF);
5773 ScalarCost = ScalarCallCost * VF.getKnownMinValue() + ScalarizationCost;
5774 } else {
5775 // There is no point attempting to calculate the scalar cost for a
5776 // scalable VF as we know it will be Invalid.
5778 "Unexpected valid cost for scalarizing scalable vectors");
5779 ScalarCost = InstructionCost::getInvalid();
5780 }
5781
5782 // Honor ForcedScalars and UniformAfterVectorization decisions.
5783 // TODO: For calls, it might still be more profitable to widen. Use
5784 // VPlan-based cost model to compare different options.
5785 if (VF.isVector() && ((ForcedScalar != ForcedScalars.end() &&
5786 ForcedScalar->second.contains(CI)) ||
5787 isUniformAfterVectorization(CI, VF))) {
5788 setCallWideningDecision(CI, VF, CM_Scalarize, nullptr,
5789 Intrinsic::not_intrinsic, std::nullopt,
5790 ScalarCost);
5791 continue;
5792 }
5793
5794 bool MaskRequired = Legal->isMaskRequired(CI);
5795 // Compute corresponding vector type for return value and arguments.
5796 Type *RetTy = toVectorizedTy(ScalarRetTy, VF);
5797 for (Type *ScalarTy : ScalarTys)
5798 Tys.push_back(toVectorizedTy(ScalarTy, VF));
5799
5800 // An in-loop reduction using an fmuladd intrinsic is a special case;
5801 // we don't want the normal cost for that intrinsic.
5803 if (auto RedCost = getReductionPatternCost(CI, VF, RetTy)) {
5806 std::nullopt, *RedCost);
5807 continue;
5808 }
5809
5810 // Find the cost of vectorizing the call, if we can find a suitable
5811 // vector variant of the function.
5812 VFInfo FuncInfo;
5813 Function *VecFunc = nullptr;
5814 // Search through any available variants for one we can use at this VF.
5815 for (VFInfo &Info : VFDatabase::getMappings(*CI)) {
5816 // Must match requested VF.
5817 if (Info.Shape.VF != VF)
5818 continue;
5819
5820 // Must take a mask argument if one is required
5821 if (MaskRequired && !Info.isMasked())
5822 continue;
5823
5824 // Check that all parameter kinds are supported
5825 bool ParamsOk = true;
5826 for (VFParameter Param : Info.Shape.Parameters) {
5827 switch (Param.ParamKind) {
5829 break;
5831 Value *ScalarParam = CI->getArgOperand(Param.ParamPos);
5832 // Make sure the scalar parameter in the loop is invariant.
5833 if (!PSE.getSE()->isLoopInvariant(PSE.getSCEV(ScalarParam),
5834 TheLoop))
5835 ParamsOk = false;
5836 break;
5837 }
5839 Value *ScalarParam = CI->getArgOperand(Param.ParamPos);
5840 // Find the stride for the scalar parameter in this loop and see if
5841 // it matches the stride for the variant.
5842 // TODO: do we need to figure out the cost of an extract to get the
5843 // first lane? Or do we hope that it will be folded away?
5844 ScalarEvolution *SE = PSE.getSE();
5845 if (!match(SE->getSCEV(ScalarParam),
5847 m_SCEV(), m_scev_SpecificSInt(Param.LinearStepOrPos),
5849 ParamsOk = false;
5850 break;
5851 }
5853 break;
5854 default:
5855 ParamsOk = false;
5856 break;
5857 }
5858 }
5859
5860 if (!ParamsOk)
5861 continue;
5862
5863 // Found a suitable candidate, stop here.
5864 VecFunc = CI->getModule()->getFunction(Info.VectorName);
5865 FuncInfo = Info;
5866 break;
5867 }
5868
5869 if (TLI && VecFunc && !CI->isNoBuiltin())
5870 VectorCost = TTI.getCallInstrCost(nullptr, RetTy, Tys, CostKind);
5871
5872 // Find the cost of an intrinsic; some targets may have instructions that
5873 // perform the operation without needing an actual call.
5875 if (IID != Intrinsic::not_intrinsic)
5877
5878 InstructionCost Cost = ScalarCost;
5879 InstWidening Decision = CM_Scalarize;
5880
5881 if (VectorCost <= Cost) {
5882 Cost = VectorCost;
5883 Decision = CM_VectorCall;
5884 }
5885
5886 if (IntrinsicCost <= Cost) {
5888 Decision = CM_IntrinsicCall;
5889 }
5890
5891 setCallWideningDecision(CI, VF, Decision, VecFunc, IID,
5893 }
5894 }
5895}
5896
5898 if (!Legal->isInvariant(Op))
5899 return false;
5900 // Consider Op invariant, if it or its operands aren't predicated
5901 // instruction in the loop. In that case, it is not trivially hoistable.
5902 auto *OpI = dyn_cast<Instruction>(Op);
5903 return !OpI || !TheLoop->contains(OpI) ||
5904 (!isPredicatedInst(OpI) &&
5905 (!isa<PHINode>(OpI) || OpI->getParent() != TheLoop->getHeader()) &&
5906 all_of(OpI->operands(),
5907 [this](Value *Op) { return shouldConsiderInvariant(Op); }));
5908}
5909
5912 ElementCount VF) {
5913 // If we know that this instruction will remain uniform, check the cost of
5914 // the scalar version.
5916 VF = ElementCount::getFixed(1);
5917
5918 if (VF.isVector() && isProfitableToScalarize(I, VF))
5919 return InstsToScalarize[VF][I];
5920
5921 // Forced scalars do not have any scalarization overhead.
5922 auto ForcedScalar = ForcedScalars.find(VF);
5923 if (VF.isVector() && ForcedScalar != ForcedScalars.end()) {
5924 auto InstSet = ForcedScalar->second;
5925 if (InstSet.count(I))
5927 VF.getKnownMinValue();
5928 }
5929
5930 Type *RetTy = I->getType();
5932 RetTy = IntegerType::get(RetTy->getContext(), MinBWs[I]);
5933 auto *SE = PSE.getSE();
5934
5935 Type *VectorTy;
5936 if (isScalarAfterVectorization(I, VF)) {
5937 [[maybe_unused]] auto HasSingleCopyAfterVectorization =
5938 [this](Instruction *I, ElementCount VF) -> bool {
5939 if (VF.isScalar())
5940 return true;
5941
5942 auto Scalarized = InstsToScalarize.find(VF);
5943 assert(Scalarized != InstsToScalarize.end() &&
5944 "VF not yet analyzed for scalarization profitability");
5945 return !Scalarized->second.count(I) &&
5946 llvm::all_of(I->users(), [&](User *U) {
5947 auto *UI = cast<Instruction>(U);
5948 return !Scalarized->second.count(UI);
5949 });
5950 };
5951
5952 // With the exception of GEPs and PHIs, after scalarization there should
5953 // only be one copy of the instruction generated in the loop. This is
5954 // because the VF is either 1, or any instructions that need scalarizing
5955 // have already been dealt with by the time we get here. As a result,
5956 // it means we don't have to multiply the instruction cost by VF.
5957 assert(I->getOpcode() == Instruction::GetElementPtr ||
5958 I->getOpcode() == Instruction::PHI ||
5959 (I->getOpcode() == Instruction::BitCast &&
5960 I->getType()->isPointerTy()) ||
5961 HasSingleCopyAfterVectorization(I, VF));
5962 VectorTy = RetTy;
5963 } else
5964 VectorTy = toVectorizedTy(RetTy, VF);
5965
5966 if (VF.isVector() && VectorTy->isVectorTy() &&
5967 !TTI.getNumberOfParts(VectorTy))
5969
5970 // TODO: We need to estimate the cost of intrinsic calls.
5971 switch (I->getOpcode()) {
5972 case Instruction::GetElementPtr:
5973 // We mark this instruction as zero-cost because the cost of GEPs in
5974 // vectorized code depends on whether the corresponding memory instruction
5975 // is scalarized or not. Therefore, we handle GEPs with the memory
5976 // instruction cost.
5977 return 0;
5978 case Instruction::Br: {
5979 // In cases of scalarized and predicated instructions, there will be VF
5980 // predicated blocks in the vectorized loop. Each branch around these
5981 // blocks requires also an extract of its vector compare i1 element.
5982 // Note that the conditional branch from the loop latch will be replaced by
5983 // a single branch controlling the loop, so there is no extra overhead from
5984 // scalarization.
5985 bool ScalarPredicatedBB = false;
5987 if (VF.isVector() && BI->isConditional() &&
5988 (PredicatedBBsAfterVectorization[VF].count(BI->getSuccessor(0)) ||
5989 PredicatedBBsAfterVectorization[VF].count(BI->getSuccessor(1))) &&
5990 BI->getParent() != TheLoop->getLoopLatch())
5991 ScalarPredicatedBB = true;
5992
5993 if (ScalarPredicatedBB) {
5994 // Not possible to scalarize scalable vector with predicated instructions.
5995 if (VF.isScalable())
5997 // Return cost for branches around scalarized and predicated blocks.
5998 auto *VecI1Ty =
6000 return (
6001 TTI.getScalarizationOverhead(
6002 VecI1Ty, APInt::getAllOnes(VF.getFixedValue()),
6003 /*Insert*/ false, /*Extract*/ true, CostKind) +
6004 (TTI.getCFInstrCost(Instruction::Br, CostKind) * VF.getFixedValue()));
6005 }
6006
6007 if (I->getParent() == TheLoop->getLoopLatch() || VF.isScalar())
6008 // The back-edge branch will remain, as will all scalar branches.
6009 return TTI.getCFInstrCost(Instruction::Br, CostKind);
6010
6011 // This branch will be eliminated by if-conversion.
6012 return 0;
6013 // Note: We currently assume zero cost for an unconditional branch inside
6014 // a predicated block since it will become a fall-through, although we
6015 // may decide in the future to call TTI for all branches.
6016 }
6017 case Instruction::Switch: {
6018 if (VF.isScalar())
6019 return TTI.getCFInstrCost(Instruction::Switch, CostKind);
6020 auto *Switch = cast<SwitchInst>(I);
6021 return Switch->getNumCases() *
6022 TTI.getCmpSelInstrCost(
6023 Instruction::ICmp,
6024 toVectorTy(Switch->getCondition()->getType(), VF),
6025 toVectorTy(Type::getInt1Ty(I->getContext()), VF),
6027 }
6028 case Instruction::PHI: {
6029 auto *Phi = cast<PHINode>(I);
6030
6031 // First-order recurrences are replaced by vector shuffles inside the loop.
6032 if (VF.isVector() && Legal->isFixedOrderRecurrence(Phi)) {
6034 std::iota(Mask.begin(), Mask.end(), VF.getKnownMinValue() - 1);
6035 return TTI.getShuffleCost(TargetTransformInfo::SK_Splice,
6036 cast<VectorType>(VectorTy),
6037 cast<VectorType>(VectorTy), Mask, CostKind,
6038 VF.getKnownMinValue() - 1);
6039 }
6040
6041 // Phi nodes in non-header blocks (not inductions, reductions, etc.) are
6042 // converted into select instructions. We require N - 1 selects per phi
6043 // node, where N is the number of incoming values.
6044 if (VF.isVector() && Phi->getParent() != TheLoop->getHeader()) {
6045 Type *ResultTy = Phi->getType();
6046
6047 // All instructions in an Any-of reduction chain are narrowed to bool.
6048 // Check if that is the case for this phi node.
6049 auto *HeaderUser = cast_if_present<PHINode>(
6050 find_singleton<User>(Phi->users(), [this](User *U, bool) -> User * {
6051 auto *Phi = dyn_cast<PHINode>(U);
6052 if (Phi && Phi->getParent() == TheLoop->getHeader())
6053 return Phi;
6054 return nullptr;
6055 }));
6056 if (HeaderUser) {
6057 auto &ReductionVars = Legal->getReductionVars();
6058 auto Iter = ReductionVars.find(HeaderUser);
6059 if (Iter != ReductionVars.end() &&
6061 Iter->second.getRecurrenceKind()))
6062 ResultTy = Type::getInt1Ty(Phi->getContext());
6063 }
6064 return (Phi->getNumIncomingValues() - 1) *
6065 TTI.getCmpSelInstrCost(
6066 Instruction::Select, toVectorTy(ResultTy, VF),
6067 toVectorTy(Type::getInt1Ty(Phi->getContext()), VF),
6069 }
6070
6071 // When tail folding with EVL, if the phi is part of an out of loop
6072 // reduction then it will be transformed into a wide vp_merge.
6073 if (VF.isVector() && foldTailWithEVL() &&
6074 Legal->getReductionVars().contains(Phi) && !isInLoopReduction(Phi)) {
6076 Intrinsic::vp_merge, toVectorTy(Phi->getType(), VF),
6077 {toVectorTy(Type::getInt1Ty(Phi->getContext()), VF)});
6078 return TTI.getIntrinsicInstrCost(ICA, CostKind);
6079 }
6080
6081 return TTI.getCFInstrCost(Instruction::PHI, CostKind);
6082 }
6083 case Instruction::UDiv:
6084 case Instruction::SDiv:
6085 case Instruction::URem:
6086 case Instruction::SRem:
6087 if (VF.isVector() && isPredicatedInst(I)) {
6088 const auto [ScalarCost, SafeDivisorCost] = getDivRemSpeculationCost(I, VF);
6089 return isDivRemScalarWithPredication(ScalarCost, SafeDivisorCost) ?
6090 ScalarCost : SafeDivisorCost;
6091 }
6092 // We've proven all lanes safe to speculate, fall through.
6093 [[fallthrough]];
6094 case Instruction::Add:
6095 case Instruction::Sub: {
6096 auto Info = Legal->getHistogramInfo(I);
6097 if (Info && VF.isVector()) {
6098 const HistogramInfo *HGram = Info.value();
6099 // Assume that a non-constant update value (or a constant != 1) requires
6100 // a multiply, and add that into the cost.
6102 ConstantInt *RHS = dyn_cast<ConstantInt>(I->getOperand(1));
6103 if (!RHS || RHS->getZExtValue() != 1)
6104 MulCost =
6105 TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy, CostKind);
6106
6107 // Find the cost of the histogram operation itself.
6108 Type *PtrTy = VectorType::get(HGram->Load->getPointerOperandType(), VF);
6109 Type *ScalarTy = I->getType();
6110 Type *MaskTy = VectorType::get(Type::getInt1Ty(I->getContext()), VF);
6111 IntrinsicCostAttributes ICA(Intrinsic::experimental_vector_histogram_add,
6112 Type::getVoidTy(I->getContext()),
6113 {PtrTy, ScalarTy, MaskTy});
6114
6115 // Add the costs together with the add/sub operation.
6116 return TTI.getIntrinsicInstrCost(ICA, CostKind) + MulCost +
6117 TTI.getArithmeticInstrCost(I->getOpcode(), VectorTy, CostKind);
6118 }
6119 [[fallthrough]];
6120 }
6121 case Instruction::FAdd:
6122 case Instruction::FSub:
6123 case Instruction::Mul:
6124 case Instruction::FMul:
6125 case Instruction::FDiv:
6126 case Instruction::FRem:
6127 case Instruction::Shl:
6128 case Instruction::LShr:
6129 case Instruction::AShr:
6130 case Instruction::And:
6131 case Instruction::Or:
6132 case Instruction::Xor: {
6133 // If we're speculating on the stride being 1, the multiplication may
6134 // fold away. We can generalize this for all operations using the notion
6135 // of neutral elements. (TODO)
6136 if (I->getOpcode() == Instruction::Mul &&
6137 ((TheLoop->isLoopInvariant(I->getOperand(0)) &&
6138 PSE.getSCEV(I->getOperand(0))->isOne()) ||
6139 (TheLoop->isLoopInvariant(I->getOperand(1)) &&
6140 PSE.getSCEV(I->getOperand(1))->isOne())))
6141 return 0;
6142
6143 // Detect reduction patterns
6144 if (auto RedCost = getReductionPatternCost(I, VF, VectorTy))
6145 return *RedCost;
6146
6147 // Certain instructions can be cheaper to vectorize if they have a constant
6148 // second vector operand. One example of this are shifts on x86.
6149 Value *Op2 = I->getOperand(1);
6150 if (!isa<Constant>(Op2) && TheLoop->isLoopInvariant(Op2) &&
6151 PSE.getSE()->isSCEVable(Op2->getType()) &&
6152 isa<SCEVConstant>(PSE.getSCEV(Op2))) {
6153 Op2 = cast<SCEVConstant>(PSE.getSCEV(Op2))->getValue();
6154 }
6155 auto Op2Info = TTI.getOperandInfo(Op2);
6156 if (Op2Info.Kind == TargetTransformInfo::OK_AnyValue &&
6159
6160 SmallVector<const Value *, 4> Operands(I->operand_values());
6161 return TTI.getArithmeticInstrCost(
6162 I->getOpcode(), VectorTy, CostKind,
6163 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
6164 Op2Info, Operands, I, TLI);
6165 }
6166 case Instruction::FNeg: {
6167 return TTI.getArithmeticInstrCost(
6168 I->getOpcode(), VectorTy, CostKind,
6169 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
6170 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
6171 I->getOperand(0), I);
6172 }
6173 case Instruction::Select: {
6175 const SCEV *CondSCEV = SE->getSCEV(SI->getCondition());
6176 bool ScalarCond = (SE->isLoopInvariant(CondSCEV, TheLoop));
6177
6178 const Value *Op0, *Op1;
6179 using namespace llvm::PatternMatch;
6180 if (!ScalarCond && (match(I, m_LogicalAnd(m_Value(Op0), m_Value(Op1))) ||
6181 match(I, m_LogicalOr(m_Value(Op0), m_Value(Op1))))) {
6182 // select x, y, false --> x & y
6183 // select x, true, y --> x | y
6184 const auto [Op1VK, Op1VP] = TTI::getOperandInfo(Op0);
6185 const auto [Op2VK, Op2VP] = TTI::getOperandInfo(Op1);
6186 assert(Op0->getType()->getScalarSizeInBits() == 1 &&
6187 Op1->getType()->getScalarSizeInBits() == 1);
6188
6189 return TTI.getArithmeticInstrCost(
6190 match(I, m_LogicalOr()) ? Instruction::Or : Instruction::And,
6191 VectorTy, CostKind, {Op1VK, Op1VP}, {Op2VK, Op2VP}, {Op0, Op1}, I);
6192 }
6193
6194 Type *CondTy = SI->getCondition()->getType();
6195 if (!ScalarCond)
6196 CondTy = VectorType::get(CondTy, VF);
6197
6199 if (auto *Cmp = dyn_cast<CmpInst>(SI->getCondition()))
6200 Pred = Cmp->getPredicate();
6201 return TTI.getCmpSelInstrCost(I->getOpcode(), VectorTy, CondTy, Pred,
6202 CostKind, {TTI::OK_AnyValue, TTI::OP_None},
6203 {TTI::OK_AnyValue, TTI::OP_None}, I);
6204 }
6205 case Instruction::ICmp:
6206 case Instruction::FCmp: {
6207 Type *ValTy = I->getOperand(0)->getType();
6208
6210 [[maybe_unused]] Instruction *Op0AsInstruction =
6211 dyn_cast<Instruction>(I->getOperand(0));
6212 assert((!canTruncateToMinimalBitwidth(Op0AsInstruction, VF) ||
6213 MinBWs[I] == MinBWs[Op0AsInstruction]) &&
6214 "if both the operand and the compare are marked for "
6215 "truncation, they must have the same bitwidth");
6216 ValTy = IntegerType::get(ValTy->getContext(), MinBWs[I]);
6217 }
6218
6219 VectorTy = toVectorTy(ValTy, VF);
6220 return TTI.getCmpSelInstrCost(
6221 I->getOpcode(), VectorTy, CmpInst::makeCmpResultType(VectorTy),
6222 cast<CmpInst>(I)->getPredicate(), CostKind,
6223 {TTI::OK_AnyValue, TTI::OP_None}, {TTI::OK_AnyValue, TTI::OP_None}, I);
6224 }
6225 case Instruction::Store:
6226 case Instruction::Load: {
6227 ElementCount Width = VF;
6228 if (Width.isVector()) {
6229 InstWidening Decision = getWideningDecision(I, Width);
6230 assert(Decision != CM_Unknown &&
6231 "CM decision should be taken at this point");
6234 if (Decision == CM_Scalarize)
6235 Width = ElementCount::getFixed(1);
6236 }
6237 VectorTy = toVectorTy(getLoadStoreType(I), Width);
6238 return getMemoryInstructionCost(I, VF);
6239 }
6240 case Instruction::BitCast:
6241 if (I->getType()->isPointerTy())
6242 return 0;
6243 [[fallthrough]];
6244 case Instruction::ZExt:
6245 case Instruction::SExt:
6246 case Instruction::FPToUI:
6247 case Instruction::FPToSI:
6248 case Instruction::FPExt:
6249 case Instruction::PtrToInt:
6250 case Instruction::IntToPtr:
6251 case Instruction::SIToFP:
6252 case Instruction::UIToFP:
6253 case Instruction::Trunc:
6254 case Instruction::FPTrunc: {
6255 // Computes the CastContextHint from a Load/Store instruction.
6256 auto ComputeCCH = [&](Instruction *I) -> TTI::CastContextHint {
6258 "Expected a load or a store!");
6259
6260 if (VF.isScalar() || !TheLoop->contains(I))
6262
6263 switch (getWideningDecision(I, VF)) {
6275 llvm_unreachable("Instr did not go through cost modelling?");
6278 llvm_unreachable_internal("Instr has invalid widening decision");
6279 }
6280
6281 llvm_unreachable("Unhandled case!");
6282 };
6283
6284 unsigned Opcode = I->getOpcode();
6286 // For Trunc, the context is the only user, which must be a StoreInst.
6287 if (Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) {
6288 if (I->hasOneUse())
6289 if (StoreInst *Store = dyn_cast<StoreInst>(*I->user_begin()))
6290 CCH = ComputeCCH(Store);
6291 }
6292 // For Z/Sext, the context is the operand, which must be a LoadInst.
6293 else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt ||
6294 Opcode == Instruction::FPExt) {
6295 if (LoadInst *Load = dyn_cast<LoadInst>(I->getOperand(0)))
6296 CCH = ComputeCCH(Load);
6297 }
6298
6299 // We optimize the truncation of induction variables having constant
6300 // integer steps. The cost of these truncations is the same as the scalar
6301 // operation.
6302 if (isOptimizableIVTruncate(I, VF)) {
6303 auto *Trunc = cast<TruncInst>(I);
6304 return TTI.getCastInstrCost(Instruction::Trunc, Trunc->getDestTy(),
6305 Trunc->getSrcTy(), CCH, CostKind, Trunc);
6306 }
6307
6308 // Detect reduction patterns
6309 if (auto RedCost = getReductionPatternCost(I, VF, VectorTy))
6310 return *RedCost;
6311
6312 Type *SrcScalarTy = I->getOperand(0)->getType();
6313 Instruction *Op0AsInstruction = dyn_cast<Instruction>(I->getOperand(0));
6314 if (canTruncateToMinimalBitwidth(Op0AsInstruction, VF))
6315 SrcScalarTy =
6316 IntegerType::get(SrcScalarTy->getContext(), MinBWs[Op0AsInstruction]);
6317 Type *SrcVecTy =
6318 VectorTy->isVectorTy() ? toVectorTy(SrcScalarTy, VF) : SrcScalarTy;
6319
6321 // If the result type is <= the source type, there will be no extend
6322 // after truncating the users to the minimal required bitwidth.
6323 if (VectorTy->getScalarSizeInBits() <= SrcVecTy->getScalarSizeInBits() &&
6324 (I->getOpcode() == Instruction::ZExt ||
6325 I->getOpcode() == Instruction::SExt))
6326 return 0;
6327 }
6328
6329 return TTI.getCastInstrCost(Opcode, VectorTy, SrcVecTy, CCH, CostKind, I);
6330 }
6331 case Instruction::Call:
6332 return getVectorCallCost(cast<CallInst>(I), VF);
6333 case Instruction::ExtractValue:
6334 return TTI.getInstructionCost(I, CostKind);
6335 case Instruction::Alloca:
6336 // We cannot easily widen alloca to a scalable alloca, as
6337 // the result would need to be a vector of pointers.
6338 if (VF.isScalable())
6340 [[fallthrough]];
6341 default:
6342 // This opcode is unknown. Assume that it is the same as 'mul'.
6343 return TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy, CostKind);
6344 } // end of switch.
6345}
6346
6348 // Ignore ephemeral values.
6350
6351 SmallVector<Value *, 4> DeadInterleavePointerOps;
6353
6354 // If a scalar epilogue is required, users outside the loop won't use
6355 // live-outs from the vector loop but from the scalar epilogue. Ignore them if
6356 // that is the case.
6357 bool RequiresScalarEpilogue = requiresScalarEpilogue(true);
6358 auto IsLiveOutDead = [this, RequiresScalarEpilogue](User *U) {
6359 return RequiresScalarEpilogue &&
6360 !TheLoop->contains(cast<Instruction>(U)->getParent());
6361 };
6362
6364 DFS.perform(LI);
6365 for (BasicBlock *BB : reverse(make_range(DFS.beginRPO(), DFS.endRPO())))
6366 for (Instruction &I : reverse(*BB)) {
6367 if (VecValuesToIgnore.contains(&I) || ValuesToIgnore.contains(&I))
6368 continue;
6369
6370 // Add instructions that would be trivially dead and are only used by
6371 // values already ignored to DeadOps to seed worklist.
6373 all_of(I.users(), [this, IsLiveOutDead](User *U) {
6374 return VecValuesToIgnore.contains(U) ||
6375 ValuesToIgnore.contains(U) || IsLiveOutDead(U);
6376 }))
6377 DeadOps.push_back(&I);
6378
6379 // For interleave groups, we only create a pointer for the start of the
6380 // interleave group. Queue up addresses of group members except the insert
6381 // position for further processing.
6382 if (isAccessInterleaved(&I)) {
6383 auto *Group = getInterleavedAccessGroup(&I);
6384 if (Group->getInsertPos() == &I)
6385 continue;
6386 Value *PointerOp = getLoadStorePointerOperand(&I);
6387 DeadInterleavePointerOps.push_back(PointerOp);
6388 }
6389
6390 // Queue branches for analysis. They are dead, if their successors only
6391 // contain dead instructions.
6392 if (auto *Br = dyn_cast<BranchInst>(&I)) {
6393 if (Br->isConditional())
6394 DeadOps.push_back(&I);
6395 }
6396 }
6397
6398 // Mark ops feeding interleave group members as free, if they are only used
6399 // by other dead computations.
6400 for (unsigned I = 0; I != DeadInterleavePointerOps.size(); ++I) {
6401 auto *Op = dyn_cast<Instruction>(DeadInterleavePointerOps[I]);
6402 if (!Op || !TheLoop->contains(Op) || any_of(Op->users(), [this](User *U) {
6403 Instruction *UI = cast<Instruction>(U);
6404 return !VecValuesToIgnore.contains(U) &&
6405 (!isAccessInterleaved(UI) ||
6406 getInterleavedAccessGroup(UI)->getInsertPos() == UI);
6407 }))
6408 continue;
6409 VecValuesToIgnore.insert(Op);
6410 append_range(DeadInterleavePointerOps, Op->operands());
6411 }
6412
6413 // Mark ops that would be trivially dead and are only used by ignored
6414 // instructions as free.
6415 BasicBlock *Header = TheLoop->getHeader();
6416
6417 // Returns true if the block contains only dead instructions. Such blocks will
6418 // be removed by VPlan-to-VPlan transforms and won't be considered by the
6419 // VPlan-based cost model, so skip them in the legacy cost-model as well.
6420 auto IsEmptyBlock = [this](BasicBlock *BB) {
6421 return all_of(*BB, [this](Instruction &I) {
6422 return ValuesToIgnore.contains(&I) || VecValuesToIgnore.contains(&I) ||
6423 (isa<BranchInst>(&I) && !cast<BranchInst>(&I)->isConditional());
6424 });
6425 };
6426 for (unsigned I = 0; I != DeadOps.size(); ++I) {
6427 auto *Op = dyn_cast<Instruction>(DeadOps[I]);
6428
6429 // Check if the branch should be considered dead.
6430 if (auto *Br = dyn_cast_or_null<BranchInst>(Op)) {
6431 BasicBlock *ThenBB = Br->getSuccessor(0);
6432 BasicBlock *ElseBB = Br->getSuccessor(1);
6433 // Don't considers branches leaving the loop for simplification.
6434 if (!TheLoop->contains(ThenBB) || !TheLoop->contains(ElseBB))
6435 continue;
6436 bool ThenEmpty = IsEmptyBlock(ThenBB);
6437 bool ElseEmpty = IsEmptyBlock(ElseBB);
6438 if ((ThenEmpty && ElseEmpty) ||
6439 (ThenEmpty && ThenBB->getSingleSuccessor() == ElseBB &&
6440 ElseBB->phis().empty()) ||
6441 (ElseEmpty && ElseBB->getSingleSuccessor() == ThenBB &&
6442 ThenBB->phis().empty())) {
6443 VecValuesToIgnore.insert(Br);
6444 DeadOps.push_back(Br->getCondition());
6445 }
6446 continue;
6447 }
6448
6449 // Skip any op that shouldn't be considered dead.
6450 if (!Op || !TheLoop->contains(Op) ||
6451 (isa<PHINode>(Op) && Op->getParent() == Header) ||
6453 any_of(Op->users(), [this, IsLiveOutDead](User *U) {
6454 return !VecValuesToIgnore.contains(U) &&
6455 !ValuesToIgnore.contains(U) && !IsLiveOutDead(U);
6456 }))
6457 continue;
6458
6459 // If all of Op's users are in ValuesToIgnore, add it to ValuesToIgnore
6460 // which applies for both scalar and vector versions. Otherwise it is only
6461 // dead in vector versions, so only add it to VecValuesToIgnore.
6462 if (all_of(Op->users(),
6463 [this](User *U) { return ValuesToIgnore.contains(U); }))
6464 ValuesToIgnore.insert(Op);
6465
6466 VecValuesToIgnore.insert(Op);
6467 append_range(DeadOps, Op->operands());
6468 }
6469
6470 // Ignore type-promoting instructions we identified during reduction
6471 // detection.
6472 for (const auto &Reduction : Legal->getReductionVars()) {
6473 const RecurrenceDescriptor &RedDes = Reduction.second;
6474 const SmallPtrSetImpl<Instruction *> &Casts = RedDes.getCastInsts();
6475 VecValuesToIgnore.insert_range(Casts);
6476 }
6477 // Ignore type-casting instructions we identified during induction
6478 // detection.
6479 for (const auto &Induction : Legal->getInductionVars()) {
6480 const InductionDescriptor &IndDes = Induction.second;
6481 const SmallVectorImpl<Instruction *> &Casts = IndDes.getCastInsts();
6482 VecValuesToIgnore.insert_range(Casts);
6483 }
6484}
6485
6487 // Avoid duplicating work finding in-loop reductions.
6488 if (!InLoopReductions.empty())
6489 return;
6490
6491 for (const auto &Reduction : Legal->getReductionVars()) {
6492 PHINode *Phi = Reduction.first;
6493 const RecurrenceDescriptor &RdxDesc = Reduction.second;
6494
6495 // We don't collect reductions that are type promoted (yet).
6496 if (RdxDesc.getRecurrenceType() != Phi->getType())
6497 continue;
6498
6499 // If the target would prefer this reduction to happen "in-loop", then we
6500 // want to record it as such.
6501 RecurKind Kind = RdxDesc.getRecurrenceKind();
6502 if (!PreferInLoopReductions && !useOrderedReductions(RdxDesc) &&
6503 !TTI.preferInLoopReduction(Kind, Phi->getType()))
6504 continue;
6505
6506 // Check that we can correctly put the reductions into the loop, by
6507 // finding the chain of operations that leads from the phi to the loop
6508 // exit value.
6509 SmallVector<Instruction *, 4> ReductionOperations =
6510 RdxDesc.getReductionOpChain(Phi, TheLoop);
6511 bool InLoop = !ReductionOperations.empty();
6512
6513 if (InLoop) {
6514 InLoopReductions.insert(Phi);
6515 // Add the elements to InLoopReductionImmediateChains for cost modelling.
6516 Instruction *LastChain = Phi;
6517 for (auto *I : ReductionOperations) {
6518 InLoopReductionImmediateChains[I] = LastChain;
6519 LastChain = I;
6520 }
6521 }
6522 LLVM_DEBUG(dbgs() << "LV: Using " << (InLoop ? "inloop" : "out of loop")
6523 << " reduction for phi: " << *Phi << "\n");
6524 }
6525}
6526
6527// This function will select a scalable VF if the target supports scalable
6528// vectors and a fixed one otherwise.
6529// TODO: we could return a pair of values that specify the max VF and
6530// min VF, to be used in `buildVPlans(MinVF, MaxVF)` instead of
6531// `buildVPlans(VF, VF)`. We cannot do it because VPLAN at the moment
6532// doesn't have a cost model that can choose which plan to execute if
6533// more than one is generated.
6536 unsigned WidestType;
6537 std::tie(std::ignore, WidestType) = CM.getSmallestAndWidestTypes();
6538
6540 TTI.enableScalableVectorization()
6543
6544 TypeSize RegSize = TTI.getRegisterBitWidth(RegKind);
6545 unsigned N = RegSize.getKnownMinValue() / WidestType;
6546 return ElementCount::get(N, RegSize.isScalable());
6547}
6548
6551 ElementCount VF = UserVF;
6552 // Outer loop handling: They may require CFG and instruction level
6553 // transformations before even evaluating whether vectorization is profitable.
6554 // Since we cannot modify the incoming IR, we need to build VPlan upfront in
6555 // the vectorization pipeline.
6556 if (!OrigLoop->isInnermost()) {
6557 // If the user doesn't provide a vectorization factor, determine a
6558 // reasonable one.
6559 if (UserVF.isZero()) {
6560 VF = determineVPlanVF(TTI, CM);
6561 LLVM_DEBUG(dbgs() << "LV: VPlan computed VF " << VF << ".\n");
6562
6563 // Make sure we have a VF > 1 for stress testing.
6564 if (VPlanBuildStressTest && (VF.isScalar() || VF.isZero())) {
6565 LLVM_DEBUG(dbgs() << "LV: VPlan stress testing: "
6566 << "overriding computed VF.\n");
6567 VF = ElementCount::getFixed(4);
6568 }
6569 } else if (UserVF.isScalable() && !TTI.supportsScalableVectors() &&
6571 LLVM_DEBUG(dbgs() << "LV: Not vectorizing. Scalable VF requested, but "
6572 << "not supported by the target.\n");
6574 "Scalable vectorization requested but not supported by the target",
6575 "the scalable user-specified vectorization width for outer-loop "
6576 "vectorization cannot be used because the target does not support "
6577 "scalable vectors.",
6578 "ScalableVFUnfeasible", ORE, OrigLoop);
6580 }
6581 assert(EnableVPlanNativePath && "VPlan-native path is not enabled.");
6583 "VF needs to be a power of two");
6584 LLVM_DEBUG(dbgs() << "LV: Using " << (!UserVF.isZero() ? "user " : "")
6585 << "VF " << VF << " to build VPlans.\n");
6586 buildVPlans(VF, VF);
6587
6588 if (VPlans.empty())
6590
6591 // For VPlan build stress testing, we bail out after VPlan construction.
6594
6595 return {VF, 0 /*Cost*/, 0 /* ScalarCost */};
6596 }
6597
6598 LLVM_DEBUG(
6599 dbgs() << "LV: Not vectorizing. Inner loops aren't supported in the "
6600 "VPlan-native path.\n");
6602}
6603
6604void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
6605 assert(OrigLoop->isInnermost() && "Inner loop expected.");
6606 CM.collectValuesToIgnore();
6607 CM.collectElementTypesForWidening();
6608
6609 FixedScalableVFPair MaxFactors = CM.computeMaxVF(UserVF, UserIC);
6610 if (!MaxFactors) // Cases that should not to be vectorized nor interleaved.
6611 return;
6612
6613 // Invalidate interleave groups if all blocks of loop will be predicated.
6614 if (CM.blockNeedsPredicationForAnyReason(OrigLoop->getHeader()) &&
6616 LLVM_DEBUG(
6617 dbgs()
6618 << "LV: Invalidate all interleaved groups due to fold-tail by masking "
6619 "which requires masked-interleaved support.\n");
6620 if (CM.InterleaveInfo.invalidateGroups())
6621 // Invalidating interleave groups also requires invalidating all decisions
6622 // based on them, which includes widening decisions and uniform and scalar
6623 // values.
6624 CM.invalidateCostModelingDecisions();
6625 }
6626
6627 if (CM.foldTailByMasking())
6628 Legal->prepareToFoldTailByMasking();
6629
6630 ElementCount MaxUserVF =
6631 UserVF.isScalable() ? MaxFactors.ScalableVF : MaxFactors.FixedVF;
6632 if (UserVF) {
6633 if (!ElementCount::isKnownLE(UserVF, MaxUserVF)) {
6635 "UserVF ignored because it may be larger than the maximal safe VF",
6636 "InvalidUserVF", ORE, OrigLoop);
6637 } else {
6639 "VF needs to be a power of two");
6640 // Collect the instructions (and their associated costs) that will be more
6641 // profitable to scalarize.
6642 CM.collectInLoopReductions();
6643 if (CM.selectUserVectorizationFactor(UserVF)) {
6644 LLVM_DEBUG(dbgs() << "LV: Using user VF " << UserVF << ".\n");
6645 buildVPlansWithVPRecipes(UserVF, UserVF);
6647 return;
6648 }
6649 reportVectorizationInfo("UserVF ignored because of invalid costs.",
6650 "InvalidCost", ORE, OrigLoop);
6651 }
6652 }
6653
6654 // Collect the Vectorization Factor Candidates.
6655 SmallVector<ElementCount> VFCandidates;
6656 for (auto VF = ElementCount::getFixed(1);
6657 ElementCount::isKnownLE(VF, MaxFactors.FixedVF); VF *= 2)
6658 VFCandidates.push_back(VF);
6659 for (auto VF = ElementCount::getScalable(1);
6660 ElementCount::isKnownLE(VF, MaxFactors.ScalableVF); VF *= 2)
6661 VFCandidates.push_back(VF);
6662
6663 CM.collectInLoopReductions();
6664 for (const auto &VF : VFCandidates) {
6665 // Collect Uniform and Scalar instructions after vectorization with VF.
6666 CM.collectNonVectorizedAndSetWideningDecisions(VF);
6667 }
6668
6669 buildVPlansWithVPRecipes(ElementCount::getFixed(1), MaxFactors.FixedVF);
6670 buildVPlansWithVPRecipes(ElementCount::getScalable(1), MaxFactors.ScalableVF);
6671
6673}
6674
6676 ElementCount VF) const {
6677 InstructionCost Cost = CM.getInstructionCost(UI, VF);
6678 if (Cost.isValid() && ForceTargetInstructionCost.getNumOccurrences())
6680 return Cost;
6681}
6682
6684 ElementCount VF) const {
6685 return CM.isUniformAfterVectorization(I, VF);
6686}
6687
6688bool VPCostContext::skipCostComputation(Instruction *UI, bool IsVector) const {
6689 return CM.ValuesToIgnore.contains(UI) ||
6690 (IsVector && CM.VecValuesToIgnore.contains(UI)) ||
6691 SkipCostComputation.contains(UI);
6692}
6693
6695LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
6696 VPCostContext &CostCtx) const {
6698 // Cost modeling for inductions is inaccurate in the legacy cost model
6699 // compared to the recipes that are generated. To match here initially during
6700 // VPlan cost model bring up directly use the induction costs from the legacy
6701 // cost model. Note that we do this as pre-processing; the VPlan may not have
6702 // any recipes associated with the original induction increment instruction
6703 // and may replace truncates with VPWidenIntOrFpInductionRecipe. We precompute
6704 // the cost of induction phis and increments (both that are represented by
6705 // recipes and those that are not), to avoid distinguishing between them here,
6706 // and skip all recipes that represent induction phis and increments (the
6707 // former case) later on, if they exist, to avoid counting them twice.
6708 // Similarly we pre-compute the cost of any optimized truncates.
6709 // TODO: Switch to more accurate costing based on VPlan.
6710 for (const auto &[IV, IndDesc] : Legal->getInductionVars()) {
6712 IV->getIncomingValueForBlock(OrigLoop->getLoopLatch()));
6713 SmallVector<Instruction *> IVInsts = {IVInc};
6714 for (unsigned I = 0; I != IVInsts.size(); I++) {
6715 for (Value *Op : IVInsts[I]->operands()) {
6716 auto *OpI = dyn_cast<Instruction>(Op);
6717 if (Op == IV || !OpI || !OrigLoop->contains(OpI) || !Op->hasOneUse())
6718 continue;
6719 IVInsts.push_back(OpI);
6720 }
6721 }
6722 IVInsts.push_back(IV);
6723 for (User *U : IV->users()) {
6724 auto *CI = cast<Instruction>(U);
6725 if (!CostCtx.CM.isOptimizableIVTruncate(CI, VF))
6726 continue;
6727 IVInsts.push_back(CI);
6728 }
6729
6730 // If the vector loop gets executed exactly once with the given VF, ignore
6731 // the costs of comparison and induction instructions, as they'll get
6732 // simplified away.
6733 // TODO: Remove this code after stepping away from the legacy cost model and
6734 // adding code to simplify VPlans before calculating their costs.
6735 auto TC = getSmallConstantTripCount(PSE.getSE(), OrigLoop);
6736 if (TC == VF && !CM.foldTailByMasking())
6737 addFullyUnrolledInstructionsToIgnore(OrigLoop, Legal->getInductionVars(),
6738 CostCtx.SkipCostComputation);
6739
6740 for (Instruction *IVInst : IVInsts) {
6741 if (CostCtx.skipCostComputation(IVInst, VF.isVector()))
6742 continue;
6743 InstructionCost InductionCost = CostCtx.getLegacyCost(IVInst, VF);
6744 LLVM_DEBUG({
6745 dbgs() << "Cost of " << InductionCost << " for VF " << VF
6746 << ": induction instruction " << *IVInst << "\n";
6747 });
6748 Cost += InductionCost;
6749 CostCtx.SkipCostComputation.insert(IVInst);
6750 }
6751 }
6752
6753 /// Compute the cost of all exiting conditions of the loop using the legacy
6754 /// cost model. This is to match the legacy behavior, which adds the cost of
6755 /// all exit conditions. Note that this over-estimates the cost, as there will
6756 /// be a single condition to control the vector loop.
6758 CM.TheLoop->getExitingBlocks(Exiting);
6759 SetVector<Instruction *> ExitInstrs;
6760 // Collect all exit conditions.
6761 for (BasicBlock *EB : Exiting) {
6762 auto *Term = dyn_cast<BranchInst>(EB->getTerminator());
6763 if (!Term || CostCtx.skipCostComputation(Term, VF.isVector()))
6764 continue;
6765 if (auto *CondI = dyn_cast<Instruction>(Term->getOperand(0))) {
6766 ExitInstrs.insert(CondI);
6767 }
6768 }
6769 // Compute the cost of all instructions only feeding the exit conditions.
6770 for (unsigned I = 0; I != ExitInstrs.size(); ++I) {
6771 Instruction *CondI = ExitInstrs[I];
6772 if (!OrigLoop->contains(CondI) ||
6773 !CostCtx.SkipCostComputation.insert(CondI).second)
6774 continue;
6775 InstructionCost CondICost = CostCtx.getLegacyCost(CondI, VF);
6776 LLVM_DEBUG({
6777 dbgs() << "Cost of " << CondICost << " for VF " << VF
6778 << ": exit condition instruction " << *CondI << "\n";
6779 });
6780 Cost += CondICost;
6781 for (Value *Op : CondI->operands()) {
6782 auto *OpI = dyn_cast<Instruction>(Op);
6783 if (!OpI || CostCtx.skipCostComputation(OpI, VF.isVector()) ||
6784 any_of(OpI->users(), [&ExitInstrs, this](User *U) {
6785 return OrigLoop->contains(cast<Instruction>(U)->getParent()) &&
6786 !ExitInstrs.contains(cast<Instruction>(U));
6787 }))
6788 continue;
6789 ExitInstrs.insert(OpI);
6790 }
6791 }
6792
6793 // Pre-compute the costs for branches except for the backedge, as the number
6794 // of replicate regions in a VPlan may not directly match the number of
6795 // branches, which would lead to different decisions.
6796 // TODO: Compute cost of branches for each replicate region in the VPlan,
6797 // which is more accurate than the legacy cost model.
6798 for (BasicBlock *BB : OrigLoop->blocks()) {
6799 if (CostCtx.skipCostComputation(BB->getTerminator(), VF.isVector()))
6800 continue;
6801 CostCtx.SkipCostComputation.insert(BB->getTerminator());
6802 if (BB == OrigLoop->getLoopLatch())
6803 continue;
6804 auto BranchCost = CostCtx.getLegacyCost(BB->getTerminator(), VF);
6805 Cost += BranchCost;
6806 }
6807
6808 // Pre-compute costs for instructions that are forced-scalar or profitable to
6809 // scalarize. Their costs will be computed separately in the legacy cost
6810 // model.
6811 for (Instruction *ForcedScalar : CM.ForcedScalars[VF]) {
6812 if (CostCtx.skipCostComputation(ForcedScalar, VF.isVector()))
6813 continue;
6814 CostCtx.SkipCostComputation.insert(ForcedScalar);
6815 InstructionCost ForcedCost = CostCtx.getLegacyCost(ForcedScalar, VF);
6816 LLVM_DEBUG({
6817 dbgs() << "Cost of " << ForcedCost << " for VF " << VF
6818 << ": forced scalar " << *ForcedScalar << "\n";
6819 });
6820 Cost += ForcedCost;
6821 }
6822 for (const auto &[Scalarized, ScalarCost] : CM.InstsToScalarize[VF]) {
6823 if (CostCtx.skipCostComputation(Scalarized, VF.isVector()))
6824 continue;
6825 CostCtx.SkipCostComputation.insert(Scalarized);
6826 LLVM_DEBUG({
6827 dbgs() << "Cost of " << ScalarCost << " for VF " << VF
6828 << ": profitable to scalarize " << *Scalarized << "\n";
6829 });
6830 Cost += ScalarCost;
6831 }
6832
6833 return Cost;
6834}
6835
6836InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan,
6837 ElementCount VF) const {
6838 VPCostContext CostCtx(CM.TTI, *CM.TLI, Plan, CM, CM.CostKind);
6839 InstructionCost Cost = precomputeCosts(Plan, VF, CostCtx);
6840
6841 // Now compute and add the VPlan-based cost.
6842 Cost += Plan.cost(VF, CostCtx);
6843#ifndef NDEBUG
6844 unsigned EstimatedWidth = estimateElementCount(VF, CM.getVScaleForTuning());
6845 LLVM_DEBUG(dbgs() << "Cost for VF " << VF << ": " << Cost
6846 << " (Estimated cost per lane: ");
6847 if (Cost.isValid()) {
6848 double CostPerLane = double(Cost.getValue()) / EstimatedWidth;
6849 LLVM_DEBUG(dbgs() << format("%.1f", CostPerLane));
6850 } else /* No point dividing an invalid cost - it will still be invalid */
6851 LLVM_DEBUG(dbgs() << "Invalid");
6852 LLVM_DEBUG(dbgs() << ")\n");
6853#endif
6854 return Cost;
6855}
6856
6857#ifndef NDEBUG
6858/// Return true if the original loop \ TheLoop contains any instructions that do
6859/// not have corresponding recipes in \p Plan and are not marked to be ignored
6860/// in \p CostCtx. This means the VPlan contains simplification that the legacy
6861/// cost-model did not account for.
6863 VPCostContext &CostCtx,
6864 Loop *TheLoop,
6865 ElementCount VF) {
6866 // First collect all instructions for the recipes in Plan.
6867 auto GetInstructionForCost = [](const VPRecipeBase *R) -> Instruction * {
6868 if (auto *S = dyn_cast<VPSingleDefRecipe>(R))
6869 return dyn_cast_or_null<Instruction>(S->getUnderlyingValue());
6870 if (auto *WidenMem = dyn_cast<VPWidenMemoryRecipe>(R))
6871 return &WidenMem->getIngredient();
6872 return nullptr;
6873 };
6874
6875 // Check if a select for a safe divisor was hoisted to the pre-header. If so,
6876 // the select doesn't need to be considered for the vector loop cost; go with
6877 // the more accurate VPlan-based cost model.
6878 for (VPRecipeBase &R : *Plan.getVectorPreheader()) {
6879 auto *VPI = dyn_cast<VPInstruction>(&R);
6880 if (!VPI || VPI->getOpcode() != Instruction::Select ||
6881 VPI->getNumUsers() != 1)
6882 continue;
6883
6884 if (auto *WR = dyn_cast<VPWidenRecipe>(*VPI->user_begin())) {
6885 switch (WR->getOpcode()) {
6886 case Instruction::UDiv:
6887 case Instruction::SDiv:
6888 case Instruction::URem:
6889 case Instruction::SRem:
6890 return true;
6891 default:
6892 break;
6893 }
6894 }
6895 }
6896
6897 DenseSet<Instruction *> SeenInstrs;
6898 auto Iter = vp_depth_first_deep(Plan.getVectorLoopRegion()->getEntry());
6900 for (VPRecipeBase &R : *VPBB) {
6901 if (auto *IR = dyn_cast<VPInterleaveRecipe>(&R)) {
6902 auto *IG = IR->getInterleaveGroup();
6903 unsigned NumMembers = IG->getNumMembers();
6904 for (unsigned I = 0; I != NumMembers; ++I) {
6905 if (Instruction *M = IG->getMember(I))
6906 SeenInstrs.insert(M);
6907 }
6908 continue;
6909 }
6910 // Unused FOR splices are removed by VPlan transforms, so the VPlan-based
6911 // cost model won't cost it whilst the legacy will.
6912 if (auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(&R)) {
6913 using namespace VPlanPatternMatch;
6914 if (none_of(FOR->users(),
6915 match_fn(m_VPInstruction<
6917 return true;
6918 }
6919 // The VPlan-based cost model is more accurate for partial reduction and
6920 // comparing against the legacy cost isn't desirable.
6922 return true;
6923
6924 // The VPlan-based cost model can analyze if recipes are scalar
6925 // recursively, but the legacy cost model cannot.
6926 if (auto *WidenMemR = dyn_cast<VPWidenMemoryRecipe>(&R)) {
6927 auto *AddrI = dyn_cast<Instruction>(
6928 getLoadStorePointerOperand(&WidenMemR->getIngredient()));
6929 if (AddrI && vputils::isSingleScalar(WidenMemR->getAddr()) !=
6930 CostCtx.isLegacyUniformAfterVectorization(AddrI, VF))
6931 return true;
6932 }
6933
6934 /// If a VPlan transform folded a recipe to one producing a single-scalar,
6935 /// but the original instruction wasn't uniform-after-vectorization in the
6936 /// legacy cost model, the legacy cost overestimates the actual cost.
6937 if (auto *RepR = dyn_cast<VPReplicateRecipe>(&R)) {
6938 if (RepR->isSingleScalar() &&
6940 RepR->getUnderlyingInstr(), VF))
6941 return true;
6942 }
6943 if (Instruction *UI = GetInstructionForCost(&R)) {
6944 // If we adjusted the predicate of the recipe, the cost in the legacy
6945 // cost model may be different.
6946 using namespace VPlanPatternMatch;
6947 CmpPredicate Pred;
6948 if (match(&R, m_Cmp(Pred, m_VPValue(), m_VPValue())) &&
6949 cast<VPRecipeWithIRFlags>(R).getPredicate() !=
6950 cast<CmpInst>(UI)->getPredicate())
6951 return true;
6952 SeenInstrs.insert(UI);
6953 }
6954 }
6955 }
6956
6957 // Return true if the loop contains any instructions that are not also part of
6958 // the VPlan or are skipped for VPlan-based cost computations. This indicates
6959 // that the VPlan contains extra simplifications.
6960 return any_of(TheLoop->blocks(), [&SeenInstrs, &CostCtx,
6961 TheLoop](BasicBlock *BB) {
6962 return any_of(*BB, [&SeenInstrs, &CostCtx, TheLoop, BB](Instruction &I) {
6963 // Skip induction phis when checking for simplifications, as they may not
6964 // be lowered directly be lowered to a corresponding PHI recipe.
6965 if (isa<PHINode>(&I) && BB == TheLoop->getHeader() &&
6966 CostCtx.CM.Legal->isInductionPhi(cast<PHINode>(&I)))
6967 return false;
6968 return !SeenInstrs.contains(&I) && !CostCtx.skipCostComputation(&I, true);
6969 });
6970 });
6971}
6972#endif
6973
6975 if (VPlans.empty())
6977 // If there is a single VPlan with a single VF, return it directly.
6978 VPlan &FirstPlan = *VPlans[0];
6979 if (VPlans.size() == 1 && size(FirstPlan.vectorFactors()) == 1)
6980 return {*FirstPlan.vectorFactors().begin(), 0, 0};
6981
6982 LLVM_DEBUG(dbgs() << "LV: Computing best VF using cost kind: "
6983 << (CM.CostKind == TTI::TCK_RecipThroughput
6984 ? "Reciprocal Throughput\n"
6985 : CM.CostKind == TTI::TCK_Latency
6986 ? "Instruction Latency\n"
6987 : CM.CostKind == TTI::TCK_CodeSize ? "Code Size\n"
6988 : CM.CostKind == TTI::TCK_SizeAndLatency
6989 ? "Code Size and Latency\n"
6990 : "Unknown\n"));
6991
6993 assert(hasPlanWithVF(ScalarVF) &&
6994 "More than a single plan/VF w/o any plan having scalar VF");
6995
6996 // TODO: Compute scalar cost using VPlan-based cost model.
6997 InstructionCost ScalarCost = CM.expectedCost(ScalarVF);
6998 LLVM_DEBUG(dbgs() << "LV: Scalar loop costs: " << ScalarCost << ".\n");
6999 VectorizationFactor ScalarFactor(ScalarVF, ScalarCost, ScalarCost);
7000 VectorizationFactor BestFactor = ScalarFactor;
7001
7002 bool ForceVectorization = Hints.getForce() == LoopVectorizeHints::FK_Enabled;
7003 if (ForceVectorization) {
7004 // Ignore scalar width, because the user explicitly wants vectorization.
7005 // Initialize cost to max so that VF = 2 is, at least, chosen during cost
7006 // evaluation.
7007 BestFactor.Cost = InstructionCost::getMax();
7008 }
7009
7010 for (auto &P : VPlans) {
7011 ArrayRef<ElementCount> VFs(P->vectorFactors().begin(),
7012 P->vectorFactors().end());
7013
7015 if (any_of(VFs, [this](ElementCount VF) {
7016 return CM.shouldConsiderRegPressureForVF(VF);
7017 }))
7018 RUs = calculateRegisterUsageForPlan(*P, VFs, TTI, CM.ValuesToIgnore);
7019
7020 for (unsigned I = 0; I < VFs.size(); I++) {
7021 ElementCount VF = VFs[I];
7022 if (VF.isScalar())
7023 continue;
7024 if (!ForceVectorization && !willGenerateVectors(*P, VF, TTI)) {
7025 LLVM_DEBUG(
7026 dbgs()
7027 << "LV: Not considering vector loop of width " << VF
7028 << " because it will not generate any vector instructions.\n");
7029 continue;
7030 }
7031 if (CM.OptForSize && !ForceVectorization && hasReplicatorRegion(*P)) {
7032 LLVM_DEBUG(
7033 dbgs()
7034 << "LV: Not considering vector loop of width " << VF
7035 << " because it would cause replicated blocks to be generated,"
7036 << " which isn't allowed when optimizing for size.\n");
7037 continue;
7038 }
7039
7040 InstructionCost Cost = cost(*P, VF);
7041 VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
7042
7043 if (CM.shouldConsiderRegPressureForVF(VF) &&
7044 RUs[I].exceedsMaxNumRegs(TTI, ForceTargetNumVectorRegs)) {
7045 LLVM_DEBUG(dbgs() << "LV(REG): Not considering vector loop of width "
7046 << VF << " because it uses too many registers\n");
7047 continue;
7048 }
7049
7050 if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail()))
7051 BestFactor = CurrentFactor;
7052
7053 // If profitable add it to ProfitableVF list.
7054 if (isMoreProfitable(CurrentFactor, ScalarFactor, P->hasScalarTail()))
7055 ProfitableVFs.push_back(CurrentFactor);
7056 }
7057 }
7058
7059#ifndef NDEBUG
7060 // Select the optimal vectorization factor according to the legacy cost-model.
7061 // This is now only used to verify the decisions by the new VPlan-based
7062 // cost-model and will be retired once the VPlan-based cost-model is
7063 // stabilized.
7064 VectorizationFactor LegacyVF = selectVectorizationFactor();
7065 VPlan &BestPlan = getPlanFor(BestFactor.Width);
7066
7067 // Pre-compute the cost and use it to check if BestPlan contains any
7068 // simplifications not accounted for in the legacy cost model. If that's the
7069 // case, don't trigger the assertion, as the extra simplifications may cause a
7070 // different VF to be picked by the VPlan-based cost model.
7071 VPCostContext CostCtx(CM.TTI, *CM.TLI, BestPlan, CM, CM.CostKind);
7072 precomputeCosts(BestPlan, BestFactor.Width, CostCtx);
7073 // Verify that the VPlan-based and legacy cost models agree, except for VPlans
7074 // with early exits and plans with additional VPlan simplifications. The
7075 // legacy cost model doesn't properly model costs for such loops.
7076 assert((BestFactor.Width == LegacyVF.Width || BestPlan.hasEarlyExit() ||
7078 CostCtx, OrigLoop,
7079 BestFactor.Width) ||
7081 getPlanFor(LegacyVF.Width), CostCtx, OrigLoop, LegacyVF.Width)) &&
7082 " VPlan cost model and legacy cost model disagreed");
7083 assert((BestFactor.Width.isScalar() || BestFactor.ScalarCost > 0) &&
7084 "when vectorizing, the scalar cost must be computed.");
7085#endif
7086
7087 LLVM_DEBUG(dbgs() << "LV: Selecting VF: " << BestFactor.Width << ".\n");
7088 return BestFactor;
7089}
7090
7092 using namespace VPlanPatternMatch;
7094 "RdxResult must be ComputeFindIVResult");
7095 VPValue *StartVPV = RdxResult->getOperand(1);
7096 match(StartVPV, m_Freeze(m_VPValue(StartVPV)));
7097 return StartVPV->getLiveInIRValue();
7098}
7099
7100// If \p EpiResumePhiR is resume VPPhi for a reduction when vectorizing the
7101// epilog loop, fix the reduction's scalar PHI node by adding the incoming value
7102// from the main vector loop.
7104 VPPhi *EpiResumePhiR, PHINode &EpiResumePhi, BasicBlock *BypassBlock) {
7105 // Get the VPInstruction computing the reduction result in the middle block.
7106 // The first operand may not be from the middle block if it is not connected
7107 // to the scalar preheader. In that case, there's nothing to fix.
7108 VPValue *Incoming = EpiResumePhiR->getOperand(0);
7111 auto *EpiRedResult = dyn_cast<VPInstruction>(Incoming);
7112 if (!EpiRedResult ||
7113 (EpiRedResult->getOpcode() != VPInstruction::ComputeAnyOfResult &&
7114 EpiRedResult->getOpcode() != VPInstruction::ComputeReductionResult &&
7115 EpiRedResult->getOpcode() != VPInstruction::ComputeFindIVResult))
7116 return;
7117
7118 auto *EpiRedHeaderPhi =
7119 cast<VPReductionPHIRecipe>(EpiRedResult->getOperand(0));
7120 RecurKind Kind = EpiRedHeaderPhi->getRecurrenceKind();
7121 Value *MainResumeValue;
7122 if (auto *VPI = dyn_cast<VPInstruction>(EpiRedHeaderPhi->getStartValue())) {
7123 assert((VPI->getOpcode() == VPInstruction::Broadcast ||
7124 VPI->getOpcode() == VPInstruction::ReductionStartVector) &&
7125 "unexpected start recipe");
7126 MainResumeValue = VPI->getOperand(0)->getUnderlyingValue();
7127 } else
7128 MainResumeValue = EpiRedHeaderPhi->getStartValue()->getUnderlyingValue();
7130 [[maybe_unused]] Value *StartV =
7131 EpiRedResult->getOperand(1)->getLiveInIRValue();
7132 auto *Cmp = cast<ICmpInst>(MainResumeValue);
7133 assert(Cmp->getPredicate() == CmpInst::ICMP_NE &&
7134 "AnyOf expected to start with ICMP_NE");
7135 assert(Cmp->getOperand(1) == StartV &&
7136 "AnyOf expected to start by comparing main resume value to original "
7137 "start value");
7138 MainResumeValue = Cmp->getOperand(0);
7140 Value *StartV = getStartValueFromReductionResult(EpiRedResult);
7141 Value *SentinelV = EpiRedResult->getOperand(2)->getLiveInIRValue();
7142 using namespace llvm::PatternMatch;
7143 Value *Cmp, *OrigResumeV, *CmpOp;
7144 [[maybe_unused]] bool IsExpectedPattern =
7145 match(MainResumeValue,
7146 m_Select(m_OneUse(m_Value(Cmp)), m_Specific(SentinelV),
7147 m_Value(OrigResumeV))) &&
7149 m_Value(CmpOp))) &&
7150 ((CmpOp == StartV && isGuaranteedNotToBeUndefOrPoison(CmpOp))));
7151 assert(IsExpectedPattern && "Unexpected reduction resume pattern");
7152 MainResumeValue = OrigResumeV;
7153 }
7154 PHINode *MainResumePhi = cast<PHINode>(MainResumeValue);
7155
7156 // When fixing reductions in the epilogue loop we should already have
7157 // created a bc.merge.rdx Phi after the main vector body. Ensure that we carry
7158 // over the incoming values correctly.
7159 EpiResumePhi.setIncomingValueForBlock(
7160 BypassBlock, MainResumePhi->getIncomingValueForBlock(BypassBlock));
7161}
7162
7164 ElementCount BestVF, unsigned BestUF, VPlan &BestVPlan,
7165 InnerLoopVectorizer &ILV, DominatorTree *DT, bool VectorizingEpilogue) {
7166 assert(BestVPlan.hasVF(BestVF) &&
7167 "Trying to execute plan with unsupported VF");
7168 assert(BestVPlan.hasUF(BestUF) &&
7169 "Trying to execute plan with unsupported UF");
7170 if (BestVPlan.hasEarlyExit())
7171 ++LoopsEarlyExitVectorized;
7172 // TODO: Move to VPlan transform stage once the transition to the VPlan-based
7173 // cost model is complete for better cost estimates.
7178 bool HasBranchWeights =
7179 hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator());
7180 if (HasBranchWeights) {
7181 std::optional<unsigned> VScale = CM.getVScaleForTuning();
7183 BestVPlan, BestVF, VScale);
7184 }
7185
7186 // Checks are the same for all VPlans, added to BestVPlan only for
7187 // compactness.
7188 attachRuntimeChecks(BestVPlan, ILV.RTChecks, HasBranchWeights);
7189
7190 // Retrieving VectorPH now when it's easier while VPlan still has Regions.
7191 VPBasicBlock *VectorPH = cast<VPBasicBlock>(BestVPlan.getVectorPreheader());
7192
7193 VPlanTransforms::optimizeForVFAndUF(BestVPlan, BestVF, BestUF, PSE);
7196 if (BestVPlan.getEntry()->getSingleSuccessor() ==
7197 BestVPlan.getScalarPreheader()) {
7198 // TODO: The vector loop would be dead, should not even try to vectorize.
7199 ORE->emit([&]() {
7200 return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationDead",
7201 OrigLoop->getStartLoc(),
7202 OrigLoop->getHeader())
7203 << "Created vector loop never executes due to insufficient trip "
7204 "count.";
7205 });
7207 }
7208
7210 BestVPlan, BestVF,
7211 TTI.getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector));
7213
7215 // Regions are dissolved after optimizing for VF and UF, which completely
7216 // removes unneeded loop regions first.
7218 // Canonicalize EVL loops after regions are dissolved.
7222 BestVPlan, VectorPH, CM.foldTailByMasking(),
7223 CM.requiresScalarEpilogue(BestVF.isVector()));
7224 VPlanTransforms::materializeVFAndVFxUF(BestVPlan, VectorPH, BestVF);
7225 VPlanTransforms::cse(BestVPlan);
7227
7228 // 0. Generate SCEV-dependent code in the entry, including TripCount, before
7229 // making any changes to the CFG.
7230 DenseMap<const SCEV *, Value *> ExpandedSCEVs =
7231 VPlanTransforms::expandSCEVs(BestVPlan, *PSE.getSE());
7232 if (!ILV.getTripCount())
7233 ILV.setTripCount(BestVPlan.getTripCount()->getLiveInIRValue());
7234 else
7235 assert(VectorizingEpilogue && "should only re-use the existing trip "
7236 "count during epilogue vectorization");
7237
7238 // Perform the actual loop transformation.
7239 VPTransformState State(&TTI, BestVF, LI, DT, ILV.AC, ILV.Builder, &BestVPlan,
7240 OrigLoop->getParentLoop(),
7241 Legal->getWidestInductionType());
7242
7243#ifdef EXPENSIVE_CHECKS
7244 assert(DT->verify(DominatorTree::VerificationLevel::Fast));
7245#endif
7246
7247 // 1. Set up the skeleton for vectorization, including vector pre-header and
7248 // middle block. The vector loop is created during VPlan execution.
7249 State.CFG.PrevBB = ILV.createVectorizedLoopSkeleton();
7251 State.CFG.PrevBB->getSingleSuccessor(), &BestVPlan);
7253
7254 assert(verifyVPlanIsValid(BestVPlan, true /*VerifyLate*/) &&
7255 "final VPlan is invalid");
7256
7257 // After vectorization, the exit blocks of the original loop will have
7258 // additional predecessors. Invalidate SCEVs for the exit phis in case SE
7259 // looked through single-entry phis.
7260 ScalarEvolution &SE = *PSE.getSE();
7261 for (VPIRBasicBlock *Exit : BestVPlan.getExitBlocks()) {
7262 if (!Exit->hasPredecessors())
7263 continue;
7264 for (VPRecipeBase &PhiR : Exit->phis())
7266 OrigLoop, cast<PHINode>(&cast<VPIRPhi>(PhiR).getInstruction()));
7267 }
7268 // Forget the original loop and block dispositions.
7269 SE.forgetLoop(OrigLoop);
7271
7273
7274 //===------------------------------------------------===//
7275 //
7276 // Notice: any optimization or new instruction that go
7277 // into the code below should also be implemented in
7278 // the cost-model.
7279 //
7280 //===------------------------------------------------===//
7281
7282 // Retrieve loop information before executing the plan, which may remove the
7283 // original loop, if it becomes unreachable.
7284 MDNode *LID = OrigLoop->getLoopID();
7285 unsigned OrigLoopInvocationWeight = 0;
7286 std::optional<unsigned> OrigAverageTripCount =
7287 getLoopEstimatedTripCount(OrigLoop, &OrigLoopInvocationWeight);
7288
7289 BestVPlan.execute(&State);
7290
7291 // 2.6. Maintain Loop Hints
7292 // Keep all loop hints from the original loop on the vector loop (we'll
7293 // replace the vectorizer-specific hints below).
7294 VPBasicBlock *HeaderVPBB = vputils::getFirstLoopHeader(BestVPlan, State.VPDT);
7295 // Add metadata to disable runtime unrolling a scalar loop when there
7296 // are no runtime checks about strides and memory. A scalar loop that is
7297 // rarely used is not worth unrolling.
7298 bool DisableRuntimeUnroll = !ILV.RTChecks.hasChecks() && !BestVF.isScalar();
7300 HeaderVPBB ? LI->getLoopFor(State.CFG.VPBB2IRBB.lookup(HeaderVPBB))
7301 : nullptr,
7302 HeaderVPBB, BestVPlan, VectorizingEpilogue, LID, OrigAverageTripCount,
7303 OrigLoopInvocationWeight,
7304 estimateElementCount(BestVF * BestUF, CM.getVScaleForTuning()),
7305 DisableRuntimeUnroll);
7306
7307 // 3. Fix the vectorized code: take care of header phi's, live-outs,
7308 // predication, updating analyses.
7309 ILV.fixVectorizedLoop(State);
7310
7312
7313 return ExpandedSCEVs;
7314}
7315
7316//===--------------------------------------------------------------------===//
7317// EpilogueVectorizerMainLoop
7318//===--------------------------------------------------------------------===//
7319
7320/// This function is partially responsible for generating the control flow
7321/// depicted in https://llvm.org/docs/Vectorizers.html#epilogue-vectorization.
7323 BasicBlock *ScalarPH = createScalarPreheader("");
7324 BasicBlock *VectorPH = ScalarPH->getSinglePredecessor();
7325
7326 // Generate the code to check the minimum iteration count of the vector
7327 // epilogue (see below).
7328 EPI.EpilogueIterationCountCheck =
7329 emitIterationCountCheck(VectorPH, ScalarPH, true);
7330 EPI.EpilogueIterationCountCheck->setName("iter.check");
7331
7332 VectorPH = cast<BranchInst>(EPI.EpilogueIterationCountCheck->getTerminator())
7333 ->getSuccessor(1);
7334 // Generate the iteration count check for the main loop, *after* the check
7335 // for the epilogue loop, so that the path-length is shorter for the case
7336 // that goes directly through the vector epilogue. The longer-path length for
7337 // the main loop is compensated for, by the gain from vectorizing the larger
7338 // trip count. Note: the branch will get updated later on when we vectorize
7339 // the epilogue.
7340 EPI.MainLoopIterationCountCheck =
7341 emitIterationCountCheck(VectorPH, ScalarPH, false);
7342
7343 return cast<BranchInst>(EPI.MainLoopIterationCountCheck->getTerminator())
7344 ->getSuccessor(1);
7345}
7346
7348 LLVM_DEBUG({
7349 dbgs() << "Create Skeleton for epilogue vectorized loop (first pass)\n"
7350 << "Main Loop VF:" << EPI.MainLoopVF
7351 << ", Main Loop UF:" << EPI.MainLoopUF
7352 << ", Epilogue Loop VF:" << EPI.EpilogueVF
7353 << ", Epilogue Loop UF:" << EPI.EpilogueUF << "\n";
7354 });
7355}
7356
7359 dbgs() << "intermediate fn:\n"
7360 << *OrigLoop->getHeader()->getParent() << "\n";
7361 });
7362}
7363
7365 BasicBlock *VectorPH, BasicBlock *Bypass, bool ForEpilogue) {
7366 assert(Bypass && "Expected valid bypass basic block.");
7369 Value *CheckMinIters = createIterationCountCheck(
7370 VectorPH, ForEpilogue ? EPI.EpilogueVF : EPI.MainLoopVF,
7371 ForEpilogue ? EPI.EpilogueUF : EPI.MainLoopUF);
7372
7373 BasicBlock *const TCCheckBlock = VectorPH;
7374 if (!ForEpilogue)
7375 TCCheckBlock->setName("vector.main.loop.iter.check");
7376
7377 // Create new preheader for vector loop.
7378 VectorPH = SplitBlock(TCCheckBlock, TCCheckBlock->getTerminator(),
7379 static_cast<DominatorTree *>(nullptr), LI, nullptr,
7380 "vector.ph");
7381 if (ForEpilogue) {
7382 // Save the trip count so we don't have to regenerate it in the
7383 // vec.epilog.iter.check. This is safe to do because the trip count
7384 // generated here dominates the vector epilog iter check.
7385 EPI.TripCount = Count;
7386 } else {
7388 }
7389
7390 BranchInst &BI = *BranchInst::Create(Bypass, VectorPH, CheckMinIters);
7391 if (hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator()))
7392 setBranchWeights(BI, MinItersBypassWeights, /*IsExpected=*/false);
7393 ReplaceInstWithInst(TCCheckBlock->getTerminator(), &BI);
7394
7395 // When vectorizing the main loop, its trip-count check is placed in a new
7396 // block, whereas the overall trip-count check is placed in the VPlan entry
7397 // block. When vectorizing the epilogue loop, its trip-count check is placed
7398 // in the VPlan entry block.
7399 if (!ForEpilogue)
7400 introduceCheckBlockInVPlan(TCCheckBlock);
7401 return TCCheckBlock;
7402}
7403
7404//===--------------------------------------------------------------------===//
7405// EpilogueVectorizerEpilogueLoop
7406//===--------------------------------------------------------------------===//
7407
7408/// This function creates a new scalar preheader, using the previous one as
7409/// entry block to the epilogue VPlan. The minimum iteration check is being
7410/// represented in VPlan.
7412 BasicBlock *NewScalarPH = createScalarPreheader("vec.epilog.");
7413 BasicBlock *OriginalScalarPH = NewScalarPH->getSinglePredecessor();
7414 OriginalScalarPH->setName("vec.epilog.iter.check");
7415 VPIRBasicBlock *NewEntry = Plan.createVPIRBasicBlock(OriginalScalarPH);
7416 VPBasicBlock *OldEntry = Plan.getEntry();
7417 for (auto &R : make_early_inc_range(*OldEntry)) {
7418 // Skip moving VPIRInstructions (including VPIRPhis), which are unmovable by
7419 // defining.
7420 if (isa<VPIRInstruction>(&R))
7421 continue;
7422 R.moveBefore(*NewEntry, NewEntry->end());
7423 }
7424
7425 VPBlockUtils::reassociateBlocks(OldEntry, NewEntry);
7426 Plan.setEntry(NewEntry);
7427 // OldEntry is now dead and will be cleaned up when the plan gets destroyed.
7428
7429 return OriginalScalarPH;
7430}
7431
7433 LLVM_DEBUG({
7434 dbgs() << "Create Skeleton for epilogue vectorized loop (second pass)\n"
7435 << "Epilogue Loop VF:" << EPI.EpilogueVF
7436 << ", Epilogue Loop UF:" << EPI.EpilogueUF << "\n";
7437 });
7438}
7439
7442 dbgs() << "final fn:\n" << *OrigLoop->getHeader()->getParent() << "\n";
7443 });
7444}
7445
7447VPRecipeBuilder::tryToWidenMemory(Instruction *I, ArrayRef<VPValue *> Operands,
7448 VFRange &Range) {
7450 "Must be called with either a load or store");
7451
7452 auto WillWiden = [&](ElementCount VF) -> bool {
7454 CM.getWideningDecision(I, VF);
7456 "CM decision should be taken at this point.");
7458 return true;
7459 if (CM.isScalarAfterVectorization(I, VF) ||
7460 CM.isProfitableToScalarize(I, VF))
7461 return false;
7463 };
7464
7466 return nullptr;
7467
7468 VPValue *Mask = nullptr;
7469 if (Legal->isMaskRequired(I))
7470 Mask = getBlockInMask(Builder.getInsertBlock());
7471
7472 // Determine if the pointer operand of the access is either consecutive or
7473 // reverse consecutive.
7475 CM.getWideningDecision(I, Range.Start);
7477 bool Consecutive =
7479
7481 if (Consecutive) {
7483 Ptr->getUnderlyingValue()->stripPointerCasts());
7484 VPSingleDefRecipe *VectorPtr;
7485 if (Reverse) {
7486 // When folding the tail, we may compute an address that we don't in the
7487 // original scalar loop: drop the GEP no-wrap flags in this case.
7488 // Otherwise preserve existing flags without no-unsigned-wrap, as we will
7489 // emit negative indices.
7490 GEPNoWrapFlags Flags =
7491 CM.foldTailByMasking() || !GEP
7493 : GEP->getNoWrapFlags().withoutNoUnsignedWrap();
7494 VectorPtr =
7496 /*Stride*/ -1, Flags, I->getDebugLoc());
7497 } else {
7498 VectorPtr = new VPVectorPointerRecipe(Ptr, getLoadStoreType(I),
7499 GEP ? GEP->getNoWrapFlags()
7501 I->getDebugLoc());
7502 }
7503 Builder.insert(VectorPtr);
7504 Ptr = VectorPtr;
7505 }
7506 if (LoadInst *Load = dyn_cast<LoadInst>(I))
7507 return new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, Reverse,
7508 VPIRMetadata(*Load, LVer), I->getDebugLoc());
7509
7510 StoreInst *Store = cast<StoreInst>(I);
7511 return new VPWidenStoreRecipe(*Store, Ptr, Operands[0], Mask, Consecutive,
7512 Reverse, VPIRMetadata(*Store, LVer),
7513 I->getDebugLoc());
7514}
7515
7516/// Creates a VPWidenIntOrFpInductionRecpipe for \p Phi. If needed, it will also
7517/// insert a recipe to expand the step for the induction recipe.
7518static VPWidenIntOrFpInductionRecipe *
7520 VPValue *Start, const InductionDescriptor &IndDesc,
7521 VPlan &Plan, ScalarEvolution &SE, Loop &OrigLoop) {
7522 assert(IndDesc.getStartValue() ==
7523 Phi->getIncomingValueForBlock(OrigLoop.getLoopPreheader()));
7524 assert(SE.isLoopInvariant(IndDesc.getStep(), &OrigLoop) &&
7525 "step must be loop invariant");
7526
7527 VPValue *Step =
7529 if (auto *TruncI = dyn_cast<TruncInst>(PhiOrTrunc)) {
7530 return new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, &Plan.getVF(),
7531 IndDesc, TruncI,
7532 TruncI->getDebugLoc());
7533 }
7534 assert(isa<PHINode>(PhiOrTrunc) && "must be a phi node here");
7535 return new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, &Plan.getVF(),
7536 IndDesc, Phi->getDebugLoc());
7537}
7538
7539VPHeaderPHIRecipe *VPRecipeBuilder::tryToOptimizeInductionPHI(
7540 PHINode *Phi, ArrayRef<VPValue *> Operands, VFRange &Range) {
7541
7542 // Check if this is an integer or fp induction. If so, build the recipe that
7543 // produces its scalar and vector values.
7544 if (auto *II = Legal->getIntOrFpInductionDescriptor(Phi))
7545 return createWidenInductionRecipes(Phi, Phi, Operands[0], *II, Plan,
7546 *PSE.getSE(), *OrigLoop);
7547
7548 // Check if this is pointer induction. If so, build the recipe for it.
7549 if (auto *II = Legal->getPointerInductionDescriptor(Phi)) {
7550 VPValue *Step = vputils::getOrCreateVPValueForSCEVExpr(Plan, II->getStep());
7551 return new VPWidenPointerInductionRecipe(
7552 Phi, Operands[0], Step, &Plan.getVFxUF(), *II,
7554 [&](ElementCount VF) {
7555 return CM.isScalarAfterVectorization(Phi, VF);
7556 },
7557 Range),
7558 Phi->getDebugLoc());
7559 }
7560 return nullptr;
7561}
7562
7563VPWidenIntOrFpInductionRecipe *VPRecipeBuilder::tryToOptimizeInductionTruncate(
7564 TruncInst *I, ArrayRef<VPValue *> Operands, VFRange &Range) {
7565 // Optimize the special case where the source is a constant integer
7566 // induction variable. Notice that we can only optimize the 'trunc' case
7567 // because (a) FP conversions lose precision, (b) sext/zext may wrap, and
7568 // (c) other casts depend on pointer size.
7569
7570 // Determine whether \p K is a truncation based on an induction variable that
7571 // can be optimized.
7572 auto IsOptimizableIVTruncate =
7573 [&](Instruction *K) -> std::function<bool(ElementCount)> {
7574 return [=](ElementCount VF) -> bool {
7575 return CM.isOptimizableIVTruncate(K, VF);
7576 };
7577 };
7578
7580 IsOptimizableIVTruncate(I), Range)) {
7581
7582 auto *Phi = cast<PHINode>(I->getOperand(0));
7583 const InductionDescriptor &II = *Legal->getIntOrFpInductionDescriptor(Phi);
7584 VPValue *Start = Plan.getOrAddLiveIn(II.getStartValue());
7585 return createWidenInductionRecipes(Phi, I, Start, II, Plan, *PSE.getSE(),
7586 *OrigLoop);
7587 }
7588 return nullptr;
7589}
7590
7591VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
7593 VFRange &Range) {
7595 [this, CI](ElementCount VF) {
7596 return CM.isScalarWithPredication(CI, VF);
7597 },
7598 Range);
7599
7600 if (IsPredicated)
7601 return nullptr;
7602
7604 if (ID && (ID == Intrinsic::assume || ID == Intrinsic::lifetime_end ||
7605 ID == Intrinsic::lifetime_start || ID == Intrinsic::sideeffect ||
7606 ID == Intrinsic::pseudoprobe ||
7607 ID == Intrinsic::experimental_noalias_scope_decl))
7608 return nullptr;
7609
7611
7612 // Is it beneficial to perform intrinsic call compared to lib call?
7613 bool ShouldUseVectorIntrinsic =
7615 [&](ElementCount VF) -> bool {
7616 return CM.getCallWideningDecision(CI, VF).Kind ==
7618 },
7619 Range);
7620 if (ShouldUseVectorIntrinsic)
7621 return new VPWidenIntrinsicRecipe(*CI, ID, Ops, CI->getType(),
7622 CI->getDebugLoc());
7623
7624 Function *Variant = nullptr;
7625 std::optional<unsigned> MaskPos;
7626 // Is better to call a vectorized version of the function than to to scalarize
7627 // the call?
7628 auto ShouldUseVectorCall = LoopVectorizationPlanner::getDecisionAndClampRange(
7629 [&](ElementCount VF) -> bool {
7630 // The following case may be scalarized depending on the VF.
7631 // The flag shows whether we can use a usual Call for vectorized
7632 // version of the instruction.
7633
7634 // If we've found a variant at a previous VF, then stop looking. A
7635 // vectorized variant of a function expects input in a certain shape
7636 // -- basically the number of input registers, the number of lanes
7637 // per register, and whether there's a mask required.
7638 // We store a pointer to the variant in the VPWidenCallRecipe, so
7639 // once we have an appropriate variant it's only valid for that VF.
7640 // This will force a different vplan to be generated for each VF that
7641 // finds a valid variant.
7642 if (Variant)
7643 return false;
7644 LoopVectorizationCostModel::CallWideningDecision Decision =
7645 CM.getCallWideningDecision(CI, VF);
7647 Variant = Decision.Variant;
7648 MaskPos = Decision.MaskPos;
7649 return true;
7650 }
7651
7652 return false;
7653 },
7654 Range);
7655 if (ShouldUseVectorCall) {
7656 if (MaskPos.has_value()) {
7657 // We have 2 cases that would require a mask:
7658 // 1) The block needs to be predicated, either due to a conditional
7659 // in the scalar loop or use of an active lane mask with
7660 // tail-folding, and we use the appropriate mask for the block.
7661 // 2) No mask is required for the block, but the only available
7662 // vector variant at this VF requires a mask, so we synthesize an
7663 // all-true mask.
7664 VPValue *Mask = nullptr;
7665 if (Legal->isMaskRequired(CI))
7666 Mask = getBlockInMask(Builder.getInsertBlock());
7667 else
7668 Mask = Plan.getOrAddLiveIn(
7669 ConstantInt::getTrue(IntegerType::getInt1Ty(CI->getContext())));
7670
7671 Ops.insert(Ops.begin() + *MaskPos, Mask);
7672 }
7673
7674 Ops.push_back(Operands.back());
7675 return new VPWidenCallRecipe(CI, Variant, Ops, CI->getDebugLoc());
7676 }
7677
7678 return nullptr;
7679}
7680
7681bool VPRecipeBuilder::shouldWiden(Instruction *I, VFRange &Range) const {
7683 !isa<StoreInst>(I) && "Instruction should have been handled earlier");
7684 // Instruction should be widened, unless it is scalar after vectorization,
7685 // scalarization is profitable or it is predicated.
7686 auto WillScalarize = [this, I](ElementCount VF) -> bool {
7687 return CM.isScalarAfterVectorization(I, VF) ||
7688 CM.isProfitableToScalarize(I, VF) ||
7689 CM.isScalarWithPredication(I, VF);
7690 };
7692 Range);
7693}
7694
7695VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
7697 switch (I->getOpcode()) {
7698 default:
7699 return nullptr;
7700 case Instruction::SDiv:
7701 case Instruction::UDiv:
7702 case Instruction::SRem:
7703 case Instruction::URem: {
7704 // If not provably safe, use a select to form a safe divisor before widening the
7705 // div/rem operation itself. Otherwise fall through to general handling below.
7706 if (CM.isPredicatedInst(I)) {
7708 VPValue *Mask = getBlockInMask(Builder.getInsertBlock());
7709 VPValue *One =
7710 Plan.getOrAddLiveIn(ConstantInt::get(I->getType(), 1u, false));
7711 auto *SafeRHS = Builder.createSelect(Mask, Ops[1], One, I->getDebugLoc());
7712 Ops[1] = SafeRHS;
7713 return new VPWidenRecipe(*I, Ops);
7714 }
7715 [[fallthrough]];
7716 }
7717 case Instruction::Add:
7718 case Instruction::And:
7719 case Instruction::AShr:
7720 case Instruction::FAdd:
7721 case Instruction::FCmp:
7722 case Instruction::FDiv:
7723 case Instruction::FMul:
7724 case Instruction::FNeg:
7725 case Instruction::FRem:
7726 case Instruction::FSub:
7727 case Instruction::ICmp:
7728 case Instruction::LShr:
7729 case Instruction::Mul:
7730 case Instruction::Or:
7731 case Instruction::Select:
7732 case Instruction::Shl:
7733 case Instruction::Sub:
7734 case Instruction::Xor:
7735 case Instruction::Freeze: {
7737 if (Instruction::isBinaryOp(I->getOpcode())) {
7738 // The legacy cost model uses SCEV to check if some of the operands are
7739 // constants. To match the legacy cost model's behavior, use SCEV to try
7740 // to replace operands with constants.
7741 ScalarEvolution &SE = *PSE.getSE();
7742 auto GetConstantViaSCEV = [this, &SE](VPValue *Op) {
7743 if (!Op->isLiveIn())
7744 return Op;
7745 Value *V = Op->getUnderlyingValue();
7746 if (isa<Constant>(V) || !SE.isSCEVable(V->getType()))
7747 return Op;
7748 auto *C = dyn_cast<SCEVConstant>(SE.getSCEV(V));
7749 if (!C)
7750 return Op;
7751 return Plan.getOrAddLiveIn(C->getValue());
7752 };
7753 // For Mul, the legacy cost model checks both operands.
7754 if (I->getOpcode() == Instruction::Mul)
7755 NewOps[0] = GetConstantViaSCEV(NewOps[0]);
7756 // For other binops, the legacy cost model only checks the second operand.
7757 NewOps[1] = GetConstantViaSCEV(NewOps[1]);
7758 }
7759 return new VPWidenRecipe(*I, NewOps);
7760 }
7761 case Instruction::ExtractValue: {
7763 Type *I32Ty = IntegerType::getInt32Ty(I->getContext());
7764 auto *EVI = cast<ExtractValueInst>(I);
7765 assert(EVI->getNumIndices() == 1 && "Expected one extractvalue index");
7766 unsigned Idx = EVI->getIndices()[0];
7767 NewOps.push_back(Plan.getOrAddLiveIn(ConstantInt::get(I32Ty, Idx, false)));
7768 return new VPWidenRecipe(*I, NewOps);
7769 }
7770 };
7771}
7772
7773VPHistogramRecipe *
7774VPRecipeBuilder::tryToWidenHistogram(const HistogramInfo *HI,
7776 // FIXME: Support other operations.
7777 unsigned Opcode = HI->Update->getOpcode();
7778 assert((Opcode == Instruction::Add || Opcode == Instruction::Sub) &&
7779 "Histogram update operation must be an Add or Sub");
7780
7782 // Bucket address.
7783 HGramOps.push_back(Operands[1]);
7784 // Increment value.
7785 HGramOps.push_back(getVPValueOrAddLiveIn(HI->Update->getOperand(1)));
7786
7787 // In case of predicated execution (due to tail-folding, or conditional
7788 // execution, or both), pass the relevant mask.
7789 if (Legal->isMaskRequired(HI->Store))
7790 HGramOps.push_back(getBlockInMask(Builder.getInsertBlock()));
7791
7792 return new VPHistogramRecipe(Opcode, HGramOps, HI->Store->getDebugLoc());
7793}
7794
7795VPReplicateRecipe *
7797 VFRange &Range) {
7799 [&](ElementCount VF) { return CM.isUniformAfterVectorization(I, VF); },
7800 Range);
7801
7802 bool IsPredicated = CM.isPredicatedInst(I);
7803
7804 // Even if the instruction is not marked as uniform, there are certain
7805 // intrinsic calls that can be effectively treated as such, so we check for
7806 // them here. Conservatively, we only do this for scalable vectors, since
7807 // for fixed-width VFs we can always fall back on full scalarization.
7808 if (!IsUniform && Range.Start.isScalable() && isa<IntrinsicInst>(I)) {
7809 switch (cast<IntrinsicInst>(I)->getIntrinsicID()) {
7810 case Intrinsic::assume:
7811 case Intrinsic::lifetime_start:
7812 case Intrinsic::lifetime_end:
7813 // For scalable vectors if one of the operands is variant then we still
7814 // want to mark as uniform, which will generate one instruction for just
7815 // the first lane of the vector. We can't scalarize the call in the same
7816 // way as for fixed-width vectors because we don't know how many lanes
7817 // there are.
7818 //
7819 // The reasons for doing it this way for scalable vectors are:
7820 // 1. For the assume intrinsic generating the instruction for the first
7821 // lane is still be better than not generating any at all. For
7822 // example, the input may be a splat across all lanes.
7823 // 2. For the lifetime start/end intrinsics the pointer operand only
7824 // does anything useful when the input comes from a stack object,
7825 // which suggests it should always be uniform. For non-stack objects
7826 // the effect is to poison the object, which still allows us to
7827 // remove the call.
7828 IsUniform = true;
7829 break;
7830 default:
7831 break;
7832 }
7833 }
7834 VPValue *BlockInMask = nullptr;
7835 if (!IsPredicated) {
7836 // Finalize the recipe for Instr, first if it is not predicated.
7837 LLVM_DEBUG(dbgs() << "LV: Scalarizing:" << *I << "\n");
7838 } else {
7839 LLVM_DEBUG(dbgs() << "LV: Scalarizing and predicating:" << *I << "\n");
7840 // Instructions marked for predication are replicated and a mask operand is
7841 // added initially. Masked replicate recipes will later be placed under an
7842 // if-then construct to prevent side-effects. Generate recipes to compute
7843 // the block mask for this region.
7844 BlockInMask = getBlockInMask(Builder.getInsertBlock());
7845 }
7846
7847 // Note that there is some custom logic to mark some intrinsics as uniform
7848 // manually above for scalable vectors, which this assert needs to account for
7849 // as well.
7850 assert((Range.Start.isScalar() || !IsUniform || !IsPredicated ||
7851 (Range.Start.isScalable() && isa<IntrinsicInst>(I))) &&
7852 "Should not predicate a uniform recipe");
7853 auto *Recipe = new VPReplicateRecipe(I, Operands, IsUniform, BlockInMask,
7854 VPIRMetadata(*I, LVer));
7855 return Recipe;
7856}
7857
7858/// Find all possible partial reductions in the loop and track all of those that
7859/// are valid so recipes can be formed later.
7861 // Find all possible partial reductions.
7863 PartialReductionChains;
7864 for (const auto &[Phi, RdxDesc] : Legal->getReductionVars()) {
7865 getScaledReductions(Phi, RdxDesc.getLoopExitInstr(), Range,
7866 PartialReductionChains);
7867 }
7868
7869 // A partial reduction is invalid if any of its extends are used by
7870 // something that isn't another partial reduction. This is because the
7871 // extends are intended to be lowered along with the reduction itself.
7872
7873 // Build up a set of partial reduction ops for efficient use checking.
7874 SmallPtrSet<User *, 4> PartialReductionOps;
7875 for (const auto &[PartialRdx, _] : PartialReductionChains)
7876 PartialReductionOps.insert(PartialRdx.ExtendUser);
7877
7878 auto ExtendIsOnlyUsedByPartialReductions =
7879 [&PartialReductionOps](Instruction *Extend) {
7880 return all_of(Extend->users(), [&](const User *U) {
7881 return PartialReductionOps.contains(U);
7882 });
7883 };
7884
7885 // Check if each use of a chain's two extends is a partial reduction
7886 // and only add those that don't have non-partial reduction users.
7887 for (auto Pair : PartialReductionChains) {
7888 PartialReductionChain Chain = Pair.first;
7889 if (ExtendIsOnlyUsedByPartialReductions(Chain.ExtendA) &&
7890 (!Chain.ExtendB || ExtendIsOnlyUsedByPartialReductions(Chain.ExtendB)))
7891 ScaledReductionMap.try_emplace(Chain.Reduction, Pair.second);
7892 }
7893}
7894
7895bool VPRecipeBuilder::getScaledReductions(
7896 Instruction *PHI, Instruction *RdxExitInstr, VFRange &Range,
7897 SmallVectorImpl<std::pair<PartialReductionChain, unsigned>> &Chains) {
7898 if (!CM.TheLoop->contains(RdxExitInstr))
7899 return false;
7900
7901 auto *Update = dyn_cast<BinaryOperator>(RdxExitInstr);
7902 if (!Update)
7903 return false;
7904
7905 Value *Op = Update->getOperand(0);
7906 Value *PhiOp = Update->getOperand(1);
7907 if (Op == PHI)
7908 std::swap(Op, PhiOp);
7909
7910 // Try and get a scaled reduction from the first non-phi operand.
7911 // If one is found, we use the discovered reduction instruction in
7912 // place of the accumulator for costing.
7913 if (auto *OpInst = dyn_cast<Instruction>(Op)) {
7914 if (getScaledReductions(PHI, OpInst, Range, Chains)) {
7915 PHI = Chains.rbegin()->first.Reduction;
7916
7917 Op = Update->getOperand(0);
7918 PhiOp = Update->getOperand(1);
7919 if (Op == PHI)
7920 std::swap(Op, PhiOp);
7921 }
7922 }
7923 if (PhiOp != PHI)
7924 return false;
7925
7926 using namespace llvm::PatternMatch;
7927
7928 // If the update is a binary operator, check both of its operands to see if
7929 // they are extends. Otherwise, see if the update comes directly from an
7930 // extend.
7931 Instruction *Exts[2] = {nullptr};
7932 BinaryOperator *ExtendUser = dyn_cast<BinaryOperator>(Op);
7933 std::optional<unsigned> BinOpc;
7934 Type *ExtOpTypes[2] = {nullptr};
7936
7937 auto CollectExtInfo = [this, &Exts, &ExtOpTypes,
7938 &ExtKinds](SmallVectorImpl<Value *> &Ops) -> bool {
7939 for (const auto &[I, OpI] : enumerate(Ops)) {
7940 Value *ExtOp;
7941 if (!match(OpI, m_ZExtOrSExt(m_Value(ExtOp))))
7942 return false;
7943 Exts[I] = cast<Instruction>(OpI);
7944
7945 // TODO: We should be able to support live-ins.
7946 if (!CM.TheLoop->contains(Exts[I]))
7947 return false;
7948
7949 ExtOpTypes[I] = ExtOp->getType();
7950 ExtKinds[I] = TTI::getPartialReductionExtendKind(Exts[I]);
7951 }
7952 return true;
7953 };
7954
7955 if (ExtendUser) {
7956 if (!ExtendUser->hasOneUse())
7957 return false;
7958
7959 // Use the side-effect of match to replace BinOp only if the pattern is
7960 // matched, we don't care at this point whether it actually matched.
7961 match(ExtendUser, m_Neg(m_BinOp(ExtendUser)));
7962
7963 SmallVector<Value *> Ops(ExtendUser->operands());
7964 if (!CollectExtInfo(Ops))
7965 return false;
7966
7967 BinOpc = std::make_optional(ExtendUser->getOpcode());
7968 } else if (match(Update, m_Add(m_Value(), m_Value()))) {
7969 // We already know the operands for Update are Op and PhiOp.
7971 if (!CollectExtInfo(Ops))
7972 return false;
7973
7974 ExtendUser = Update;
7975 BinOpc = std::nullopt;
7976 } else
7977 return false;
7978
7979 PartialReductionChain Chain(RdxExitInstr, Exts[0], Exts[1], ExtendUser);
7980
7981 TypeSize PHISize = PHI->getType()->getPrimitiveSizeInBits();
7982 TypeSize ASize = ExtOpTypes[0]->getPrimitiveSizeInBits();
7983 if (!PHISize.hasKnownScalarFactor(ASize))
7984 return false;
7985 unsigned TargetScaleFactor = PHISize.getKnownScalarFactor(ASize);
7986
7988 [&](ElementCount VF) {
7990 Update->getOpcode(), ExtOpTypes[0], ExtOpTypes[1],
7991 PHI->getType(), VF, ExtKinds[0], ExtKinds[1], BinOpc,
7992 CM.CostKind);
7993 return Cost.isValid();
7994 },
7995 Range)) {
7996 Chains.emplace_back(Chain, TargetScaleFactor);
7997 return true;
7998 }
7999
8000 return false;
8001}
8002
8004 VFRange &Range) {
8005 // First, check for specific widening recipes that deal with inductions, Phi
8006 // nodes, calls and memory operations.
8007 VPRecipeBase *Recipe;
8008 Instruction *Instr = R->getUnderlyingInstr();
8009 SmallVector<VPValue *, 4> Operands(R->operands());
8010 if (auto *PhiR = dyn_cast<VPPhi>(R)) {
8011 VPBasicBlock *Parent = PhiR->getParent();
8012 [[maybe_unused]] VPRegionBlock *LoopRegionOf =
8013 Parent->getEnclosingLoopRegion();
8014 assert(LoopRegionOf && LoopRegionOf->getEntry() == Parent &&
8015 "Non-header phis should have been handled during predication");
8016 auto *Phi = cast<PHINode>(R->getUnderlyingInstr());
8017 assert(Operands.size() == 2 && "Must have 2 operands for header phis");
8018 if ((Recipe = tryToOptimizeInductionPHI(Phi, Operands, Range)))
8019 return Recipe;
8020
8021 VPHeaderPHIRecipe *PhiRecipe = nullptr;
8022 assert((Legal->isReductionVariable(Phi) ||
8023 Legal->isFixedOrderRecurrence(Phi)) &&
8024 "can only widen reductions and fixed-order recurrences here");
8025 VPValue *StartV = Operands[0];
8026 if (Legal->isReductionVariable(Phi)) {
8027 const RecurrenceDescriptor &RdxDesc = Legal->getRecurrenceDescriptor(Phi);
8028 assert(RdxDesc.getRecurrenceStartValue() ==
8029 Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader()));
8030
8031 // If the PHI is used by a partial reduction, set the scale factor.
8032 unsigned ScaleFactor =
8033 getScalingForReduction(RdxDesc.getLoopExitInstr()).value_or(1);
8034 PhiRecipe = new VPReductionPHIRecipe(
8035 Phi, RdxDesc.getRecurrenceKind(), *StartV, CM.isInLoopReduction(Phi),
8036 CM.useOrderedReductions(RdxDesc), ScaleFactor);
8037 } else {
8038 // TODO: Currently fixed-order recurrences are modeled as chains of
8039 // first-order recurrences. If there are no users of the intermediate
8040 // recurrences in the chain, the fixed order recurrence should be modeled
8041 // directly, enabling more efficient codegen.
8042 PhiRecipe = new VPFirstOrderRecurrencePHIRecipe(Phi, *StartV);
8043 }
8044 // Add backedge value.
8045 PhiRecipe->addOperand(Operands[1]);
8046 return PhiRecipe;
8047 }
8048 assert(!R->isPhi() && "only VPPhi nodes expected at this point");
8049
8050 if (isa<TruncInst>(Instr) && (Recipe = tryToOptimizeInductionTruncate(
8051 cast<TruncInst>(Instr), Operands, Range)))
8052 return Recipe;
8053
8054 // All widen recipes below deal only with VF > 1.
8056 [&](ElementCount VF) { return VF.isScalar(); }, Range))
8057 return nullptr;
8058
8059 if (auto *CI = dyn_cast<CallInst>(Instr))
8060 return tryToWidenCall(CI, Operands, Range);
8061
8062 if (StoreInst *SI = dyn_cast<StoreInst>(Instr))
8063 if (auto HistInfo = Legal->getHistogramInfo(SI))
8064 return tryToWidenHistogram(*HistInfo, Operands);
8065
8066 if (isa<LoadInst>(Instr) || isa<StoreInst>(Instr))
8067 return tryToWidenMemory(Instr, Operands, Range);
8068
8069 if (std::optional<unsigned> ScaleFactor = getScalingForReduction(Instr)) {
8070 if (auto PartialRed =
8071 tryToCreatePartialReduction(Instr, Operands, ScaleFactor.value()))
8072 return PartialRed;
8073 }
8074
8075 if (!shouldWiden(Instr, Range))
8076 return nullptr;
8077
8078 if (auto *GEP = dyn_cast<GetElementPtrInst>(Instr))
8079 return new VPWidenGEPRecipe(GEP, Operands);
8080
8081 if (auto *SI = dyn_cast<SelectInst>(Instr)) {
8082 return new VPWidenSelectRecipe(*SI, Operands);
8083 }
8084
8085 if (auto *CI = dyn_cast<CastInst>(Instr)) {
8086 return new VPWidenCastRecipe(CI->getOpcode(), Operands[0], CI->getType(),
8087 *CI);
8088 }
8089
8090 return tryToWiden(Instr, Operands);
8091}
8092
8096 unsigned ScaleFactor) {
8097 assert(Operands.size() == 2 &&
8098 "Unexpected number of operands for partial reduction");
8099
8100 VPValue *BinOp = Operands[0];
8102 VPRecipeBase *BinOpRecipe = BinOp->getDefiningRecipe();
8103 if (isa<VPReductionPHIRecipe>(BinOpRecipe) ||
8104 isa<VPPartialReductionRecipe>(BinOpRecipe))
8105 std::swap(BinOp, Accumulator);
8106
8107 if (ScaleFactor !=
8108 vputils::getVFScaleFactor(Accumulator->getDefiningRecipe()))
8109 return nullptr;
8110
8111 unsigned ReductionOpcode = Reduction->getOpcode();
8112 if (ReductionOpcode == Instruction::Sub) {
8113 auto *const Zero = ConstantInt::get(Reduction->getType(), 0);
8115 Ops.push_back(Plan.getOrAddLiveIn(Zero));
8116 Ops.push_back(BinOp);
8117 BinOp = new VPWidenRecipe(*Reduction, Ops);
8118 Builder.insert(BinOp->getDefiningRecipe());
8119 ReductionOpcode = Instruction::Add;
8120 }
8121
8122 VPValue *Cond = nullptr;
8123 if (CM.blockNeedsPredicationForAnyReason(Reduction->getParent())) {
8124 assert((ReductionOpcode == Instruction::Add ||
8125 ReductionOpcode == Instruction::Sub) &&
8126 "Expected an ADD or SUB operation for predicated partial "
8127 "reductions (because the neutral element in the mask is zero)!");
8128 Cond = getBlockInMask(Builder.getInsertBlock());
8129 VPValue *Zero =
8130 Plan.getOrAddLiveIn(ConstantInt::get(Reduction->getType(), 0));
8131 BinOp = Builder.createSelect(Cond, BinOp, Zero, Reduction->getDebugLoc());
8132 }
8133 return new VPPartialReductionRecipe(ReductionOpcode, Accumulator, BinOp, Cond,
8134 ScaleFactor, Reduction);
8135}
8136
8137void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
8138 ElementCount MaxVF) {
8139 if (ElementCount::isKnownGT(MinVF, MaxVF))
8140 return;
8141
8142 assert(OrigLoop->isInnermost() && "Inner loop expected.");
8143
8144 const LoopAccessInfo *LAI = Legal->getLAI();
8146 OrigLoop, LI, DT, PSE.getSE());
8147 if (!LAI->getRuntimePointerChecking()->getChecks().empty() &&
8149 // Only use noalias metadata when using memory checks guaranteeing no
8150 // overlap across all iterations.
8151 LVer.prepareNoAliasMetadata();
8152 }
8153
8154 // Create initial base VPlan0, to serve as common starting point for all
8155 // candidates built later for specific VF ranges.
8156 auto VPlan0 = VPlanTransforms::buildVPlan0(
8157 OrigLoop, *LI, Legal->getWidestInductionType(),
8158 getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()), PSE);
8159
8160 auto MaxVFTimes2 = MaxVF * 2;
8161 for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) {
8162 VFRange SubRange = {VF, MaxVFTimes2};
8163 if (auto Plan = tryToBuildVPlanWithVPRecipes(
8164 std::unique_ptr<VPlan>(VPlan0->duplicate()), SubRange, &LVer)) {
8165 // Now optimize the initial VPlan.
8167 *Plan, CM.getMinimalBitwidths());
8169 // TODO: try to put it close to addActiveLaneMask().
8170 if (CM.foldTailWithEVL())
8172 *Plan, CM.getMaxSafeElements());
8173 assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
8174 VPlans.push_back(std::move(Plan));
8175 }
8176 VF = SubRange.End;
8177 }
8178}
8179
8180/// Create and return a ResumePhi for \p WideIV, unless it is truncated. If the
8181/// induction recipe is not canonical, creates a VPDerivedIVRecipe to compute
8182/// the end value of the induction.
8184 VPWidenInductionRecipe *WideIV, VPBuilder &VectorPHBuilder,
8185 VPBuilder &ScalarPHBuilder, VPTypeAnalysis &TypeInfo, VPValue *VectorTC) {
8186 auto *WideIntOrFp = dyn_cast<VPWidenIntOrFpInductionRecipe>(WideIV);
8187 // Truncated wide inductions resume from the last lane of their vector value
8188 // in the last vector iteration which is handled elsewhere.
8189 if (WideIntOrFp && WideIntOrFp->getTruncInst())
8190 return nullptr;
8191
8192 VPValue *Start = WideIV->getStartValue();
8193 VPValue *Step = WideIV->getStepValue();
8195 VPValue *EndValue = VectorTC;
8196 if (!WideIntOrFp || !WideIntOrFp->isCanonical()) {
8197 EndValue = VectorPHBuilder.createDerivedIV(
8198 ID.getKind(), dyn_cast_or_null<FPMathOperator>(ID.getInductionBinOp()),
8199 Start, VectorTC, Step);
8200 }
8201
8202 // EndValue is derived from the vector trip count (which has the same type as
8203 // the widest induction) and thus may be wider than the induction here.
8204 Type *ScalarTypeOfWideIV = TypeInfo.inferScalarType(WideIV);
8205 if (ScalarTypeOfWideIV != TypeInfo.inferScalarType(EndValue)) {
8206 EndValue = VectorPHBuilder.createScalarCast(Instruction::Trunc, EndValue,
8207 ScalarTypeOfWideIV,
8208 WideIV->getDebugLoc());
8209 }
8210
8211 auto *ResumePhiRecipe = ScalarPHBuilder.createScalarPhi(
8212 {EndValue, Start}, WideIV->getDebugLoc(), "bc.resume.val");
8213 return ResumePhiRecipe;
8214}
8215
8216/// Create resume phis in the scalar preheader for first-order recurrences,
8217/// reductions and inductions, and update the VPIRInstructions wrapping the
8218/// original phis in the scalar header. End values for inductions are added to
8219/// \p IVEndValues.
8220static void addScalarResumePhis(VPRecipeBuilder &Builder, VPlan &Plan,
8221 DenseMap<VPValue *, VPValue *> &IVEndValues) {
8222 VPTypeAnalysis TypeInfo(Plan);
8223 auto *ScalarPH = Plan.getScalarPreheader();
8224 auto *MiddleVPBB = cast<VPBasicBlock>(ScalarPH->getPredecessors()[0]);
8225 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
8226 VPBuilder VectorPHBuilder(
8227 cast<VPBasicBlock>(VectorRegion->getSinglePredecessor()));
8228 VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
8229 VPBuilder ScalarPHBuilder(ScalarPH);
8230 for (VPRecipeBase &ScalarPhiR : Plan.getScalarHeader()->phis()) {
8231 auto *ScalarPhiIRI = cast<VPIRPhi>(&ScalarPhiR);
8232
8233 // TODO: Extract final value from induction recipe initially, optimize to
8234 // pre-computed end value together in optimizeInductionExitUsers.
8235 auto *VectorPhiR =
8236 cast<VPHeaderPHIRecipe>(Builder.getRecipe(&ScalarPhiIRI->getIRPhi()));
8237 if (auto *WideIVR = dyn_cast<VPWidenInductionRecipe>(VectorPhiR)) {
8239 WideIVR, VectorPHBuilder, ScalarPHBuilder, TypeInfo,
8240 &Plan.getVectorTripCount())) {
8241 assert(isa<VPPhi>(ResumePhi) && "Expected a phi");
8242 IVEndValues[WideIVR] = ResumePhi->getOperand(0);
8243 ScalarPhiIRI->addOperand(ResumePhi);
8244 continue;
8245 }
8246 // TODO: Also handle truncated inductions here. Computing end-values
8247 // separately should be done as VPlan-to-VPlan optimization, after
8248 // legalizing all resume values to use the last lane from the loop.
8249 assert(cast<VPWidenIntOrFpInductionRecipe>(VectorPhiR)->getTruncInst() &&
8250 "should only skip truncated wide inductions");
8251 continue;
8252 }
8253
8254 // The backedge value provides the value to resume coming out of a loop,
8255 // which for FORs is a vector whose last element needs to be extracted. The
8256 // start value provides the value if the loop is bypassed.
8257 bool IsFOR = isa<VPFirstOrderRecurrencePHIRecipe>(VectorPhiR);
8258 auto *ResumeFromVectorLoop = VectorPhiR->getBackedgeValue();
8259 assert(VectorRegion->getSingleSuccessor() == Plan.getMiddleBlock() &&
8260 "Cannot handle loops with uncountable early exits");
8261 if (IsFOR)
8262 ResumeFromVectorLoop = MiddleBuilder.createNaryOp(
8263 VPInstruction::ExtractLastElement, {ResumeFromVectorLoop}, {},
8264 "vector.recur.extract");
8265 StringRef Name = IsFOR ? "scalar.recur.init" : "bc.merge.rdx";
8266 auto *ResumePhiR = ScalarPHBuilder.createScalarPhi(
8267 {ResumeFromVectorLoop, VectorPhiR->getStartValue()}, {}, Name);
8268 ScalarPhiIRI->addOperand(ResumePhiR);
8269 }
8270}
8271
8272/// Handle users in the exit block for first order reductions in the original
8273/// exit block. The penultimate value of recurrences is fed to their LCSSA phi
8274/// users in the original exit block using the VPIRInstruction wrapping to the
8275/// LCSSA phi.
8277 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
8278 auto *ScalarPHVPBB = Plan.getScalarPreheader();
8279 auto *MiddleVPBB = Plan.getMiddleBlock();
8280 VPBuilder ScalarPHBuilder(ScalarPHVPBB);
8281 VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
8282
8283 auto IsScalableOne = [](ElementCount VF) -> bool {
8284 return VF == ElementCount::getScalable(1);
8285 };
8286
8287 for (auto &HeaderPhi : VectorRegion->getEntryBasicBlock()->phis()) {
8288 auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(&HeaderPhi);
8289 if (!FOR)
8290 continue;
8291
8292 assert(VectorRegion->getSingleSuccessor() == Plan.getMiddleBlock() &&
8293 "Cannot handle loops with uncountable early exits");
8294
8295 // This is the second phase of vectorizing first-order recurrences, creating
8296 // extract for users outside the loop. An overview of the transformation is
8297 // described below. Suppose we have the following loop with some use after
8298 // the loop of the last a[i-1],
8299 //
8300 // for (int i = 0; i < n; ++i) {
8301 // t = a[i - 1];
8302 // b[i] = a[i] - t;
8303 // }
8304 // use t;
8305 //
8306 // There is a first-order recurrence on "a". For this loop, the shorthand
8307 // scalar IR looks like:
8308 //
8309 // scalar.ph:
8310 // s.init = a[-1]
8311 // br scalar.body
8312 //
8313 // scalar.body:
8314 // i = phi [0, scalar.ph], [i+1, scalar.body]
8315 // s1 = phi [s.init, scalar.ph], [s2, scalar.body]
8316 // s2 = a[i]
8317 // b[i] = s2 - s1
8318 // br cond, scalar.body, exit.block
8319 //
8320 // exit.block:
8321 // use = lcssa.phi [s1, scalar.body]
8322 //
8323 // In this example, s1 is a recurrence because it's value depends on the
8324 // previous iteration. In the first phase of vectorization, we created a
8325 // VPFirstOrderRecurrencePHIRecipe v1 for s1. Now we create the extracts
8326 // for users in the scalar preheader and exit block.
8327 //
8328 // vector.ph:
8329 // v_init = vector(..., ..., ..., a[-1])
8330 // br vector.body
8331 //
8332 // vector.body
8333 // i = phi [0, vector.ph], [i+4, vector.body]
8334 // v1 = phi [v_init, vector.ph], [v2, vector.body]
8335 // v2 = a[i, i+1, i+2, i+3]
8336 // b[i] = v2 - v1
8337 // // Next, third phase will introduce v1' = splice(v1(3), v2(0, 1, 2))
8338 // b[i, i+1, i+2, i+3] = v2 - v1
8339 // br cond, vector.body, middle.block
8340 //
8341 // middle.block:
8342 // vector.recur.extract.for.phi = v2(2)
8343 // vector.recur.extract = v2(3)
8344 // br cond, scalar.ph, exit.block
8345 //
8346 // scalar.ph:
8347 // scalar.recur.init = phi [vector.recur.extract, middle.block],
8348 // [s.init, otherwise]
8349 // br scalar.body
8350 //
8351 // scalar.body:
8352 // i = phi [0, scalar.ph], [i+1, scalar.body]
8353 // s1 = phi [scalar.recur.init, scalar.ph], [s2, scalar.body]
8354 // s2 = a[i]
8355 // b[i] = s2 - s1
8356 // br cond, scalar.body, exit.block
8357 //
8358 // exit.block:
8359 // lo = lcssa.phi [s1, scalar.body],
8360 // [vector.recur.extract.for.phi, middle.block]
8361 //
8362 // Now update VPIRInstructions modeling LCSSA phis in the exit block.
8363 // Extract the penultimate value of the recurrence and use it as operand for
8364 // the VPIRInstruction modeling the phi.
8365 for (VPUser *U : FOR->users()) {
8366 using namespace llvm::VPlanPatternMatch;
8367 if (!match(U, m_ExtractLastElement(m_Specific(FOR))))
8368 continue;
8369 // For VF vscale x 1, if vscale = 1, we are unable to extract the
8370 // penultimate value of the recurrence. Instead we rely on the existing
8371 // extract of the last element from the result of
8372 // VPInstruction::FirstOrderRecurrenceSplice.
8373 // TODO: Consider vscale_range info and UF.
8375 Range))
8376 return;
8377 VPValue *PenultimateElement = MiddleBuilder.createNaryOp(
8378 VPInstruction::ExtractPenultimateElement, {FOR->getBackedgeValue()},
8379 {}, "vector.recur.extract.for.phi");
8380 cast<VPInstruction>(U)->replaceAllUsesWith(PenultimateElement);
8381 }
8382 }
8383}
8384
8385VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
8386 VPlanPtr Plan, VFRange &Range, LoopVersioning *LVer) {
8387
8388 using namespace llvm::VPlanPatternMatch;
8389 SmallPtrSet<const InterleaveGroup<Instruction> *, 1> InterleaveGroups;
8390
8391 // ---------------------------------------------------------------------------
8392 // Build initial VPlan: Scan the body of the loop in a topological order to
8393 // visit each basic block after having visited its predecessor basic blocks.
8394 // ---------------------------------------------------------------------------
8395
8396 bool RequiresScalarEpilogueCheck =
8398 [this](ElementCount VF) {
8399 return !CM.requiresScalarEpilogue(VF.isVector());
8400 },
8401 Range);
8402 VPlanTransforms::handleEarlyExits(*Plan, Legal->hasUncountableEarlyExit());
8403 VPlanTransforms::addMiddleCheck(*Plan, RequiresScalarEpilogueCheck,
8404 CM.foldTailByMasking());
8405
8407
8408 // Don't use getDecisionAndClampRange here, because we don't know the UF
8409 // so this function is better to be conservative, rather than to split
8410 // it up into different VPlans.
8411 // TODO: Consider using getDecisionAndClampRange here to split up VPlans.
8412 bool IVUpdateMayOverflow = false;
8413 for (ElementCount VF : Range)
8414 IVUpdateMayOverflow |= !isIndvarOverflowCheckKnownFalse(&CM, VF);
8415
8416 TailFoldingStyle Style = CM.getTailFoldingStyle(IVUpdateMayOverflow);
8417 // Use NUW for the induction increment if we proved that it won't overflow in
8418 // the vector loop or when not folding the tail. In the later case, we know
8419 // that the canonical induction increment will not overflow as the vector trip
8420 // count is >= increment and a multiple of the increment.
8421 bool HasNUW = !IVUpdateMayOverflow || Style == TailFoldingStyle::None;
8422 if (!HasNUW) {
8423 auto *IVInc = Plan->getVectorLoopRegion()
8424 ->getExitingBasicBlock()
8425 ->getTerminator()
8426 ->getOperand(0);
8427 assert(match(IVInc, m_VPInstruction<Instruction::Add>(
8428 m_Specific(Plan->getCanonicalIV()), m_VPValue())) &&
8429 "Did not find the canonical IV increment");
8430 cast<VPRecipeWithIRFlags>(IVInc)->dropPoisonGeneratingFlags();
8431 }
8432
8433 // ---------------------------------------------------------------------------
8434 // Pre-construction: record ingredients whose recipes we'll need to further
8435 // process after constructing the initial VPlan.
8436 // ---------------------------------------------------------------------------
8437
8438 // For each interleave group which is relevant for this (possibly trimmed)
8439 // Range, add it to the set of groups to be later applied to the VPlan and add
8440 // placeholders for its members' Recipes which we'll be replacing with a
8441 // single VPInterleaveRecipe.
8442 for (InterleaveGroup<Instruction> *IG : IAI.getInterleaveGroups()) {
8443 auto ApplyIG = [IG, this](ElementCount VF) -> bool {
8444 bool Result = (VF.isVector() && // Query is illegal for VF == 1
8445 CM.getWideningDecision(IG->getInsertPos(), VF) ==
8447 // For scalable vectors, the interleave factors must be <= 8 since we
8448 // require the (de)interleaveN intrinsics instead of shufflevectors.
8449 assert((!Result || !VF.isScalable() || IG->getFactor() <= 8) &&
8450 "Unsupported interleave factor for scalable vectors");
8451 return Result;
8452 };
8453 if (!getDecisionAndClampRange(ApplyIG, Range))
8454 continue;
8455 InterleaveGroups.insert(IG);
8456 }
8457
8458 // ---------------------------------------------------------------------------
8459 // Predicate and linearize the top-level loop region.
8460 // ---------------------------------------------------------------------------
8461 auto BlockMaskCache = VPlanTransforms::introduceMasksAndLinearize(
8462 *Plan, CM.foldTailByMasking());
8463
8464 // ---------------------------------------------------------------------------
8465 // Construct wide recipes and apply predication for original scalar
8466 // VPInstructions in the loop.
8467 // ---------------------------------------------------------------------------
8468 VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
8469 Builder, BlockMaskCache, LVer);
8470 RecipeBuilder.collectScaledReductions(Range);
8471
8472 // Scan the body of the loop in a topological order to visit each basic block
8473 // after having visited its predecessor basic blocks.
8474 VPRegionBlock *LoopRegion = Plan->getVectorLoopRegion();
8475 VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
8476 ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
8477 HeaderVPBB);
8478
8479 auto *MiddleVPBB = Plan->getMiddleBlock();
8480 VPBasicBlock::iterator MBIP = MiddleVPBB->getFirstNonPhi();
8481 // Mapping from VPValues in the initial plan to their widened VPValues. Needed
8482 // temporarily to update created block masks.
8483 DenseMap<VPValue *, VPValue *> Old2New;
8484 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
8485 // Convert input VPInstructions to widened recipes.
8486 for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
8487 auto *SingleDef = cast<VPSingleDefRecipe>(&R);
8488 auto *UnderlyingValue = SingleDef->getUnderlyingValue();
8489 // Skip recipes that do not need transforming, including canonical IV,
8490 // wide canonical IV and VPInstructions without underlying values. The
8491 // latter are added above for masking.
8492 // FIXME: Migrate code relying on the underlying instruction from VPlan0
8493 // to construct recipes below to not use the underlying instruction.
8495 &R) ||
8496 (isa<VPInstruction>(&R) && !UnderlyingValue))
8497 continue;
8498
8499 // FIXME: VPlan0, which models a copy of the original scalar loop, should
8500 // not use VPWidenPHIRecipe to model the phis.
8502 UnderlyingValue && "unsupported recipe");
8503
8504 // TODO: Gradually replace uses of underlying instruction by analyses on
8505 // VPlan.
8506 Instruction *Instr = cast<Instruction>(UnderlyingValue);
8507 Builder.setInsertPoint(SingleDef);
8508
8509 // The stores with invariant address inside the loop will be deleted, and
8510 // in the exit block, a uniform store recipe will be created for the final
8511 // invariant store of the reduction.
8512 StoreInst *SI;
8513 if ((SI = dyn_cast<StoreInst>(Instr)) &&
8514 Legal->isInvariantAddressOfReduction(SI->getPointerOperand())) {
8515 // Only create recipe for the final invariant store of the reduction.
8516 if (Legal->isInvariantStoreOfReduction(SI)) {
8517 auto *Recipe =
8518 new VPReplicateRecipe(SI, R.operands(), true /* IsUniform */,
8519 nullptr /*Mask*/, VPIRMetadata(*SI, LVer));
8520 Recipe->insertBefore(*MiddleVPBB, MBIP);
8521 }
8522 R.eraseFromParent();
8523 continue;
8524 }
8525
8526 VPRecipeBase *Recipe =
8527 RecipeBuilder.tryToCreateWidenRecipe(SingleDef, Range);
8528 if (!Recipe)
8529 Recipe = RecipeBuilder.handleReplication(Instr, R.operands(), Range);
8530
8531 RecipeBuilder.setRecipe(Instr, Recipe);
8532 if (isa<VPWidenIntOrFpInductionRecipe>(Recipe) && isa<TruncInst>(Instr)) {
8533 // Optimized a truncate to VPWidenIntOrFpInductionRecipe. It needs to be
8534 // moved to the phi section in the header.
8535 Recipe->insertBefore(*HeaderVPBB, HeaderVPBB->getFirstNonPhi());
8536 } else {
8537 Builder.insert(Recipe);
8538 }
8539 if (Recipe->getNumDefinedValues() == 1) {
8540 SingleDef->replaceAllUsesWith(Recipe->getVPSingleValue());
8541 Old2New[SingleDef] = Recipe->getVPSingleValue();
8542 } else {
8543 assert(Recipe->getNumDefinedValues() == 0 &&
8544 "Unexpected multidef recipe");
8545 R.eraseFromParent();
8546 }
8547 }
8548 }
8549
8550 // replaceAllUsesWith above may invalidate the block masks. Update them here.
8551 // TODO: Include the masks as operands in the predicated VPlan directly
8552 // to remove the need to keep a map of masks beyond the predication
8553 // transform.
8554 RecipeBuilder.updateBlockMaskCache(Old2New);
8555 for (VPValue *Old : Old2New.keys())
8556 Old->getDefiningRecipe()->eraseFromParent();
8557
8558 assert(isa<VPRegionBlock>(Plan->getVectorLoopRegion()) &&
8559 !Plan->getVectorLoopRegion()->getEntryBasicBlock()->empty() &&
8560 "entry block must be set to a VPRegionBlock having a non-empty entry "
8561 "VPBasicBlock");
8562
8563 // Update wide induction increments to use the same step as the corresponding
8564 // wide induction. This enables detecting induction increments directly in
8565 // VPlan and removes redundant splats.
8566 for (const auto &[Phi, ID] : Legal->getInductionVars()) {
8567 auto *IVInc = cast<Instruction>(
8568 Phi->getIncomingValueForBlock(OrigLoop->getLoopLatch()));
8569 if (IVInc->getOperand(0) != Phi || IVInc->getOpcode() != Instruction::Add)
8570 continue;
8571 VPWidenInductionRecipe *WideIV =
8572 cast<VPWidenInductionRecipe>(RecipeBuilder.getRecipe(Phi));
8573 VPRecipeBase *R = RecipeBuilder.getRecipe(IVInc);
8574 R->setOperand(1, WideIV->getStepValue());
8575 }
8576
8578 DenseMap<VPValue *, VPValue *> IVEndValues;
8579 addScalarResumePhis(RecipeBuilder, *Plan, IVEndValues);
8580
8581 // ---------------------------------------------------------------------------
8582 // Transform initial VPlan: Apply previously taken decisions, in order, to
8583 // bring the VPlan to its final state.
8584 // ---------------------------------------------------------------------------
8585
8586 // Adjust the recipes for any inloop reductions.
8587 adjustRecipesForReductions(Plan, RecipeBuilder, Range.Start);
8588
8589 // Apply mandatory transformation to handle FP maxnum/minnum reduction with
8590 // NaNs if possible, bail out otherwise.
8592 *Plan))
8593 return nullptr;
8594
8595 // Transform recipes to abstract recipes if it is legal and beneficial and
8596 // clamp the range for better cost estimation.
8597 // TODO: Enable following transform when the EVL-version of extended-reduction
8598 // and mulacc-reduction are implemented.
8599 if (!CM.foldTailWithEVL()) {
8600 VPCostContext CostCtx(CM.TTI, *CM.TLI, *Plan, CM, CM.CostKind);
8602 CostCtx, Range);
8603 }
8604
8605 for (ElementCount VF : Range)
8606 Plan->addVF(VF);
8607 Plan->setName("Initial VPlan");
8608
8609 // Interleave memory: for each Interleave Group we marked earlier as relevant
8610 // for this VPlan, replace the Recipes widening its memory instructions with a
8611 // single VPInterleaveRecipe at its insertion point.
8613 InterleaveGroups, RecipeBuilder,
8614 CM.isScalarEpilogueAllowed());
8615
8616 // Replace VPValues for known constant strides.
8618 Legal->getLAI()->getSymbolicStrides());
8619
8620 auto BlockNeedsPredication = [this](BasicBlock *BB) {
8621 return Legal->blockNeedsPredication(BB);
8622 };
8624 BlockNeedsPredication);
8625
8626 // Sink users of fixed-order recurrence past the recipe defining the previous
8627 // value and introduce FirstOrderRecurrenceSplice VPInstructions.
8629 *Plan, Builder))
8630 return nullptr;
8631
8632 if (useActiveLaneMask(Style)) {
8633 // TODO: Move checks to VPlanTransforms::addActiveLaneMask once
8634 // TailFoldingStyle is visible there.
8635 bool ForControlFlow = useActiveLaneMaskForControlFlow(Style);
8636 bool WithoutRuntimeCheck =
8637 Style == TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck;
8638 VPlanTransforms::addActiveLaneMask(*Plan, ForControlFlow,
8639 WithoutRuntimeCheck);
8640 }
8641 VPlanTransforms::optimizeInductionExitUsers(*Plan, IVEndValues, *PSE.getSE());
8642
8643 assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
8644 return Plan;
8645}
8646
8647VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
8648 // Outer loop handling: They may require CFG and instruction level
8649 // transformations before even evaluating whether vectorization is profitable.
8650 // Since we cannot modify the incoming IR, we need to build VPlan upfront in
8651 // the vectorization pipeline.
8652 assert(!OrigLoop->isInnermost());
8653 assert(EnableVPlanNativePath && "VPlan-native path is not enabled.");
8654
8655 auto Plan = VPlanTransforms::buildVPlan0(
8656 OrigLoop, *LI, Legal->getWidestInductionType(),
8657 getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()), PSE);
8659 /*HasUncountableExit*/ false);
8660 VPlanTransforms::addMiddleCheck(*Plan, /*RequiresScalarEpilogue*/ true,
8661 /*TailFolded*/ false);
8662
8664
8665 for (ElementCount VF : Range)
8666 Plan->addVF(VF);
8667
8669 Plan,
8670 [this](PHINode *P) {
8671 return Legal->getIntOrFpInductionDescriptor(P);
8672 },
8673 *TLI))
8674 return nullptr;
8675
8676 // Collect mapping of IR header phis to header phi recipes, to be used in
8677 // addScalarResumePhis.
8678 DenseMap<VPBasicBlock *, VPValue *> BlockMaskCache;
8679 VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
8680 Builder, BlockMaskCache, nullptr /*LVer*/);
8681 for (auto &R : Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
8683 continue;
8684 auto *HeaderR = cast<VPHeaderPHIRecipe>(&R);
8685 RecipeBuilder.setRecipe(HeaderR->getUnderlyingInstr(), HeaderR);
8686 }
8687 DenseMap<VPValue *, VPValue *> IVEndValues;
8688 // TODO: IVEndValues are not used yet in the native path, to optimize exit
8689 // values.
8690 addScalarResumePhis(RecipeBuilder, *Plan, IVEndValues);
8691
8692 assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
8693 return Plan;
8694}
8695
8696// Adjust the recipes for reductions. For in-loop reductions the chain of
8697// instructions leading from the loop exit instr to the phi need to be converted
8698// to reductions, with one operand being vector and the other being the scalar
8699// reduction chain. For other reductions, a select is introduced between the phi
8700// and users outside the vector region when folding the tail.
8701//
8702// A ComputeReductionResult recipe is added to the middle block, also for
8703// in-loop reductions which compute their result in-loop, because generating
8704// the subsequent bc.merge.rdx phi is driven by ComputeReductionResult recipes.
8705//
8706// Adjust AnyOf reductions; replace the reduction phi for the selected value
8707// with a boolean reduction phi node to check if the condition is true in any
8708// iteration. The final value is selected by the final ComputeReductionResult.
8709void LoopVectorizationPlanner::adjustRecipesForReductions(
8710 VPlanPtr &Plan, VPRecipeBuilder &RecipeBuilder, ElementCount MinVF) {
8711 using namespace VPlanPatternMatch;
8712 VPRegionBlock *VectorLoopRegion = Plan->getVectorLoopRegion();
8713 VPBasicBlock *Header = VectorLoopRegion->getEntryBasicBlock();
8714 VPBasicBlock *MiddleVPBB = Plan->getMiddleBlock();
8716
8717 for (VPRecipeBase &R : Header->phis()) {
8718 auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
8719 if (!PhiR || !PhiR->isInLoop() || (MinVF.isScalar() && !PhiR->isOrdered()))
8720 continue;
8721
8722 RecurKind Kind = PhiR->getRecurrenceKind();
8723 assert(
8726 "AnyOf and FindIV reductions are not allowed for in-loop reductions");
8727
8728 // Collect the chain of "link" recipes for the reduction starting at PhiR.
8729 SetVector<VPSingleDefRecipe *> Worklist;
8730 Worklist.insert(PhiR);
8731 for (unsigned I = 0; I != Worklist.size(); ++I) {
8732 VPSingleDefRecipe *Cur = Worklist[I];
8733 for (VPUser *U : Cur->users()) {
8734 auto *UserRecipe = cast<VPSingleDefRecipe>(U);
8735 if (!UserRecipe->getParent()->getEnclosingLoopRegion()) {
8736 assert((UserRecipe->getParent() == MiddleVPBB ||
8737 UserRecipe->getParent() == Plan->getScalarPreheader()) &&
8738 "U must be either in the loop region, the middle block or the "
8739 "scalar preheader.");
8740 continue;
8741 }
8742 Worklist.insert(UserRecipe);
8743 }
8744 }
8745
8746 // Visit operation "Links" along the reduction chain top-down starting from
8747 // the phi until LoopExitValue. We keep track of the previous item
8748 // (PreviousLink) to tell which of the two operands of a Link will remain
8749 // scalar and which will be reduced. For minmax by select(cmp), Link will be
8750 // the select instructions. Blend recipes of in-loop reduction phi's will
8751 // get folded to their non-phi operand, as the reduction recipe handles the
8752 // condition directly.
8753 VPSingleDefRecipe *PreviousLink = PhiR; // Aka Worklist[0].
8754 for (VPSingleDefRecipe *CurrentLink : drop_begin(Worklist)) {
8755 if (auto *Blend = dyn_cast<VPBlendRecipe>(CurrentLink)) {
8756 assert(Blend->getNumIncomingValues() == 2 &&
8757 "Blend must have 2 incoming values");
8758 if (Blend->getIncomingValue(0) == PhiR) {
8759 Blend->replaceAllUsesWith(Blend->getIncomingValue(1));
8760 } else {
8761 assert(Blend->getIncomingValue(1) == PhiR &&
8762 "PhiR must be an operand of the blend");
8763 Blend->replaceAllUsesWith(Blend->getIncomingValue(0));
8764 }
8765 continue;
8766 }
8767
8768 Instruction *CurrentLinkI = CurrentLink->getUnderlyingInstr();
8769
8770 // Index of the first operand which holds a non-mask vector operand.
8771 unsigned IndexOfFirstOperand;
8772 // Recognize a call to the llvm.fmuladd intrinsic.
8773 bool IsFMulAdd = (Kind == RecurKind::FMulAdd);
8774 VPValue *VecOp;
8775 VPBasicBlock *LinkVPBB = CurrentLink->getParent();
8776 if (IsFMulAdd) {
8777 assert(
8779 "Expected instruction to be a call to the llvm.fmuladd intrinsic");
8780 assert(((MinVF.isScalar() && isa<VPReplicateRecipe>(CurrentLink)) ||
8781 isa<VPWidenIntrinsicRecipe>(CurrentLink)) &&
8782 CurrentLink->getOperand(2) == PreviousLink &&
8783 "expected a call where the previous link is the added operand");
8784
8785 // If the instruction is a call to the llvm.fmuladd intrinsic then we
8786 // need to create an fmul recipe (multiplying the first two operands of
8787 // the fmuladd together) to use as the vector operand for the fadd
8788 // reduction.
8789 VPInstruction *FMulRecipe = new VPInstruction(
8790 Instruction::FMul,
8791 {CurrentLink->getOperand(0), CurrentLink->getOperand(1)},
8792 CurrentLinkI->getFastMathFlags());
8793 LinkVPBB->insert(FMulRecipe, CurrentLink->getIterator());
8794 VecOp = FMulRecipe;
8795 } else if (PhiR->isInLoop() && Kind == RecurKind::AddChainWithSubs &&
8796 CurrentLinkI->getOpcode() == Instruction::Sub) {
8797 Type *PhiTy = PhiR->getUnderlyingValue()->getType();
8798 auto *Zero = Plan->getOrAddLiveIn(ConstantInt::get(PhiTy, 0));
8799 VPWidenRecipe *Sub = new VPWidenRecipe(
8800 Instruction::Sub, {Zero, CurrentLink->getOperand(1)}, {},
8801 VPIRMetadata(), CurrentLinkI->getDebugLoc());
8802 Sub->setUnderlyingValue(CurrentLinkI);
8803 LinkVPBB->insert(Sub, CurrentLink->getIterator());
8804 VecOp = Sub;
8805 } else {
8807 if (isa<VPWidenRecipe>(CurrentLink)) {
8808 assert(isa<CmpInst>(CurrentLinkI) &&
8809 "need to have the compare of the select");
8810 continue;
8811 }
8812 assert(isa<VPWidenSelectRecipe>(CurrentLink) &&
8813 "must be a select recipe");
8814 IndexOfFirstOperand = 1;
8815 } else {
8816 assert((MinVF.isScalar() || isa<VPWidenRecipe>(CurrentLink)) &&
8817 "Expected to replace a VPWidenSC");
8818 IndexOfFirstOperand = 0;
8819 }
8820 // Note that for non-commutable operands (cmp-selects), the semantics of
8821 // the cmp-select are captured in the recurrence kind.
8822 unsigned VecOpId =
8823 CurrentLink->getOperand(IndexOfFirstOperand) == PreviousLink
8824 ? IndexOfFirstOperand + 1
8825 : IndexOfFirstOperand;
8826 VecOp = CurrentLink->getOperand(VecOpId);
8827 assert(VecOp != PreviousLink &&
8828 CurrentLink->getOperand(CurrentLink->getNumOperands() - 1 -
8829 (VecOpId - IndexOfFirstOperand)) ==
8830 PreviousLink &&
8831 "PreviousLink must be the operand other than VecOp");
8832 }
8833
8834 VPValue *CondOp = nullptr;
8835 if (CM.blockNeedsPredicationForAnyReason(CurrentLinkI->getParent()))
8836 CondOp = RecipeBuilder.getBlockInMask(CurrentLink->getParent());
8837
8838 // TODO: Retrieve FMFs from recipes directly.
8839 RecurrenceDescriptor RdxDesc = Legal->getRecurrenceDescriptor(
8840 cast<PHINode>(PhiR->getUnderlyingInstr()));
8841 // Non-FP RdxDescs will have all fast math flags set, so clear them.
8842 FastMathFlags FMFs = isa<FPMathOperator>(CurrentLinkI)
8843 ? RdxDesc.getFastMathFlags()
8844 : FastMathFlags();
8845 auto *RedRecipe = new VPReductionRecipe(
8846 Kind, FMFs, CurrentLinkI, PreviousLink, VecOp, CondOp,
8847 PhiR->isOrdered(), CurrentLinkI->getDebugLoc());
8848 // Append the recipe to the end of the VPBasicBlock because we need to
8849 // ensure that it comes after all of it's inputs, including CondOp.
8850 // Delete CurrentLink as it will be invalid if its operand is replaced
8851 // with a reduction defined at the bottom of the block in the next link.
8852 if (LinkVPBB->getNumSuccessors() == 0)
8853 RedRecipe->insertBefore(&*std::prev(std::prev(LinkVPBB->end())));
8854 else
8855 LinkVPBB->appendRecipe(RedRecipe);
8856
8857 CurrentLink->replaceAllUsesWith(RedRecipe);
8858 ToDelete.push_back(CurrentLink);
8859 PreviousLink = RedRecipe;
8860 }
8861 }
8862 VPBasicBlock *LatchVPBB = VectorLoopRegion->getExitingBasicBlock();
8863 Builder.setInsertPoint(&*std::prev(std::prev(LatchVPBB->end())));
8864 VPBasicBlock::iterator IP = MiddleVPBB->getFirstNonPhi();
8865 for (VPRecipeBase &R :
8866 Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
8867 VPReductionPHIRecipe *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
8868 if (!PhiR)
8869 continue;
8870
8871 const RecurrenceDescriptor &RdxDesc = Legal->getRecurrenceDescriptor(
8873 Type *PhiTy = PhiR->getUnderlyingValue()->getType();
8874 // If tail is folded by masking, introduce selects between the phi
8875 // and the users outside the vector region of each reduction, at the
8876 // beginning of the dedicated latch block.
8877 auto *OrigExitingVPV = PhiR->getBackedgeValue();
8878 auto *NewExitingVPV = PhiR->getBackedgeValue();
8879 // Don't output selects for partial reductions because they have an output
8880 // with fewer lanes than the VF. So the operands of the select would have
8881 // different numbers of lanes. Partial reductions mask the input instead.
8882 if (!PhiR->isInLoop() && CM.foldTailByMasking() &&
8883 !isa<VPPartialReductionRecipe>(OrigExitingVPV->getDefiningRecipe())) {
8884 VPValue *Cond = RecipeBuilder.getBlockInMask(PhiR->getParent());
8885 std::optional<FastMathFlags> FMFs =
8886 PhiTy->isFloatingPointTy()
8887 ? std::make_optional(RdxDesc.getFastMathFlags())
8888 : std::nullopt;
8889 NewExitingVPV =
8890 Builder.createSelect(Cond, OrigExitingVPV, PhiR, {}, "", FMFs);
8891 OrigExitingVPV->replaceUsesWithIf(NewExitingVPV, [](VPUser &U, unsigned) {
8892 return isa<VPInstruction>(&U) &&
8893 (cast<VPInstruction>(&U)->getOpcode() ==
8895 cast<VPInstruction>(&U)->getOpcode() ==
8897 cast<VPInstruction>(&U)->getOpcode() ==
8899 });
8900 if (CM.usePredicatedReductionSelect())
8901 PhiR->setOperand(1, NewExitingVPV);
8902 }
8903
8904 // We want code in the middle block to appear to execute on the location of
8905 // the scalar loop's latch terminator because: (a) it is all compiler
8906 // generated, (b) these instructions are always executed after evaluating
8907 // the latch conditional branch, and (c) other passes may add new
8908 // predecessors which terminate on this line. This is the easiest way to
8909 // ensure we don't accidentally cause an extra step back into the loop while
8910 // debugging.
8911 DebugLoc ExitDL = OrigLoop->getLoopLatch()->getTerminator()->getDebugLoc();
8912
8913 // TODO: At the moment ComputeReductionResult also drives creation of the
8914 // bc.merge.rdx phi nodes, hence it needs to be created unconditionally here
8915 // even for in-loop reductions, until the reduction resume value handling is
8916 // also modeled in VPlan.
8917 VPInstruction *FinalReductionResult;
8918 VPBuilder::InsertPointGuard Guard(Builder);
8919 Builder.setInsertPoint(MiddleVPBB, IP);
8920 RecurKind RecurrenceKind = PhiR->getRecurrenceKind();
8922 VPValue *Start = PhiR->getStartValue();
8923 VPValue *Sentinel = Plan->getOrAddLiveIn(RdxDesc.getSentinelValue());
8924 FinalReductionResult =
8925 Builder.createNaryOp(VPInstruction::ComputeFindIVResult,
8926 {PhiR, Start, Sentinel, NewExitingVPV}, ExitDL);
8927 } else if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RecurrenceKind)) {
8928 VPValue *Start = PhiR->getStartValue();
8929 FinalReductionResult =
8930 Builder.createNaryOp(VPInstruction::ComputeAnyOfResult,
8931 {PhiR, Start, NewExitingVPV}, ExitDL);
8932 } else {
8933 VPIRFlags Flags =
8935 ? VPIRFlags(RdxDesc.getFastMathFlags())
8936 : VPIRFlags();
8937 FinalReductionResult =
8938 Builder.createNaryOp(VPInstruction::ComputeReductionResult,
8939 {PhiR, NewExitingVPV}, Flags, ExitDL);
8940 }
8941 // If the vector reduction can be performed in a smaller type, we truncate
8942 // then extend the loop exit value to enable InstCombine to evaluate the
8943 // entire expression in the smaller type.
8944 if (MinVF.isVector() && PhiTy != RdxDesc.getRecurrenceType() &&
8946 assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
8948 "Unexpected truncated min-max recurrence!");
8949 Type *RdxTy = RdxDesc.getRecurrenceType();
8950 auto *Trunc =
8951 new VPWidenCastRecipe(Instruction::Trunc, NewExitingVPV, RdxTy);
8952 Instruction::CastOps ExtendOpc =
8953 RdxDesc.isSigned() ? Instruction::SExt : Instruction::ZExt;
8954 auto *Extnd = new VPWidenCastRecipe(ExtendOpc, Trunc, PhiTy);
8955 Trunc->insertAfter(NewExitingVPV->getDefiningRecipe());
8956 Extnd->insertAfter(Trunc);
8957 if (PhiR->getOperand(1) == NewExitingVPV)
8958 PhiR->setOperand(1, Extnd->getVPSingleValue());
8959
8960 // Update ComputeReductionResult with the truncated exiting value and
8961 // extend its result.
8962 FinalReductionResult->setOperand(1, Trunc);
8963 FinalReductionResult =
8964 Builder.createScalarCast(ExtendOpc, FinalReductionResult, PhiTy, {});
8965 }
8966
8967 // Update all users outside the vector region. Also replace redundant
8968 // ExtractLastElement.
8969 for (auto *U : to_vector(OrigExitingVPV->users())) {
8970 auto *Parent = cast<VPRecipeBase>(U)->getParent();
8971 if (FinalReductionResult == U || Parent->getParent())
8972 continue;
8973 U->replaceUsesOfWith(OrigExitingVPV, FinalReductionResult);
8975 cast<VPInstruction>(U)->replaceAllUsesWith(FinalReductionResult);
8976 }
8977
8978 // Adjust AnyOf reductions; replace the reduction phi for the selected value
8979 // with a boolean reduction phi node to check if the condition is true in
8980 // any iteration. The final value is selected by the final
8981 // ComputeReductionResult.
8982 if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RecurrenceKind)) {
8983 auto *Select = cast<VPRecipeBase>(*find_if(PhiR->users(), [](VPUser *U) {
8984 return isa<VPWidenSelectRecipe>(U) ||
8985 (isa<VPReplicateRecipe>(U) &&
8986 cast<VPReplicateRecipe>(U)->getUnderlyingInstr()->getOpcode() ==
8987 Instruction::Select);
8988 }));
8989 VPValue *Cmp = Select->getOperand(0);
8990 // If the compare is checking the reduction PHI node, adjust it to check
8991 // the start value.
8992 if (VPRecipeBase *CmpR = Cmp->getDefiningRecipe())
8993 CmpR->replaceUsesOfWith(PhiR, PhiR->getStartValue());
8994 Builder.setInsertPoint(Select);
8995
8996 // If the true value of the select is the reduction phi, the new value is
8997 // selected if the negated condition is true in any iteration.
8998 if (Select->getOperand(1) == PhiR)
8999 Cmp = Builder.createNot(Cmp);
9000 VPValue *Or = Builder.createOr(PhiR, Cmp);
9001 Select->getVPSingleValue()->replaceAllUsesWith(Or);
9002 // Delete Select now that it has invalid types.
9003 ToDelete.push_back(Select);
9004
9005 // Convert the reduction phi to operate on bools.
9006 PhiR->setOperand(0, Plan->getOrAddLiveIn(ConstantInt::getFalse(
9007 OrigLoop->getHeader()->getContext())));
9008 continue;
9009 }
9010
9012 RdxDesc.getRecurrenceKind())) {
9013 // Adjust the start value for FindFirstIV/FindLastIV recurrences to use
9014 // the sentinel value after generating the ResumePhi recipe, which uses
9015 // the original start value.
9016 PhiR->setOperand(0, Plan->getOrAddLiveIn(RdxDesc.getSentinelValue()));
9017 }
9018 RecurKind RK = RdxDesc.getRecurrenceKind();
9022 VPBuilder PHBuilder(Plan->getVectorPreheader());
9023 VPValue *Iden = Plan->getOrAddLiveIn(
9024 getRecurrenceIdentity(RK, PhiTy, RdxDesc.getFastMathFlags()));
9025 // If the PHI is used by a partial reduction, set the scale factor.
9026 unsigned ScaleFactor =
9027 RecipeBuilder.getScalingForReduction(RdxDesc.getLoopExitInstr())
9028 .value_or(1);
9029 Type *I32Ty = IntegerType::getInt32Ty(PhiTy->getContext());
9030 auto *ScaleFactorVPV =
9031 Plan->getOrAddLiveIn(ConstantInt::get(I32Ty, ScaleFactor));
9032 VPValue *StartV = PHBuilder.createNaryOp(
9034 {PhiR->getStartValue(), Iden, ScaleFactorVPV},
9035 PhiTy->isFloatingPointTy() ? RdxDesc.getFastMathFlags()
9036 : FastMathFlags());
9037 PhiR->setOperand(0, StartV);
9038 }
9039 }
9040 for (VPRecipeBase *R : ToDelete)
9041 R->eraseFromParent();
9042
9044}
9045
9046void LoopVectorizationPlanner::attachRuntimeChecks(
9047 VPlan &Plan, GeneratedRTChecks &RTChecks, bool HasBranchWeights) const {
9048 const auto &[SCEVCheckCond, SCEVCheckBlock] = RTChecks.getSCEVChecks();
9049 if (SCEVCheckBlock && SCEVCheckBlock->hasNPredecessors(0)) {
9050 assert((!CM.OptForSize ||
9051 CM.Hints->getForce() == LoopVectorizeHints::FK_Enabled) &&
9052 "Cannot SCEV check stride or overflow when optimizing for size");
9053 VPlanTransforms::attachCheckBlock(Plan, SCEVCheckCond, SCEVCheckBlock,
9054 HasBranchWeights);
9055 }
9056 const auto &[MemCheckCond, MemCheckBlock] = RTChecks.getMemRuntimeChecks();
9057 if (MemCheckBlock && MemCheckBlock->hasNPredecessors(0)) {
9058 // VPlan-native path does not do any analysis for runtime checks
9059 // currently.
9060 assert((!EnableVPlanNativePath || OrigLoop->isInnermost()) &&
9061 "Runtime checks are not supported for outer loops yet");
9062
9063 if (CM.OptForSize) {
9064 assert(
9065 CM.Hints->getForce() == LoopVectorizeHints::FK_Enabled &&
9066 "Cannot emit memory checks when optimizing for size, unless forced "
9067 "to vectorize.");
9068 ORE->emit([&]() {
9069 return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationCodeSize",
9070 OrigLoop->getStartLoc(),
9071 OrigLoop->getHeader())
9072 << "Code-size may be reduced by not forcing "
9073 "vectorization, or by source-code modifications "
9074 "eliminating the need for runtime checks "
9075 "(e.g., adding 'restrict').";
9076 });
9077 }
9078 VPlanTransforms::attachCheckBlock(Plan, MemCheckCond, MemCheckBlock,
9079 HasBranchWeights);
9080 }
9081}
9082
9084 VPlan &Plan, ElementCount VF, unsigned UF,
9085 ElementCount MinProfitableTripCount) const {
9086 // vscale is not necessarily a power-of-2, which means we cannot guarantee
9087 // an overflow to zero when updating induction variables and so an
9088 // additional overflow check is required before entering the vector loop.
9089 bool IsIndvarOverflowCheckNeededForVF =
9090 VF.isScalable() && !TTI.isVScaleKnownToBeAPowerOfTwo() &&
9091 !isIndvarOverflowCheckKnownFalse(&CM, VF, UF) &&
9092 CM.getTailFoldingStyle() !=
9094 const uint32_t *BranchWeigths =
9095 hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator())
9097 : nullptr;
9099 Plan, VF, UF, MinProfitableTripCount,
9100 CM.requiresScalarEpilogue(VF.isVector()), CM.foldTailByMasking(),
9101 IsIndvarOverflowCheckNeededForVF, OrigLoop, BranchWeigths,
9102 OrigLoop->getLoopPredecessor()->getTerminator()->getDebugLoc(),
9103 *PSE.getSE());
9104}
9105
9107 assert(!State.Lane && "VPDerivedIVRecipe being replicated.");
9108
9109 // Fast-math-flags propagate from the original induction instruction.
9110 IRBuilder<>::FastMathFlagGuard FMFG(State.Builder);
9111 if (FPBinOp)
9112 State.Builder.setFastMathFlags(FPBinOp->getFastMathFlags());
9113
9114 Value *Step = State.get(getStepValue(), VPLane(0));
9115 Value *Index = State.get(getOperand(1), VPLane(0));
9116 Value *DerivedIV = emitTransformedIndex(
9117 State.Builder, Index, getStartValue()->getLiveInIRValue(), Step, Kind,
9119 DerivedIV->setName(Name);
9120 State.set(this, DerivedIV, VPLane(0));
9121}
9122
9123// Determine how to lower the scalar epilogue, which depends on 1) optimising
9124// for minimum code-size, 2) predicate compiler options, 3) loop hints forcing
9125// predication, and 4) a TTI hook that analyses whether the loop is suitable
9126// for predication.
9131 // 1) OptSize takes precedence over all other options, i.e. if this is set,
9132 // don't look at hints or options, and don't request a scalar epilogue.
9133 // (For PGSO, as shouldOptimizeForSize isn't currently accessible from
9134 // LoopAccessInfo (due to code dependency and not being able to reliably get
9135 // PSI/BFI from a loop analysis under NPM), we cannot suppress the collection
9136 // of strides in LoopAccessInfo::analyzeLoop() and vectorize without
9137 // versioning when the vectorization is forced, unlike hasOptSize. So revert
9138 // back to the old way and vectorize with versioning when forced. See D81345.)
9139 if (F->hasOptSize() || (llvm::shouldOptimizeForSize(L->getHeader(), PSI, BFI,
9143
9144 // 2) If set, obey the directives
9145 if (PreferPredicateOverEpilogue.getNumOccurrences()) {
9153 };
9154 }
9155
9156 // 3) If set, obey the hints
9157 switch (Hints.getPredicate()) {
9162 };
9163
9164 // 4) if the TTI hook indicates this is profitable, request predication.
9165 TailFoldingInfo TFI(TLI, &LVL, IAI);
9166 if (TTI->preferPredicateOverEpilogue(&TFI))
9168
9170}
9171
9172// Process the loop in the VPlan-native vectorization path. This path builds
9173// VPlan upfront in the vectorization pipeline, which allows to apply
9174// VPlan-to-VPlan transformations from the very beginning without modifying the
9175// input LLVM IR.
9182 LoopVectorizationRequirements &Requirements) {
9183
9185 LLVM_DEBUG(dbgs() << "LV: cannot compute the outer-loop trip count\n");
9186 return false;
9187 }
9188 assert(EnableVPlanNativePath && "VPlan-native path is disabled.");
9189 Function *F = L->getHeader()->getParent();
9190 InterleavedAccessInfo IAI(PSE, L, DT, LI, LVL->getLAI());
9191
9193 getScalarEpilogueLowering(F, L, Hints, PSI, BFI, TTI, TLI, *LVL, &IAI);
9194
9195 LoopVectorizationCostModel CM(SEL, L, PSE, LI, LVL, *TTI, TLI, DB, AC, ORE, F,
9196 &Hints, IAI, PSI, BFI);
9197 // Use the planner for outer loop vectorization.
9198 // TODO: CM is not used at this point inside the planner. Turn CM into an
9199 // optional argument if we don't need it in the future.
9200 LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, LVL, CM, IAI, PSE, Hints,
9201 ORE);
9202
9203 // Get user vectorization factor.
9204 ElementCount UserVF = Hints.getWidth();
9205
9207
9208 // Plan how to best vectorize, return the best VF and its cost.
9209 const VectorizationFactor VF = LVP.planInVPlanNativePath(UserVF);
9210
9211 // If we are stress testing VPlan builds, do not attempt to generate vector
9212 // code. Masked vector code generation support will follow soon.
9213 // Also, do not attempt to vectorize if no vector code will be produced.
9215 return false;
9216
9217 VPlan &BestPlan = LVP.getPlanFor(VF.Width);
9218
9219 {
9220 GeneratedRTChecks Checks(PSE, DT, LI, TTI, F->getDataLayout(), CM.CostKind);
9221 InnerLoopVectorizer LB(L, PSE, LI, DT, TTI, AC, VF.Width, /*UF=*/1, &CM,
9222 BFI, PSI, Checks, BestPlan);
9223 LLVM_DEBUG(dbgs() << "Vectorizing outer loop in \""
9224 << L->getHeader()->getParent()->getName() << "\"\n");
9225 LVP.addMinimumIterationCheck(BestPlan, VF.Width, /*UF=*/1,
9227
9228 LVP.executePlan(VF.Width, /*UF=*/1, BestPlan, LB, DT, false);
9229 }
9230
9231 reportVectorization(ORE, L, VF, 1);
9232
9233 assert(!verifyFunction(*L->getHeader()->getParent(), &dbgs()));
9234 return true;
9235}
9236
9237// Emit a remark if there are stores to floats that required a floating point
9238// extension. If the vectorized loop was generated with floating point there
9239// will be a performance penalty from the conversion overhead and the change in
9240// the vector width.
9243 for (BasicBlock *BB : L->getBlocks()) {
9244 for (Instruction &Inst : *BB) {
9245 if (auto *S = dyn_cast<StoreInst>(&Inst)) {
9246 if (S->getValueOperand()->getType()->isFloatTy())
9247 Worklist.push_back(S);
9248 }
9249 }
9250 }
9251
9252 // Traverse the floating point stores upwards searching, for floating point
9253 // conversions.
9256 while (!Worklist.empty()) {
9257 auto *I = Worklist.pop_back_val();
9258 if (!L->contains(I))
9259 continue;
9260 if (!Visited.insert(I).second)
9261 continue;
9262
9263 // Emit a remark if the floating point store required a floating
9264 // point conversion.
9265 // TODO: More work could be done to identify the root cause such as a
9266 // constant or a function return type and point the user to it.
9267 if (isa<FPExtInst>(I) && EmittedRemark.insert(I).second)
9268 ORE->emit([&]() {
9269 return OptimizationRemarkAnalysis(LV_NAME, "VectorMixedPrecision",
9270 I->getDebugLoc(), L->getHeader())
9271 << "floating point conversion changes vector width. "
9272 << "Mixed floating point precision requires an up/down "
9273 << "cast that will negatively impact performance.";
9274 });
9275
9276 for (Use &Op : I->operands())
9277 if (auto *OpI = dyn_cast<Instruction>(Op))
9278 Worklist.push_back(OpI);
9279 }
9280}
9281
9282/// For loops with uncountable early exits, find the cost of doing work when
9283/// exiting the loop early, such as calculating the final exit values of
9284/// variables used outside the loop.
9285/// TODO: This is currently overly pessimistic because the loop may not take
9286/// the early exit, but better to keep this conservative for now. In future,
9287/// it might be possible to relax this by using branch probabilities.
9289 VPlan &Plan, ElementCount VF) {
9290 InstructionCost Cost = 0;
9291 for (auto *ExitVPBB : Plan.getExitBlocks()) {
9292 for (auto *PredVPBB : ExitVPBB->getPredecessors()) {
9293 // If the predecessor is not the middle.block, then it must be the
9294 // vector.early.exit block, which may contain work to calculate the exit
9295 // values of variables used outside the loop.
9296 if (PredVPBB != Plan.getMiddleBlock()) {
9297 LLVM_DEBUG(dbgs() << "Calculating cost of work in exit block "
9298 << PredVPBB->getName() << ":\n");
9299 Cost += PredVPBB->cost(VF, CostCtx);
9300 }
9301 }
9302 }
9303 return Cost;
9304}
9305
9306/// This function determines whether or not it's still profitable to vectorize
9307/// the loop given the extra work we have to do outside of the loop:
9308/// 1. Perform the runtime checks before entering the loop to ensure it's safe
9309/// to vectorize.
9310/// 2. In the case of loops with uncountable early exits, we may have to do
9311/// extra work when exiting the loop early, such as calculating the final
9312/// exit values of variables used outside the loop.
9313static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks,
9314 VectorizationFactor &VF, Loop *L,
9316 VPCostContext &CostCtx, VPlan &Plan,
9318 std::optional<unsigned> VScale) {
9319 InstructionCost TotalCost = Checks.getCost();
9320 if (!TotalCost.isValid())
9321 return false;
9322
9323 // Add on the cost of any work required in the vector early exit block, if
9324 // one exists.
9325 TotalCost += calculateEarlyExitCost(CostCtx, Plan, VF.Width);
9326
9327 // When interleaving only scalar and vector cost will be equal, which in turn
9328 // would lead to a divide by 0. Fall back to hard threshold.
9329 if (VF.Width.isScalar()) {
9330 // TODO: Should we rename VectorizeMemoryCheckThreshold?
9331 if (TotalCost > VectorizeMemoryCheckThreshold) {
9332 LLVM_DEBUG(
9333 dbgs()
9334 << "LV: Interleaving only is not profitable due to runtime checks\n");
9335 return false;
9336 }
9337 return true;
9338 }
9339
9340 // The scalar cost should only be 0 when vectorizing with a user specified
9341 // VF/IC. In those cases, runtime checks should always be generated.
9342 uint64_t ScalarC = VF.ScalarCost.getValue();
9343 if (ScalarC == 0)
9344 return true;
9345
9346 // First, compute the minimum iteration count required so that the vector
9347 // loop outperforms the scalar loop.
9348 // The total cost of the scalar loop is
9349 // ScalarC * TC
9350 // where
9351 // * TC is the actual trip count of the loop.
9352 // * ScalarC is the cost of a single scalar iteration.
9353 //
9354 // The total cost of the vector loop is
9355 // RtC + VecC * (TC / VF) + EpiC
9356 // where
9357 // * RtC is the cost of the generated runtime checks plus the cost of
9358 // performing any additional work in the vector.early.exit block for loops
9359 // with uncountable early exits.
9360 // * VecC is the cost of a single vector iteration.
9361 // * TC is the actual trip count of the loop
9362 // * VF is the vectorization factor
9363 // * EpiCost is the cost of the generated epilogue, including the cost
9364 // of the remaining scalar operations.
9365 //
9366 // Vectorization is profitable once the total vector cost is less than the
9367 // total scalar cost:
9368 // RtC + VecC * (TC / VF) + EpiC < ScalarC * TC
9369 //
9370 // Now we can compute the minimum required trip count TC as
9371 // VF * (RtC + EpiC) / (ScalarC * VF - VecC) < TC
9372 //
9373 // For now we assume the epilogue cost EpiC = 0 for simplicity. Note that
9374 // the computations are performed on doubles, not integers and the result
9375 // is rounded up, hence we get an upper estimate of the TC.
9376 unsigned IntVF = estimateElementCount(VF.Width, VScale);
9377 uint64_t RtC = TotalCost.getValue();
9378 uint64_t Div = ScalarC * IntVF - VF.Cost.getValue();
9379 uint64_t MinTC1 = Div == 0 ? 0 : divideCeil(RtC * IntVF, Div);
9380
9381 // Second, compute a minimum iteration count so that the cost of the
9382 // runtime checks is only a fraction of the total scalar loop cost. This
9383 // adds a loop-dependent bound on the overhead incurred if the runtime
9384 // checks fail. In case the runtime checks fail, the cost is RtC + ScalarC
9385 // * TC. To bound the runtime check to be a fraction 1/X of the scalar
9386 // cost, compute
9387 // RtC < ScalarC * TC * (1 / X) ==> RtC * X / ScalarC < TC
9388 uint64_t MinTC2 = divideCeil(RtC * 10, ScalarC);
9389
9390 // Now pick the larger minimum. If it is not a multiple of VF and a scalar
9391 // epilogue is allowed, choose the next closest multiple of VF. This should
9392 // partly compensate for ignoring the epilogue cost.
9393 uint64_t MinTC = std::max(MinTC1, MinTC2);
9394 if (SEL == CM_ScalarEpilogueAllowed)
9395 MinTC = alignTo(MinTC, IntVF);
9397
9398 LLVM_DEBUG(
9399 dbgs() << "LV: Minimum required TC for runtime checks to be profitable:"
9400 << VF.MinProfitableTripCount << "\n");
9401
9402 // Skip vectorization if the expected trip count is less than the minimum
9403 // required trip count.
9404 if (auto ExpectedTC = getSmallBestKnownTC(PSE, L)) {
9405 if (ElementCount::isKnownLT(*ExpectedTC, VF.MinProfitableTripCount)) {
9406 LLVM_DEBUG(dbgs() << "LV: Vectorization is not beneficial: expected "
9407 "trip count < minimum profitable VF ("
9408 << *ExpectedTC << " < " << VF.MinProfitableTripCount
9409 << ")\n");
9410
9411 return false;
9412 }
9413 }
9414 return true;
9415}
9416
9418 : InterleaveOnlyWhenForced(Opts.InterleaveOnlyWhenForced ||
9420 VectorizeOnlyWhenForced(Opts.VectorizeOnlyWhenForced ||
9422
9423/// Prepare \p MainPlan for vectorizing the main vector loop during epilogue
9424/// vectorization. Remove ResumePhis from \p MainPlan for inductions that
9425/// don't have a corresponding wide induction in \p EpiPlan.
9426static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
9427 // Collect PHI nodes of widened phis in the VPlan for the epilogue. Those
9428 // will need their resume-values computed in the main vector loop. Others
9429 // can be removed from the main VPlan.
9430 SmallPtrSet<PHINode *, 2> EpiWidenedPhis;
9431 for (VPRecipeBase &R :
9434 continue;
9435 EpiWidenedPhis.insert(
9436 cast<PHINode>(R.getVPSingleValue()->getUnderlyingValue()));
9437 }
9438 for (VPRecipeBase &R :
9439 make_early_inc_range(MainPlan.getScalarHeader()->phis())) {
9440 auto *VPIRInst = cast<VPIRPhi>(&R);
9441 if (EpiWidenedPhis.contains(&VPIRInst->getIRPhi()))
9442 continue;
9443 // There is no corresponding wide induction in the epilogue plan that would
9444 // need a resume value. Remove the VPIRInst wrapping the scalar header phi
9445 // together with the corresponding ResumePhi. The resume values for the
9446 // scalar loop will be created during execution of EpiPlan.
9447 VPRecipeBase *ResumePhi = VPIRInst->getOperand(0)->getDefiningRecipe();
9448 VPIRInst->eraseFromParent();
9449 ResumePhi->eraseFromParent();
9450 }
9452
9453 using namespace VPlanPatternMatch;
9454 // When vectorizing the epilogue, FindFirstIV & FindLastIV reductions can
9455 // introduce multiple uses of undef/poison. If the reduction start value may
9456 // be undef or poison it needs to be frozen and the frozen start has to be
9457 // used when computing the reduction result. We also need to use the frozen
9458 // value in the resume phi generated by the main vector loop, as this is also
9459 // used to compute the reduction result after the epilogue vector loop.
9460 auto AddFreezeForFindLastIVReductions = [](VPlan &Plan,
9461 bool UpdateResumePhis) {
9462 VPBuilder Builder(Plan.getEntry());
9463 for (VPRecipeBase &R : *Plan.getMiddleBlock()) {
9464 auto *VPI = dyn_cast<VPInstruction>(&R);
9465 if (!VPI || VPI->getOpcode() != VPInstruction::ComputeFindIVResult)
9466 continue;
9467 VPValue *OrigStart = VPI->getOperand(1);
9469 continue;
9470 VPInstruction *Freeze =
9471 Builder.createNaryOp(Instruction::Freeze, {OrigStart}, {}, "fr");
9472 VPI->setOperand(1, Freeze);
9473 if (UpdateResumePhis)
9474 OrigStart->replaceUsesWithIf(Freeze, [Freeze](VPUser &U, unsigned) {
9475 return Freeze != &U && isa<VPPhi>(&U);
9476 });
9477 }
9478 };
9479 AddFreezeForFindLastIVReductions(MainPlan, true);
9480 AddFreezeForFindLastIVReductions(EpiPlan, false);
9481
9482 VPBasicBlock *MainScalarPH = MainPlan.getScalarPreheader();
9483 VPValue *VectorTC = &MainPlan.getVectorTripCount();
9484 // If there is a suitable resume value for the canonical induction in the
9485 // scalar (which will become vector) epilogue loop, use it and move it to the
9486 // beginning of the scalar preheader. Otherwise create it below.
9487 auto ResumePhiIter =
9488 find_if(MainScalarPH->phis(), [VectorTC](VPRecipeBase &R) {
9489 return match(&R, m_VPInstruction<Instruction::PHI>(m_Specific(VectorTC),
9490 m_ZeroInt()));
9491 });
9492 VPPhi *ResumePhi = nullptr;
9493 if (ResumePhiIter == MainScalarPH->phis().end()) {
9494 VPBuilder ScalarPHBuilder(MainScalarPH, MainScalarPH->begin());
9495 ResumePhi = ScalarPHBuilder.createScalarPhi(
9496 {VectorTC, MainPlan.getCanonicalIV()->getStartValue()}, {},
9497 "vec.epilog.resume.val");
9498 } else {
9499 ResumePhi = cast<VPPhi>(&*ResumePhiIter);
9500 if (MainScalarPH->begin() == MainScalarPH->end())
9501 ResumePhi->moveBefore(*MainScalarPH, MainScalarPH->end());
9502 else if (&*MainScalarPH->begin() != ResumePhi)
9503 ResumePhi->moveBefore(*MainScalarPH, MainScalarPH->begin());
9504 }
9505 // Add a user to to make sure the resume phi won't get removed.
9506 VPBuilder(MainScalarPH)
9508}
9509
9510/// Prepare \p Plan for vectorizing the epilogue loop. That is, re-use expanded
9511/// SCEVs from \p ExpandedSCEVs and set resume values for header recipes. Some
9512/// reductions require creating new instructions to compute the resume values.
9513/// They are collected in a vector and returned. They must be moved to the
9514/// preheader of the vector epilogue loop, after created by the execution of \p
9515/// Plan.
9517 VPlan &Plan, Loop *L, const SCEV2ValueTy &ExpandedSCEVs,
9519 ScalarEvolution &SE) {
9520 VPRegionBlock *VectorLoop = Plan.getVectorLoopRegion();
9521 VPBasicBlock *Header = VectorLoop->getEntryBasicBlock();
9522 Header->setName("vec.epilog.vector.body");
9523
9524 // Ensure that the start values for all header phi recipes are updated before
9525 // vectorizing the epilogue loop.
9527 // When vectorizing the epilogue loop, the canonical induction start
9528 // value needs to be changed from zero to the value after the main
9529 // vector loop. Find the resume value created during execution of the main
9530 // VPlan. It must be the first phi in the loop preheader.
9531 // FIXME: Improve modeling for canonical IV start values in the epilogue
9532 // loop.
9533 using namespace llvm::PatternMatch;
9534 PHINode *EPResumeVal = &*L->getLoopPreheader()->phis().begin();
9535 for (Value *Inc : EPResumeVal->incoming_values()) {
9536 if (match(Inc, m_SpecificInt(0)))
9537 continue;
9538 assert(!EPI.VectorTripCount &&
9539 "Must only have a single non-zero incoming value");
9540 EPI.VectorTripCount = Inc;
9541 }
9542 // If we didn't find a non-zero vector trip count, all incoming values
9543 // must be zero, which also means the vector trip count is zero. Pick the
9544 // first zero as vector trip count.
9545 // TODO: We should not choose VF * UF so the main vector loop is known to
9546 // be dead.
9547 if (!EPI.VectorTripCount) {
9548 assert(EPResumeVal->getNumIncomingValues() > 0 &&
9549 all_of(EPResumeVal->incoming_values(),
9550 [](Value *Inc) { return match(Inc, m_SpecificInt(0)); }) &&
9551 "all incoming values must be 0");
9552 EPI.VectorTripCount = EPResumeVal->getOperand(0);
9553 }
9554 VPValue *VPV = Plan.getOrAddLiveIn(EPResumeVal);
9555 assert(all_of(IV->users(),
9556 [](const VPUser *U) {
9557 return isa<VPScalarIVStepsRecipe>(U) ||
9558 isa<VPDerivedIVRecipe>(U) ||
9559 cast<VPRecipeBase>(U)->isScalarCast() ||
9560 cast<VPInstruction>(U)->getOpcode() ==
9561 Instruction::Add;
9562 }) &&
9563 "the canonical IV should only be used by its increment or "
9564 "ScalarIVSteps when resetting the start value");
9565 IV->setOperand(0, VPV);
9566
9568 SmallVector<Instruction *> InstsToMove;
9569 for (VPRecipeBase &R : drop_begin(Header->phis())) {
9570 Value *ResumeV = nullptr;
9571 // TODO: Move setting of resume values to prepareToExecute.
9572 if (auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(&R)) {
9573 auto *RdxResult =
9574 cast<VPInstruction>(*find_if(ReductionPhi->users(), [](VPUser *U) {
9575 auto *VPI = dyn_cast<VPInstruction>(U);
9576 return VPI &&
9577 (VPI->getOpcode() == VPInstruction::ComputeAnyOfResult ||
9578 VPI->getOpcode() == VPInstruction::ComputeReductionResult ||
9579 VPI->getOpcode() == VPInstruction::ComputeFindIVResult);
9580 }));
9581 ResumeV = cast<PHINode>(ReductionPhi->getUnderlyingInstr())
9582 ->getIncomingValueForBlock(L->getLoopPreheader());
9583 RecurKind RK = ReductionPhi->getRecurrenceKind();
9585 Value *StartV = RdxResult->getOperand(1)->getLiveInIRValue();
9586 // VPReductionPHIRecipes for AnyOf reductions expect a boolean as
9587 // start value; compare the final value from the main vector loop
9588 // to the start value.
9589 BasicBlock *PBB = cast<Instruction>(ResumeV)->getParent();
9590 IRBuilder<> Builder(PBB, PBB->getFirstNonPHIIt());
9591 ResumeV = Builder.CreateICmpNE(ResumeV, StartV);
9592 if (auto *I = dyn_cast<Instruction>(ResumeV))
9593 InstsToMove.push_back(I);
9595 Value *StartV = getStartValueFromReductionResult(RdxResult);
9596 ToFrozen[StartV] = cast<PHINode>(ResumeV)->getIncomingValueForBlock(
9598
9599 // VPReductionPHIRecipe for FindFirstIV/FindLastIV reductions requires
9600 // an adjustment to the resume value. The resume value is adjusted to
9601 // the sentinel value when the final value from the main vector loop
9602 // equals the start value. This ensures correctness when the start value
9603 // might not be less than the minimum value of a monotonically
9604 // increasing induction variable.
9605 BasicBlock *ResumeBB = cast<Instruction>(ResumeV)->getParent();
9606 IRBuilder<> Builder(ResumeBB, ResumeBB->getFirstNonPHIIt());
9607 Value *Cmp = Builder.CreateICmpEQ(ResumeV, ToFrozen[StartV]);
9608 if (auto *I = dyn_cast<Instruction>(Cmp))
9609 InstsToMove.push_back(I);
9610 Value *Sentinel = RdxResult->getOperand(2)->getLiveInIRValue();
9611 ResumeV = Builder.CreateSelect(Cmp, Sentinel, ResumeV);
9612 if (auto *I = dyn_cast<Instruction>(ResumeV))
9613 InstsToMove.push_back(I);
9614 } else {
9615 VPValue *StartVal = Plan.getOrAddLiveIn(ResumeV);
9616 auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
9617 if (auto *VPI = dyn_cast<VPInstruction>(PhiR->getStartValue())) {
9618 assert(VPI->getOpcode() == VPInstruction::ReductionStartVector &&
9619 "unexpected start value");
9620 VPI->setOperand(0, StartVal);
9621 continue;
9622 }
9623 }
9624 } else {
9625 // Retrieve the induction resume values for wide inductions from
9626 // their original phi nodes in the scalar loop.
9627 PHINode *IndPhi = cast<VPWidenInductionRecipe>(&R)->getPHINode();
9628 // Hook up to the PHINode generated by a ResumePhi recipe of main
9629 // loop VPlan, which feeds the scalar loop.
9630 ResumeV = IndPhi->getIncomingValueForBlock(L->getLoopPreheader());
9631 }
9632 assert(ResumeV && "Must have a resume value");
9633 VPValue *StartVal = Plan.getOrAddLiveIn(ResumeV);
9634 cast<VPHeaderPHIRecipe>(&R)->setStartValue(StartVal);
9635 }
9636
9637 // For some VPValues in the epilogue plan we must re-use the generated IR
9638 // values from the main plan. Replace them with live-in VPValues.
9639 // TODO: This is a workaround needed for epilogue vectorization and it
9640 // should be removed once induction resume value creation is done
9641 // directly in VPlan.
9642 for (auto &R : make_early_inc_range(*Plan.getEntry())) {
9643 // Re-use frozen values from the main plan for Freeze VPInstructions in the
9644 // epilogue plan. This ensures all users use the same frozen value.
9645 auto *VPI = dyn_cast<VPInstruction>(&R);
9646 if (VPI && VPI->getOpcode() == Instruction::Freeze) {
9647 VPI->replaceAllUsesWith(Plan.getOrAddLiveIn(
9648 ToFrozen.lookup(VPI->getOperand(0)->getLiveInIRValue())));
9649 continue;
9650 }
9651
9652 // Re-use the trip count and steps expanded for the main loop, as
9653 // skeleton creation needs it as a value that dominates both the scalar
9654 // and vector epilogue loops
9655 auto *ExpandR = dyn_cast<VPExpandSCEVRecipe>(&R);
9656 if (!ExpandR)
9657 continue;
9658 VPValue *ExpandedVal =
9659 Plan.getOrAddLiveIn(ExpandedSCEVs.lookup(ExpandR->getSCEV()));
9660 ExpandR->replaceAllUsesWith(ExpandedVal);
9661 if (Plan.getTripCount() == ExpandR)
9662 Plan.resetTripCount(ExpandedVal);
9663 ExpandR->eraseFromParent();
9664 }
9665
9666 auto VScale = CM.getVScaleForTuning();
9667 unsigned MainLoopStep =
9668 estimateElementCount(EPI.MainLoopVF * EPI.MainLoopUF, VScale);
9669 unsigned EpilogueLoopStep =
9670 estimateElementCount(EPI.EpilogueVF * EPI.EpilogueUF, VScale);
9672 Plan, EPI.TripCount, EPI.VectorTripCount,
9674 EPI.EpilogueUF, MainLoopStep, EpilogueLoopStep, SE);
9675
9676 return InstsToMove;
9677}
9678
9679// Generate bypass values from the additional bypass block. Note that when the
9680// vectorized epilogue is skipped due to iteration count check, then the
9681// resume value for the induction variable comes from the trip count of the
9682// main vector loop, passed as the second argument.
9684 PHINode *OrigPhi, const InductionDescriptor &II, IRBuilder<> &BypassBuilder,
9685 const SCEV2ValueTy &ExpandedSCEVs, Value *MainVectorTripCount,
9686 Instruction *OldInduction) {
9687 Value *Step = getExpandedStep(II, ExpandedSCEVs);
9688 // For the primary induction the additional bypass end value is known.
9689 // Otherwise it is computed.
9690 Value *EndValueFromAdditionalBypass = MainVectorTripCount;
9691 if (OrigPhi != OldInduction) {
9692 auto *BinOp = II.getInductionBinOp();
9693 // Fast-math-flags propagate from the original induction instruction.
9695 BypassBuilder.setFastMathFlags(BinOp->getFastMathFlags());
9696
9697 // Compute the end value for the additional bypass.
9698 EndValueFromAdditionalBypass =
9699 emitTransformedIndex(BypassBuilder, MainVectorTripCount,
9700 II.getStartValue(), Step, II.getKind(), BinOp);
9701 EndValueFromAdditionalBypass->setName("ind.end");
9702 }
9703 return EndValueFromAdditionalBypass;
9704}
9705
9707 VPlan &BestEpiPlan,
9709 const SCEV2ValueTy &ExpandedSCEVs,
9710 Value *MainVectorTripCount) {
9711 // Fix reduction resume values from the additional bypass block.
9712 BasicBlock *PH = L->getLoopPreheader();
9713 for (auto *Pred : predecessors(PH)) {
9714 for (PHINode &Phi : PH->phis()) {
9715 if (Phi.getBasicBlockIndex(Pred) != -1)
9716 continue;
9717 Phi.addIncoming(Phi.getIncomingValueForBlock(BypassBlock), Pred);
9718 }
9719 }
9720 auto *ScalarPH = cast<VPIRBasicBlock>(BestEpiPlan.getScalarPreheader());
9721 if (ScalarPH->hasPredecessors()) {
9722 // If ScalarPH has predecessors, we may need to update its reduction
9723 // resume values.
9724 for (const auto &[R, IRPhi] :
9725 zip(ScalarPH->phis(), ScalarPH->getIRBasicBlock()->phis())) {
9727 BypassBlock);
9728 }
9729 }
9730
9731 // Fix induction resume values from the additional bypass block.
9732 IRBuilder<> BypassBuilder(BypassBlock, BypassBlock->getFirstInsertionPt());
9733 for (const auto &[IVPhi, II] : LVL.getInductionVars()) {
9734 auto *Inc = cast<PHINode>(IVPhi->getIncomingValueForBlock(PH));
9736 IVPhi, II, BypassBuilder, ExpandedSCEVs, MainVectorTripCount,
9737 LVL.getPrimaryInduction());
9738 // TODO: Directly add as extra operand to the VPResumePHI recipe.
9739 Inc->setIncomingValueForBlock(BypassBlock, V);
9740 }
9741}
9742
9743/// Connect the epilogue vector loop generated for \p EpiPlan to the main vector
9744// loop, after both plans have executed, updating branches from the iteration
9745// and runtime checks of the main loop, as well as updating various phis. \p
9746// InstsToMove contains instructions that need to be moved to the preheader of
9747// the epilogue vector loop.
9749 VPlan &EpiPlan, Loop *L, EpilogueLoopVectorizationInfo &EPI,
9751 DenseMap<const SCEV *, Value *> &ExpandedSCEVs, GeneratedRTChecks &Checks,
9752 ArrayRef<Instruction *> InstsToMove) {
9753 BasicBlock *VecEpilogueIterationCountCheck =
9754 cast<VPIRBasicBlock>(EpiPlan.getEntry())->getIRBasicBlock();
9755
9756 BasicBlock *VecEpiloguePreHeader =
9757 cast<BranchInst>(VecEpilogueIterationCountCheck->getTerminator())
9758 ->getSuccessor(1);
9759 // Adjust the control flow taking the state info from the main loop
9760 // vectorization into account.
9762 "expected this to be saved from the previous pass.");
9763 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);
9765 VecEpilogueIterationCountCheck, VecEpiloguePreHeader);
9766
9768 VecEpilogueIterationCountCheck},
9770 VecEpiloguePreHeader}});
9771
9772 BasicBlock *ScalarPH =
9773 cast<VPIRBasicBlock>(EpiPlan.getScalarPreheader())->getIRBasicBlock();
9775 VecEpilogueIterationCountCheck, ScalarPH);
9776 DTU.applyUpdates(
9778 VecEpilogueIterationCountCheck},
9780
9781 // Adjust the terminators of runtime check blocks and phis using them.
9782 BasicBlock *SCEVCheckBlock = Checks.getSCEVChecks().second;
9783 BasicBlock *MemCheckBlock = Checks.getMemRuntimeChecks().second;
9784 if (SCEVCheckBlock) {
9785 SCEVCheckBlock->getTerminator()->replaceUsesOfWith(
9786 VecEpilogueIterationCountCheck, ScalarPH);
9787 DTU.applyUpdates({{DominatorTree::Delete, SCEVCheckBlock,
9788 VecEpilogueIterationCountCheck},
9789 {DominatorTree::Insert, SCEVCheckBlock, ScalarPH}});
9790 }
9791 if (MemCheckBlock) {
9792 MemCheckBlock->getTerminator()->replaceUsesOfWith(
9793 VecEpilogueIterationCountCheck, ScalarPH);
9794 DTU.applyUpdates(
9795 {{DominatorTree::Delete, MemCheckBlock, VecEpilogueIterationCountCheck},
9796 {DominatorTree::Insert, MemCheckBlock, ScalarPH}});
9797 }
9798
9799 // The vec.epilog.iter.check block may contain Phi nodes from inductions
9800 // or reductions which merge control-flow from the latch block and the
9801 // middle block. Update the incoming values here and move the Phi into the
9802 // preheader.
9803 SmallVector<PHINode *, 4> PhisInBlock(
9804 llvm::make_pointer_range(VecEpilogueIterationCountCheck->phis()));
9805
9806 for (PHINode *Phi : PhisInBlock) {
9807 Phi->moveBefore(VecEpiloguePreHeader->getFirstNonPHIIt());
9808 Phi->replaceIncomingBlockWith(
9809 VecEpilogueIterationCountCheck->getSinglePredecessor(),
9810 VecEpilogueIterationCountCheck);
9811
9812 // If the phi doesn't have an incoming value from the
9813 // EpilogueIterationCountCheck, we are done. Otherwise remove the
9814 // incoming value and also those from other check blocks. This is needed
9815 // for reduction phis only.
9816 if (none_of(Phi->blocks(), [&](BasicBlock *IncB) {
9817 return EPI.EpilogueIterationCountCheck == IncB;
9818 }))
9819 continue;
9820 Phi->removeIncomingValue(EPI.EpilogueIterationCountCheck);
9821 if (SCEVCheckBlock)
9822 Phi->removeIncomingValue(SCEVCheckBlock);
9823 if (MemCheckBlock)
9824 Phi->removeIncomingValue(MemCheckBlock);
9825 }
9826
9827 auto IP = VecEpiloguePreHeader->getFirstNonPHIIt();
9828 for (auto *I : InstsToMove)
9829 I->moveBefore(IP);
9830
9831 // VecEpilogueIterationCountCheck conditionally skips over the epilogue loop
9832 // after executing the main loop. We need to update the resume values of
9833 // inductions and reductions during epilogue vectorization.
9834 fixScalarResumeValuesFromBypass(VecEpilogueIterationCountCheck, L, EpiPlan,
9835 LVL, ExpandedSCEVs, EPI.VectorTripCount);
9836}
9837
9839 assert((EnableVPlanNativePath || L->isInnermost()) &&
9840 "VPlan-native path is not enabled. Only process inner loops.");
9841
9842 LLVM_DEBUG(dbgs() << "\nLV: Checking a loop in '"
9843 << L->getHeader()->getParent()->getName() << "' from "
9844 << L->getLocStr() << "\n");
9845
9846 LoopVectorizeHints Hints(L, InterleaveOnlyWhenForced, *ORE, TTI);
9847
9848 LLVM_DEBUG(
9849 dbgs() << "LV: Loop hints:"
9850 << " force="
9852 ? "disabled"
9854 ? "enabled"
9855 : "?"))
9856 << " width=" << Hints.getWidth()
9857 << " interleave=" << Hints.getInterleave() << "\n");
9858
9859 // Function containing loop
9860 Function *F = L->getHeader()->getParent();
9861
9862 // Looking at the diagnostic output is the only way to determine if a loop
9863 // was vectorized (other than looking at the IR or machine code), so it
9864 // is important to generate an optimization remark for each loop. Most of
9865 // these messages are generated as OptimizationRemarkAnalysis. Remarks
9866 // generated as OptimizationRemark and OptimizationRemarkMissed are
9867 // less verbose reporting vectorized loops and unvectorized loops that may
9868 // benefit from vectorization, respectively.
9869
9870 if (!Hints.allowVectorization(F, L, VectorizeOnlyWhenForced)) {
9871 LLVM_DEBUG(dbgs() << "LV: Loop hints prevent vectorization.\n");
9872 return false;
9873 }
9874
9875 PredicatedScalarEvolution PSE(*SE, *L);
9876
9877 // Check if it is legal to vectorize the loop.
9878 LoopVectorizationRequirements Requirements;
9879 LoopVectorizationLegality LVL(L, PSE, DT, TTI, TLI, F, *LAIs, LI, ORE,
9880 &Requirements, &Hints, DB, AC, BFI, PSI, AA);
9882 LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Cannot prove legality.\n");
9883 Hints.emitRemarkWithHints();
9884 return false;
9885 }
9886
9888 reportVectorizationFailure("Auto-vectorization of loops with uncountable "
9889 "early exit is not enabled",
9890 "UncountableEarlyExitLoopsDisabled", ORE, L);
9891 return false;
9892 }
9893
9894 if (!LVL.getPotentiallyFaultingLoads().empty()) {
9895 reportVectorizationFailure("Auto-vectorization of loops with potentially "
9896 "faulting load is not supported",
9897 "PotentiallyFaultingLoadsNotSupported", ORE, L);
9898 return false;
9899 }
9900
9901 // Entrance to the VPlan-native vectorization path. Outer loops are processed
9902 // here. They may require CFG and instruction level transformations before
9903 // even evaluating whether vectorization is profitable. Since we cannot modify
9904 // the incoming IR, we need to build VPlan upfront in the vectorization
9905 // pipeline.
9906 if (!L->isInnermost())
9907 return processLoopInVPlanNativePath(L, PSE, LI, DT, &LVL, TTI, TLI, DB, AC,
9908 ORE, BFI, PSI, Hints, Requirements);
9909
9910 assert(L->isInnermost() && "Inner loop expected.");
9911
9912 InterleavedAccessInfo IAI(PSE, L, DT, LI, LVL.getLAI());
9913 bool UseInterleaved = TTI->enableInterleavedAccessVectorization();
9914
9915 // If an override option has been passed in for interleaved accesses, use it.
9916 if (EnableInterleavedMemAccesses.getNumOccurrences() > 0)
9917 UseInterleaved = EnableInterleavedMemAccesses;
9918
9919 // Analyze interleaved memory accesses.
9920 if (UseInterleaved)
9922
9923 if (LVL.hasUncountableEarlyExit()) {
9924 BasicBlock *LoopLatch = L->getLoopLatch();
9925 if (IAI.requiresScalarEpilogue() ||
9927 [LoopLatch](BasicBlock *BB) { return BB != LoopLatch; })) {
9928 reportVectorizationFailure("Auto-vectorization of early exit loops "
9929 "requiring a scalar epilogue is unsupported",
9930 "UncountableEarlyExitUnsupported", ORE, L);
9931 return false;
9932 }
9933 }
9934
9935 // Check the function attributes and profiles to find out if this function
9936 // should be optimized for size.
9938 getScalarEpilogueLowering(F, L, Hints, PSI, BFI, TTI, TLI, LVL, &IAI);
9939
9940 // Check the loop for a trip count threshold: vectorize loops with a tiny trip
9941 // count by optimizing for size, to minimize overheads.
9942 auto ExpectedTC = getSmallBestKnownTC(PSE, L);
9943 if (ExpectedTC && ExpectedTC->isFixed() &&
9944 ExpectedTC->getFixedValue() < TinyTripCountVectorThreshold) {
9945 LLVM_DEBUG(dbgs() << "LV: Found a loop with a very small trip count. "
9946 << "This loop is worth vectorizing only if no scalar "
9947 << "iteration overheads are incurred.");
9949 LLVM_DEBUG(dbgs() << " But vectorizing was explicitly forced.\n");
9950 else {
9951 LLVM_DEBUG(dbgs() << "\n");
9952 // Predicate tail-folded loops are efficient even when the loop
9953 // iteration count is low. However, setting the epilogue policy to
9954 // `CM_ScalarEpilogueNotAllowedLowTripLoop` prevents vectorizing loops
9955 // with runtime checks. It's more effective to let
9956 // `isOutsideLoopWorkProfitable` determine if vectorization is
9957 // beneficial for the loop.
9960 }
9961 }
9962
9963 // Check the function attributes to see if implicit floats or vectors are
9964 // allowed.
9965 if (F->hasFnAttribute(Attribute::NoImplicitFloat)) {
9967 "Can't vectorize when the NoImplicitFloat attribute is used",
9968 "loop not vectorized due to NoImplicitFloat attribute",
9969 "NoImplicitFloat", ORE, L);
9970 Hints.emitRemarkWithHints();
9971 return false;
9972 }
9973
9974 // Check if the target supports potentially unsafe FP vectorization.
9975 // FIXME: Add a check for the type of safety issue (denormal, signaling)
9976 // for the target we're vectorizing for, to make sure none of the
9977 // additional fp-math flags can help.
9978 if (Hints.isPotentiallyUnsafe() &&
9979 TTI->isFPVectorizationPotentiallyUnsafe()) {
9981 "Potentially unsafe FP op prevents vectorization",
9982 "loop not vectorized due to unsafe FP support.",
9983 "UnsafeFP", ORE, L);
9984 Hints.emitRemarkWithHints();
9985 return false;
9986 }
9987
9988 bool AllowOrderedReductions;
9989 // If the flag is set, use that instead and override the TTI behaviour.
9990 if (ForceOrderedReductions.getNumOccurrences() > 0)
9991 AllowOrderedReductions = ForceOrderedReductions;
9992 else
9993 AllowOrderedReductions = TTI->enableOrderedReductions();
9994 if (!LVL.canVectorizeFPMath(AllowOrderedReductions)) {
9995 ORE->emit([&]() {
9996 auto *ExactFPMathInst = Requirements.getExactFPInst();
9997 return OptimizationRemarkAnalysisFPCommute(DEBUG_TYPE, "CantReorderFPOps",
9998 ExactFPMathInst->getDebugLoc(),
9999 ExactFPMathInst->getParent())
10000 << "loop not vectorized: cannot prove it is safe to reorder "
10001 "floating-point operations";
10002 });
10003 LLVM_DEBUG(dbgs() << "LV: loop not vectorized: cannot prove it is safe to "
10004 "reorder floating-point operations\n");
10005 Hints.emitRemarkWithHints();
10006 return false;
10007 }
10008
10009 // Use the cost model.
10010 LoopVectorizationCostModel CM(SEL, L, PSE, LI, &LVL, *TTI, TLI, DB, AC, ORE,
10011 F, &Hints, IAI, PSI, BFI);
10012 // Use the planner for vectorization.
10013 LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, &LVL, CM, IAI, PSE, Hints,
10014 ORE);
10015
10016 // Get user vectorization factor and interleave count.
10017 ElementCount UserVF = Hints.getWidth();
10018 unsigned UserIC = Hints.getInterleave();
10019
10020 // Plan how to best vectorize.
10021 LVP.plan(UserVF, UserIC);
10023 unsigned IC = 1;
10024
10025 if (ORE->allowExtraAnalysis(LV_NAME))
10027
10028 GeneratedRTChecks Checks(PSE, DT, LI, TTI, F->getDataLayout(), CM.CostKind);
10029 if (LVP.hasPlanWithVF(VF.Width)) {
10030 // Select the interleave count.
10031 IC = LVP.selectInterleaveCount(LVP.getPlanFor(VF.Width), VF.Width, VF.Cost);
10032
10033 unsigned SelectedIC = std::max(IC, UserIC);
10034 // Optimistically generate runtime checks if they are needed. Drop them if
10035 // they turn out to not be profitable.
10036 if (VF.Width.isVector() || SelectedIC > 1) {
10037 Checks.create(L, *LVL.getLAI(), PSE.getPredicate(), VF.Width, SelectedIC);
10038
10039 // Bail out early if either the SCEV or memory runtime checks are known to
10040 // fail. In that case, the vector loop would never execute.
10041 using namespace llvm::PatternMatch;
10042 if (Checks.getSCEVChecks().first &&
10043 match(Checks.getSCEVChecks().first, m_One()))
10044 return false;
10045 if (Checks.getMemRuntimeChecks().first &&
10046 match(Checks.getMemRuntimeChecks().first, m_One()))
10047 return false;
10048 }
10049
10050 // Check if it is profitable to vectorize with runtime checks.
10051 bool ForceVectorization =
10053 VPCostContext CostCtx(CM.TTI, *CM.TLI, LVP.getPlanFor(VF.Width), CM,
10054 CM.CostKind);
10055 if (!ForceVectorization &&
10056 !isOutsideLoopWorkProfitable(Checks, VF, L, PSE, CostCtx,
10057 LVP.getPlanFor(VF.Width), SEL,
10058 CM.getVScaleForTuning())) {
10059 ORE->emit([&]() {
10061 DEBUG_TYPE, "CantReorderMemOps", L->getStartLoc(),
10062 L->getHeader())
10063 << "loop not vectorized: cannot prove it is safe to reorder "
10064 "memory operations";
10065 });
10066 LLVM_DEBUG(dbgs() << "LV: Too many memory checks needed.\n");
10067 Hints.emitRemarkWithHints();
10068 return false;
10069 }
10070 }
10071
10072 // Identify the diagnostic messages that should be produced.
10073 std::pair<StringRef, std::string> VecDiagMsg, IntDiagMsg;
10074 bool VectorizeLoop = true, InterleaveLoop = true;
10075 if (VF.Width.isScalar()) {
10076 LLVM_DEBUG(dbgs() << "LV: Vectorization is possible but not beneficial.\n");
10077 VecDiagMsg = {
10078 "VectorizationNotBeneficial",
10079 "the cost-model indicates that vectorization is not beneficial"};
10080 VectorizeLoop = false;
10081 }
10082
10083 if (!LVP.hasPlanWithVF(VF.Width) && UserIC > 1) {
10084 // Tell the user interleaving was avoided up-front, despite being explicitly
10085 // requested.
10086 LLVM_DEBUG(dbgs() << "LV: Ignoring UserIC, because vectorization and "
10087 "interleaving should be avoided up front\n");
10088 IntDiagMsg = {"InterleavingAvoided",
10089 "Ignoring UserIC, because interleaving was avoided up front"};
10090 InterleaveLoop = false;
10091 } else if (IC == 1 && UserIC <= 1) {
10092 // Tell the user interleaving is not beneficial.
10093 LLVM_DEBUG(dbgs() << "LV: Interleaving is not beneficial.\n");
10094 IntDiagMsg = {
10095 "InterleavingNotBeneficial",
10096 "the cost-model indicates that interleaving is not beneficial"};
10097 InterleaveLoop = false;
10098 if (UserIC == 1) {
10099 IntDiagMsg.first = "InterleavingNotBeneficialAndDisabled";
10100 IntDiagMsg.second +=
10101 " and is explicitly disabled or interleave count is set to 1";
10102 }
10103 } else if (IC > 1 && UserIC == 1) {
10104 // Tell the user interleaving is beneficial, but it explicitly disabled.
10105 LLVM_DEBUG(dbgs() << "LV: Interleaving is beneficial but is explicitly "
10106 "disabled.\n");
10107 IntDiagMsg = {"InterleavingBeneficialButDisabled",
10108 "the cost-model indicates that interleaving is beneficial "
10109 "but is explicitly disabled or interleave count is set to 1"};
10110 InterleaveLoop = false;
10111 }
10112
10113 // If there is a histogram in the loop, do not just interleave without
10114 // vectorizing. The order of operations will be incorrect without the
10115 // histogram intrinsics, which are only used for recipes with VF > 1.
10116 if (!VectorizeLoop && InterleaveLoop && LVL.hasHistograms()) {
10117 LLVM_DEBUG(dbgs() << "LV: Not interleaving without vectorization due "
10118 << "to histogram operations.\n");
10119 IntDiagMsg = {
10120 "HistogramPreventsScalarInterleaving",
10121 "Unable to interleave without vectorization due to constraints on "
10122 "the order of histogram operations"};
10123 InterleaveLoop = false;
10124 }
10125
10126 // Override IC if user provided an interleave count.
10127 IC = UserIC > 0 ? UserIC : IC;
10128
10129 // Emit diagnostic messages, if any.
10130 const char *VAPassName = Hints.vectorizeAnalysisPassName();
10131 if (!VectorizeLoop && !InterleaveLoop) {
10132 // Do not vectorize or interleaving the loop.
10133 ORE->emit([&]() {
10134 return OptimizationRemarkMissed(VAPassName, VecDiagMsg.first,
10135 L->getStartLoc(), L->getHeader())
10136 << VecDiagMsg.second;
10137 });
10138 ORE->emit([&]() {
10139 return OptimizationRemarkMissed(LV_NAME, IntDiagMsg.first,
10140 L->getStartLoc(), L->getHeader())
10141 << IntDiagMsg.second;
10142 });
10143 return false;
10144 }
10145
10146 if (!VectorizeLoop && InterleaveLoop) {
10147 LLVM_DEBUG(dbgs() << "LV: Interleave Count is " << IC << '\n');
10148 ORE->emit([&]() {
10149 return OptimizationRemarkAnalysis(VAPassName, VecDiagMsg.first,
10150 L->getStartLoc(), L->getHeader())
10151 << VecDiagMsg.second;
10152 });
10153 } else if (VectorizeLoop && !InterleaveLoop) {
10154 LLVM_DEBUG(dbgs() << "LV: Found a vectorizable loop (" << VF.Width
10155 << ") in " << L->getLocStr() << '\n');
10156 ORE->emit([&]() {
10157 return OptimizationRemarkAnalysis(LV_NAME, IntDiagMsg.first,
10158 L->getStartLoc(), L->getHeader())
10159 << IntDiagMsg.second;
10160 });
10161 } else if (VectorizeLoop && InterleaveLoop) {
10162 LLVM_DEBUG(dbgs() << "LV: Found a vectorizable loop (" << VF.Width
10163 << ") in " << L->getLocStr() << '\n');
10164 LLVM_DEBUG(dbgs() << "LV: Interleave Count is " << IC << '\n');
10165 }
10166
10167 // Report the vectorization decision.
10168 if (VF.Width.isScalar()) {
10169 using namespace ore;
10170 assert(IC > 1);
10171 ORE->emit([&]() {
10172 return OptimizationRemark(LV_NAME, "Interleaved", L->getStartLoc(),
10173 L->getHeader())
10174 << "interleaved loop (interleaved count: "
10175 << NV("InterleaveCount", IC) << ")";
10176 });
10177 } else {
10178 // Report the vectorization decision.
10179 reportVectorization(ORE, L, VF, IC);
10180 }
10181 if (ORE->allowExtraAnalysis(LV_NAME))
10183
10184 // If we decided that it is *legal* to interleave or vectorize the loop, then
10185 // do it.
10186
10187 VPlan &BestPlan = LVP.getPlanFor(VF.Width);
10188 // Consider vectorizing the epilogue too if it's profitable.
10189 VectorizationFactor EpilogueVF =
10191 if (EpilogueVF.Width.isVector()) {
10192 std::unique_ptr<VPlan> BestMainPlan(BestPlan.duplicate());
10193
10194 // The first pass vectorizes the main loop and creates a scalar epilogue
10195 // to be vectorized by executing the plan (potentially with a different
10196 // factor) again shortly afterwards.
10197 VPlan &BestEpiPlan = LVP.getPlanFor(EpilogueVF.Width);
10198 BestEpiPlan.getMiddleBlock()->setName("vec.epilog.middle.block");
10199 BestEpiPlan.getVectorPreheader()->setName("vec.epilog.ph");
10200 preparePlanForMainVectorLoop(*BestMainPlan, BestEpiPlan);
10201 EpilogueLoopVectorizationInfo EPI(VF.Width, IC, EpilogueVF.Width, 1,
10202 BestEpiPlan);
10203 EpilogueVectorizerMainLoop MainILV(L, PSE, LI, DT, TTI, AC, EPI, &CM, BFI,
10204 PSI, Checks, *BestMainPlan);
10205 auto ExpandedSCEVs = LVP.executePlan(EPI.MainLoopVF, EPI.MainLoopUF,
10206 *BestMainPlan, MainILV, DT, false);
10207 ++LoopsVectorized;
10208
10209 // Second pass vectorizes the epilogue and adjusts the control flow
10210 // edges from the first pass.
10211 EpilogueVectorizerEpilogueLoop EpilogILV(L, PSE, LI, DT, TTI, AC, EPI, &CM,
10212 BFI, PSI, Checks, BestEpiPlan);
10214 BestEpiPlan, L, ExpandedSCEVs, EPI, CM, *PSE.getSE());
10215 LVP.executePlan(EPI.EpilogueVF, EPI.EpilogueUF, BestEpiPlan, EpilogILV, DT,
10216 true);
10217 connectEpilogueVectorLoop(BestEpiPlan, L, EPI, DT, LVL, ExpandedSCEVs,
10218 Checks, InstsToMove);
10219 ++LoopsEpilogueVectorized;
10220 } else {
10221 InnerLoopVectorizer LB(L, PSE, LI, DT, TTI, AC, VF.Width, IC, &CM, BFI, PSI,
10222 Checks, BestPlan);
10223 // TODO: Move to general VPlan pipeline once epilogue loops are also
10224 // supported.
10227 IC, PSE);
10228 LVP.addMinimumIterationCheck(BestPlan, VF.Width, IC,
10230
10231 LVP.executePlan(VF.Width, IC, BestPlan, LB, DT, false);
10232 ++LoopsVectorized;
10233 }
10234
10235 assert(DT->verify(DominatorTree::VerificationLevel::Fast) &&
10236 "DT not preserved correctly");
10237 assert(!verifyFunction(*F, &dbgs()));
10238
10239 return true;
10240}
10241
10243
10244 // Don't attempt if
10245 // 1. the target claims to have no vector registers, and
10246 // 2. interleaving won't help ILP.
10247 //
10248 // The second condition is necessary because, even if the target has no
10249 // vector registers, loop vectorization may still enable scalar
10250 // interleaving.
10251 if (!TTI->getNumberOfRegisters(TTI->getRegisterClassForType(true)) &&
10252 TTI->getMaxInterleaveFactor(ElementCount::getFixed(1)) < 2)
10253 return LoopVectorizeResult(false, false);
10254
10255 bool Changed = false, CFGChanged = false;
10256
10257 // The vectorizer requires loops to be in simplified form.
10258 // Since simplification may add new inner loops, it has to run before the
10259 // legality and profitability checks. This means running the loop vectorizer
10260 // will simplify all loops, regardless of whether anything end up being
10261 // vectorized.
10262 for (const auto &L : *LI)
10263 Changed |= CFGChanged |=
10264 simplifyLoop(L, DT, LI, SE, AC, nullptr, false /* PreserveLCSSA */);
10265
10266 // Build up a worklist of inner-loops to vectorize. This is necessary as
10267 // the act of vectorizing or partially unrolling a loop creates new loops
10268 // and can invalidate iterators across the loops.
10269 SmallVector<Loop *, 8> Worklist;
10270
10271 for (Loop *L : *LI)
10272 collectSupportedLoops(*L, LI, ORE, Worklist);
10273
10274 LoopsAnalyzed += Worklist.size();
10275
10276 // Now walk the identified inner loops.
10277 while (!Worklist.empty()) {
10278 Loop *L = Worklist.pop_back_val();
10279
10280 // For the inner loops we actually process, form LCSSA to simplify the
10281 // transform.
10282 Changed |= formLCSSARecursively(*L, *DT, LI, SE);
10283
10284 Changed |= CFGChanged |= processLoop(L);
10285
10286 if (Changed) {
10287 LAIs->clear();
10288
10289#ifndef NDEBUG
10290 if (VerifySCEV)
10291 SE->verify();
10292#endif
10293 }
10294 }
10295
10296 // Process each loop nest in the function.
10297 return LoopVectorizeResult(Changed, CFGChanged);
10298}
10299
10302 LI = &AM.getResult<LoopAnalysis>(F);
10303 // There are no loops in the function. Return before computing other
10304 // expensive analyses.
10305 if (LI->empty())
10306 return PreservedAnalyses::all();
10315 AA = &AM.getResult<AAManager>(F);
10316
10317 auto &MAMProxy = AM.getResult<ModuleAnalysisManagerFunctionProxy>(F);
10318 PSI = MAMProxy.getCachedResult<ProfileSummaryAnalysis>(*F.getParent());
10319 BFI = nullptr;
10320 if (PSI && PSI->hasProfileSummary())
10322 LoopVectorizeResult Result = runImpl(F);
10323 if (!Result.MadeAnyChange)
10324 return PreservedAnalyses::all();
10326
10327 if (isAssignmentTrackingEnabled(*F.getParent())) {
10328 for (auto &BB : F)
10330 }
10331
10332 PA.preserve<LoopAnalysis>();
10336
10337 if (Result.MadeCFGChange) {
10338 // Making CFG changes likely means a loop got vectorized. Indicate that
10339 // extra simplification passes should be run.
10340 // TODO: MadeCFGChanges is not a prefect proxy. Extra passes should only
10341 // be run if runtime checks have been added.
10344 } else {
10346 }
10347 return PA;
10348}
10349
10351 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
10352 static_cast<PassInfoMixin<LoopVectorizePass> *>(this)->printPipeline(
10353 OS, MapClassName2PassName);
10354
10355 OS << '<';
10356 OS << (InterleaveOnlyWhenForced ? "" : "no-") << "interleave-forced-only;";
10357 OS << (VectorizeOnlyWhenForced ? "" : "no-") << "vectorize-forced-only;";
10358 OS << '>';
10359}
for(const MachineOperand &MO :llvm::drop_begin(OldMI.operands(), Desc.getNumOperands()))
static unsigned getIntrinsicID(const SDNode *N)
unsigned RegSize
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
aarch64 promote const
AMDGPU Lower Kernel Arguments
AMDGPU Register Bank Select
Rewrite undef for PHI
This file implements a class to represent arbitrary precision integral constant values and operations...
@ PostInc
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static bool isEqual(const Function &Caller, const Function &Callee)
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
This is the interface for LLVM's primary stateless and local alias analysis.
static bool IsEmptyBlock(MachineBasicBlock *MBB)
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static cl::opt< OutputCostKind > CostKind("cost-kind", cl::desc("Target cost kind"), cl::init(OutputCostKind::RecipThroughput), cl::values(clEnumValN(OutputCostKind::RecipThroughput, "throughput", "Reciprocal throughput"), clEnumValN(OutputCostKind::Latency, "latency", "Instruction latency"), clEnumValN(OutputCostKind::CodeSize, "code-size", "Code size"), clEnumValN(OutputCostKind::SizeAndLatency, "size-latency", "Code size and latency"), clEnumValN(OutputCostKind::All, "all", "Print all cost kinds")))
static cl::opt< IntrinsicCostStrategy > IntrinsicCost("intrinsic-cost-strategy", cl::desc("Costing strategy for intrinsic instructions"), cl::init(IntrinsicCostStrategy::InstructionCost), cl::values(clEnumValN(IntrinsicCostStrategy::InstructionCost, "instruction-cost", "Use TargetTransformInfo::getInstructionCost"), clEnumValN(IntrinsicCostStrategy::IntrinsicCost, "intrinsic-cost", "Use TargetTransformInfo::getIntrinsicInstrCost"), clEnumValN(IntrinsicCostStrategy::TypeBasedIntrinsicCost, "type-based-intrinsic-cost", "Calculate the intrinsic cost based only on argument types")))
static InstructionCost getCost(Instruction &Inst, TTI::TargetCostKind CostKind, TargetTransformInfo &TTI, TargetLibraryInfo &TLI)
Definition CostModel.cpp:74
This file defines DenseMapInfo traits for DenseMap.
This file defines the DenseMap class.
#define DEBUG_TYPE
This is the interface for a simple mod/ref and alias analysis over globals.
Hexagon Common GEP
#define _
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
Module.h This file contains the declarations for the Module class.
This defines the Use class.
static bool hasNoUnsignedWrap(BinaryOperator &I)
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
static std::pair< Value *, APInt > getMask(Value *WideMask, unsigned Factor, ElementCount LeafValueEC)
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Legalize the Machine IR a function s Machine IR
Definition Legalizer.cpp:80
static cl::opt< unsigned, true > VectorizationFactor("force-vector-width", cl::Hidden, cl::desc("Sets the SIMD width. Zero is autoselect."), cl::location(VectorizerParams::VectorizationFactor))
This header provides classes for managing per-loop analyses.
static const char * VerboseDebug
#define LV_NAME
This file defines the LoopVectorizationLegality class.
This file provides a LoopVectorizationPlanner class.
static void collectSupportedLoops(Loop &L, LoopInfo *LI, OptimizationRemarkEmitter *ORE, SmallVectorImpl< Loop * > &V)
static cl::opt< unsigned > EpilogueVectorizationMinVF("epilogue-vectorization-minimum-VF", cl::Hidden, cl::desc("Only loops with vectorization factor equal to or larger than " "the specified value are considered for epilogue vectorization."))
static cl::opt< unsigned > EpilogueVectorizationForceVF("epilogue-vectorization-force-VF", cl::init(1), cl::Hidden, cl::desc("When epilogue vectorization is enabled, and a value greater than " "1 is specified, forces the given VF for all applicable epilogue " "loops."))
static void addScalarResumePhis(VPRecipeBuilder &Builder, VPlan &Plan, DenseMap< VPValue *, VPValue * > &IVEndValues)
Create resume phis in the scalar preheader for first-order recurrences, reductions and inductions,...
static Type * maybeVectorizeType(Type *Ty, ElementCount VF)
static ElementCount determineVPlanVF(const TargetTransformInfo &TTI, LoopVectorizationCostModel &CM)
static ElementCount getSmallConstantTripCount(ScalarEvolution *SE, const Loop *L)
A version of ScalarEvolution::getSmallConstantTripCount that returns an ElementCount to include loops...
static cl::opt< unsigned > VectorizeMemoryCheckThreshold("vectorize-memory-check-threshold", cl::init(128), cl::Hidden, cl::desc("The maximum allowed number of runtime memory checks"))
static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan)
Prepare MainPlan for vectorizing the main vector loop during epilogue vectorization.
static cl::opt< unsigned > TinyTripCountVectorThreshold("vectorizer-min-trip-count", cl::init(16), cl::Hidden, cl::desc("Loops with a constant trip count that is smaller than this " "value are vectorized only if no scalar iteration overheads " "are incurred."))
Loops with a known constant trip count below this number are vectorized only if no scalar iteration o...
static void debugVectorizationMessage(const StringRef Prefix, const StringRef DebugMsg, Instruction *I)
Write a DebugMsg about vectorization to the debug output stream.
static cl::opt< bool > EnableCondStoresVectorization("enable-cond-stores-vec", cl::init(true), cl::Hidden, cl::desc("Enable if predication of stores during vectorization."))
static void legacyCSE(BasicBlock *BB)
FIXME: This legacy common-subexpression-elimination routine is scheduled for removal,...
static VPIRBasicBlock * replaceVPBBWithIRVPBB(VPBasicBlock *VPBB, BasicBlock *IRBB, VPlan *Plan=nullptr)
Replace VPBB with a VPIRBasicBlock wrapping IRBB.
static VPInstruction * addResumePhiRecipeForInduction(VPWidenInductionRecipe *WideIV, VPBuilder &VectorPHBuilder, VPBuilder &ScalarPHBuilder, VPTypeAnalysis &TypeInfo, VPValue *VectorTC)
Create and return a ResumePhi for WideIV, unless it is truncated.
static Value * emitTransformedIndex(IRBuilderBase &B, Value *Index, Value *StartValue, Value *Step, InductionDescriptor::InductionKind InductionKind, const BinaryOperator *InductionBinOp)
Compute the transformed value of Index at offset StartValue using step StepValue.
static DebugLoc getDebugLocFromInstOrOperands(Instruction *I)
Look for a meaningful debug location on the instruction or its operands.
static Value * createInductionAdditionalBypassValues(PHINode *OrigPhi, const InductionDescriptor &II, IRBuilder<> &BypassBuilder, const SCEV2ValueTy &ExpandedSCEVs, Value *MainVectorTripCount, Instruction *OldInduction)
static void fixReductionScalarResumeWhenVectorizingEpilog(VPPhi *EpiResumePhiR, PHINode &EpiResumePhi, BasicBlock *BypassBlock)
static Value * getStartValueFromReductionResult(VPInstruction *RdxResult)
static cl::opt< bool > ForceTargetSupportsScalableVectors("force-target-supports-scalable-vectors", cl::init(false), cl::Hidden, cl::desc("Pretend that scalable vectors are supported, even if the target does " "not support them. This flag should only be used for testing."))
static bool useActiveLaneMaskForControlFlow(TailFoldingStyle Style)
static cl::opt< bool > EnableEarlyExitVectorization("enable-early-exit-vectorization", cl::init(true), cl::Hidden, cl::desc("Enable vectorization of early exit loops with uncountable exits."))
static cl::opt< bool > ConsiderRegPressure("vectorizer-consider-reg-pressure", cl::init(false), cl::Hidden, cl::desc("Discard VFs if their register pressure is too high."))
static unsigned estimateElementCount(ElementCount VF, std::optional< unsigned > VScale)
This function attempts to return a value that represents the ElementCount at runtime.
static constexpr uint32_t MinItersBypassWeights[]
static cl::opt< unsigned > ForceTargetNumScalarRegs("force-target-num-scalar-regs", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's number of scalar registers."))
static cl::opt< bool > UseWiderVFIfCallVariantsPresent("vectorizer-maximize-bandwidth-for-vector-calls", cl::init(true), cl::Hidden, cl::desc("Try wider VFs if they enable the use of vector variants"))
static std::optional< unsigned > getMaxVScale(const Function &F, const TargetTransformInfo &TTI)
static cl::opt< unsigned > SmallLoopCost("small-loop-cost", cl::init(20), cl::Hidden, cl::desc("The cost of a loop that is considered 'small' by the interleaver."))
static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L, EpilogueLoopVectorizationInfo &EPI, DominatorTree *DT, LoopVectorizationLegality &LVL, DenseMap< const SCEV *, Value * > &ExpandedSCEVs, GeneratedRTChecks &Checks, ArrayRef< Instruction * > InstsToMove)
Connect the epilogue vector loop generated for EpiPlan to the main vector.
static bool planContainsAdditionalSimplifications(VPlan &Plan, VPCostContext &CostCtx, Loop *TheLoop, ElementCount VF)
Return true if the original loop \ TheLoop contains any instructions that do not have corresponding r...
static cl::opt< unsigned > ForceTargetNumVectorRegs("force-target-num-vector-regs", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's number of vector registers."))
static bool isExplicitVecOuterLoop(Loop *OuterLp, OptimizationRemarkEmitter *ORE)
static cl::opt< bool > EnableIndVarRegisterHeur("enable-ind-var-reg-heur", cl::init(true), cl::Hidden, cl::desc("Count the induction variable only once when interleaving"))
static cl::opt< TailFoldingStyle > ForceTailFoldingStyle("force-tail-folding-style", cl::desc("Force the tail folding style"), cl::init(TailFoldingStyle::None), cl::values(clEnumValN(TailFoldingStyle::None, "none", "Disable tail folding"), clEnumValN(TailFoldingStyle::Data, "data", "Create lane mask for data only, using active.lane.mask intrinsic"), clEnumValN(TailFoldingStyle::DataWithoutLaneMask, "data-without-lane-mask", "Create lane mask with compare/stepvector"), clEnumValN(TailFoldingStyle::DataAndControlFlow, "data-and-control", "Create lane mask using active.lane.mask intrinsic, and use " "it for both data and control flow"), clEnumValN(TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck, "data-and-control-without-rt-check", "Similar to data-and-control, but remove the runtime check"), clEnumValN(TailFoldingStyle::DataWithEVL, "data-with-evl", "Use predicated EVL instructions for tail folding. If EVL " "is unsupported, fallback to data-without-lane-mask.")))
static cl::opt< bool > EnableEpilogueVectorization("enable-epilogue-vectorization", cl::init(true), cl::Hidden, cl::desc("Enable vectorization of epilogue loops."))
static ScalarEpilogueLowering getScalarEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI, TargetTransformInfo *TTI, TargetLibraryInfo *TLI, LoopVectorizationLegality &LVL, InterleavedAccessInfo *IAI)
static cl::opt< bool > PreferPredicatedReductionSelect("prefer-predicated-reduction-select", cl::init(false), cl::Hidden, cl::desc("Prefer predicating a reduction operation over an after loop select."))
static VPWidenIntOrFpInductionRecipe * createWidenInductionRecipes(PHINode *Phi, Instruction *PhiOrTrunc, VPValue *Start, const InductionDescriptor &IndDesc, VPlan &Plan, ScalarEvolution &SE, Loop &OrigLoop)
Creates a VPWidenIntOrFpInductionRecpipe for Phi.
static cl::opt< bool > PreferInLoopReductions("prefer-inloop-reductions", cl::init(false), cl::Hidden, cl::desc("Prefer in-loop vector reductions, " "overriding the targets preference."))
static SmallVector< Instruction * > preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L, const SCEV2ValueTy &ExpandedSCEVs, EpilogueLoopVectorizationInfo &EPI, LoopVectorizationCostModel &CM, ScalarEvolution &SE)
Prepare Plan for vectorizing the epilogue loop.
static cl::opt< bool > EnableLoadStoreRuntimeInterleave("enable-loadstore-runtime-interleave", cl::init(true), cl::Hidden, cl::desc("Enable runtime interleaving until load/store ports are saturated"))
static cl::opt< bool > VPlanBuildStressTest("vplan-build-stress-test", cl::init(false), cl::Hidden, cl::desc("Build VPlan for every supported loop nest in the function and bail " "out right after the build (stress test the VPlan H-CFG construction " "in the VPlan-native vectorization path)."))
static bool hasIrregularType(Type *Ty, const DataLayout &DL)
A helper function that returns true if the given type is irregular.
static cl::opt< bool > LoopVectorizeWithBlockFrequency("loop-vectorize-with-block-frequency", cl::init(true), cl::Hidden, cl::desc("Enable the use of the block frequency analysis to access PGO " "heuristics minimizing code growth in cold regions and being more " "aggressive in hot regions."))
static std::optional< ElementCount > getSmallBestKnownTC(PredicatedScalarEvolution &PSE, Loop *L, bool CanUseConstantMax=true)
Returns "best known" trip count, which is either a valid positive trip count or std::nullopt when an ...
static Value * getExpandedStep(const InductionDescriptor &ID, const SCEV2ValueTy &ExpandedSCEVs)
Return the expanded step for ID using ExpandedSCEVs to look up SCEV expansion results.
static bool useActiveLaneMask(TailFoldingStyle Style)
static bool hasReplicatorRegion(VPlan &Plan)
static bool isIndvarOverflowCheckKnownFalse(const LoopVectorizationCostModel *Cost, ElementCount VF, std::optional< unsigned > UF=std::nullopt)
For the given VF and UF and maximum trip count computed for the loop, return whether the induction va...
static void addFullyUnrolledInstructionsToIgnore(Loop *L, const LoopVectorizationLegality::InductionList &IL, SmallPtrSetImpl< Instruction * > &InstsToIgnore)
Knowing that loop L executes a single vector iteration, add instructions that will get simplified and...
static cl::opt< PreferPredicateTy::Option > PreferPredicateOverEpilogue("prefer-predicate-over-epilogue", cl::init(PreferPredicateTy::ScalarEpilogue), cl::Hidden, cl::desc("Tail-folding and predication preferences over creating a scalar " "epilogue loop."), cl::values(clEnumValN(PreferPredicateTy::ScalarEpilogue, "scalar-epilogue", "Don't tail-predicate loops, create scalar epilogue"), clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue, "predicate-else-scalar-epilogue", "prefer tail-folding, create scalar epilogue if tail " "folding fails."), clEnumValN(PreferPredicateTy::PredicateOrDontVectorize, "predicate-dont-vectorize", "prefers tail-folding, don't attempt vectorization if " "tail-folding fails.")))
static cl::opt< bool > EnableInterleavedMemAccesses("enable-interleaved-mem-accesses", cl::init(false), cl::Hidden, cl::desc("Enable vectorization on interleaved memory accesses in a loop"))
static cl::opt< bool > EnableMaskedInterleavedMemAccesses("enable-masked-interleaved-mem-accesses", cl::init(false), cl::Hidden, cl::desc("Enable vectorization on masked interleaved memory accesses in a loop"))
An interleave-group may need masking if it resides in a block that needs predication,...
static cl::opt< bool > ForceOrderedReductions("force-ordered-reductions", cl::init(false), cl::Hidden, cl::desc("Enable the vectorisation of loops with in-order (strict) " "FP reductions"))
static const SCEV * getAddressAccessSCEV(Value *Ptr, LoopVectorizationLegality *Legal, PredicatedScalarEvolution &PSE, const Loop *TheLoop)
Gets Address Access SCEV after verifying that the access pattern is loop invariant except the inducti...
static cl::opt< cl::boolOrDefault > ForceSafeDivisor("force-widen-divrem-via-safe-divisor", cl::Hidden, cl::desc("Override cost based safe divisor widening for div/rem instructions"))
static InstructionCost calculateEarlyExitCost(VPCostContext &CostCtx, VPlan &Plan, ElementCount VF)
For loops with uncountable early exits, find the cost of doing work when exiting the loop early,...
static cl::opt< unsigned > ForceTargetMaxVectorInterleaveFactor("force-target-max-vector-interleave", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's max interleave factor for " "vectorized loops."))
static bool processLoopInVPlanNativePath(Loop *L, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, LoopVectorizationLegality *LVL, TargetTransformInfo *TTI, TargetLibraryInfo *TLI, DemandedBits *DB, AssumptionCache *AC, OptimizationRemarkEmitter *ORE, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, LoopVectorizeHints &Hints, LoopVectorizationRequirements &Requirements)
static bool useMaskedInterleavedAccesses(const TargetTransformInfo &TTI)
static cl::opt< unsigned > NumberOfStoresToPredicate("vectorize-num-stores-pred", cl::init(1), cl::Hidden, cl::desc("Max number of stores to be predicated behind an if."))
The number of stores in a loop that are allowed to need predication.
static cl::opt< unsigned > MaxNestedScalarReductionIC("max-nested-scalar-reduction-interleave", cl::init(2), cl::Hidden, cl::desc("The maximum interleave count to use when interleaving a scalar " "reduction in a nested loop."))
static cl::opt< unsigned > ForceTargetMaxScalarInterleaveFactor("force-target-max-scalar-interleave", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's max interleave factor for " "scalar loops."))
static void checkMixedPrecision(Loop *L, OptimizationRemarkEmitter *ORE)
static bool willGenerateVectors(VPlan &Plan, ElementCount VF, const TargetTransformInfo &TTI)
Check if any recipe of Plan will generate a vector value, which will be assigned a vector register.
static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks, VectorizationFactor &VF, Loop *L, PredicatedScalarEvolution &PSE, VPCostContext &CostCtx, VPlan &Plan, ScalarEpilogueLowering SEL, std::optional< unsigned > VScale)
This function determines whether or not it's still profitable to vectorize the loop given the extra w...
static void addExitUsersForFirstOrderRecurrences(VPlan &Plan, VFRange &Range)
Handle users in the exit block for first order reductions in the original exit block.
static void fixScalarResumeValuesFromBypass(BasicBlock *BypassBlock, Loop *L, VPlan &BestEpiPlan, LoopVectorizationLegality &LVL, const SCEV2ValueTy &ExpandedSCEVs, Value *MainVectorTripCount)
static cl::opt< bool > MaximizeBandwidth("vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden, cl::desc("Maximize bandwidth when selecting vectorization factor which " "will be determined by the smallest type in loop."))
static OptimizationRemarkAnalysis createLVAnalysis(const char *PassName, StringRef RemarkName, Loop *TheLoop, Instruction *I, DebugLoc DL={})
Create an analysis remark that explains why vectorization failed.
#define F(x, y, z)
Definition MD5.cpp:55
#define I(x, y, z)
Definition MD5.cpp:58
mir Rename Register Operands
This file implements a map that provides insertion order iteration.
This file contains the declarations for metadata subclasses.
#define T
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
#define P(N)
This file contains the declarations for profiling metadata utility functions.
const SmallVectorImpl< MachineOperand > & Cond
static BinaryOperator * CreateMul(Value *S1, Value *S2, const Twine &Name, BasicBlock::iterator InsertBefore, Value *FlagsOp)
static BinaryOperator * CreateAdd(Value *S1, Value *S2, const Twine &Name, BasicBlock::iterator InsertBefore, Value *FlagsOp)
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
static InstructionCost getScalarizationOverhead(const TargetTransformInfo &TTI, Type *ScalarTy, VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract, TTI::TargetCostKind CostKind, bool ForPoisonSrc=true, ArrayRef< Value * > VL={})
This is similar to TargetTransformInfo::getScalarizationOverhead, but if ScalarTy is a FixedVectorTyp...
This file contains some templates that are useful if you are working with the STL at all.
#define OP(OPC)
Definition Instruction.h:46
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition Statistic.h:171
#define LLVM_DEBUG(...)
Definition Debug.h:114
#define DEBUG_WITH_TYPE(TYPE,...)
DEBUG_WITH_TYPE macro - This macro should be used by passes to emit debug information.
Definition Debug.h:72
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")
This pass exposes codegen information to IR-level passes.
LocallyHashedType DenseMapInfo< LocallyHashedType >::Empty
This file implements the TypeSwitch template, which mimics a switch() statement whose cases are type ...
This file contains the declarations of different VPlan-related auxiliary helpers.
This file provides utility VPlan to VPlan transformations.
This file declares the class VPlanVerifier, which contains utility functions to check the consistency...
This file contains the declarations of the Vectorization Plan base classes:
static const char PassName[]
Value * RHS
Value * LHS
static const uint32_t IV[8]
Definition blake3_impl.h:83
A manager for alias analyses.
Class for arbitrary precision integers.
Definition APInt.h:78
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition APInt.h:234
uint64_t getZExtValue() const
Get zero extended value.
Definition APInt.h:1540
unsigned getActiveBits() const
Compute the number of active bits in the value.
Definition APInt.h:1512
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:41
size_t size() const
size - Get the array size.
Definition ArrayRef.h:147
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
LLVM_ABI unsigned getVScaleRangeMin() const
Returns the minimum value for the vscale_range attribute.
LLVM Basic Block Representation.
Definition BasicBlock.h:62
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
Definition BasicBlock.h:528
LLVM_ABI const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const Function * getParent() const
Return the enclosing method, or null if none.
Definition BasicBlock.h:213
LLVM_ABI InstListType::const_iterator getFirstNonPHIIt() const
Returns an iterator to the first instruction in this block that is not a PHINode instruction.
LLVM_ABI const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
LLVM_ABI const BasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this basic block belongs to.
LLVM_ABI LLVMContext & getContext() const
Get the context in which this basic block lives.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition BasicBlock.h:233
BinaryOps getOpcode() const
Definition InstrTypes.h:374
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Conditional or Unconditional Branch instruction.
bool isConditional() const
static BranchInst * Create(BasicBlock *IfTrue, InsertPosition InsertBefore=nullptr)
BasicBlock * getSuccessor(unsigned i) const
Represents analyses that only rely on functions' control flow.
Definition Analysis.h:73
bool isNoBuiltin() const
Return true if the call should not be treated as a call to a builtin.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Value * getArgOperand(unsigned i) const
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
unsigned arg_size() const
This class represents a function call, abstracting a target machine's calling convention.
static Type * makeCmpResultType(Type *opnd_type)
Create a result type for fcmp/icmp.
Definition InstrTypes.h:984
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:678
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:701
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:703
@ ICMP_NE
not equal
Definition InstrTypes.h:700
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:704
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
Definition InstrTypes.h:791
An abstraction over a floating-point predicate, and a pack of an integer predicate with samesign info...
This is the shared class of boolean and integer constants.
Definition Constants.h:87
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:63
A debug info location.
Definition DebugLoc.h:124
static DebugLoc getTemporary()
Definition DebugLoc.h:161
static DebugLoc getUnknown()
Definition DebugLoc.h:162
An analysis that produces DemandedBits for a function.
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
Definition DenseMap.h:194
iterator find(const_arg_type_t< KeyT > Val)
Definition DenseMap.h:167
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)
Definition DenseMap.h:237
iterator end()
Definition DenseMap.h:81
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
Definition DenseMap.h:158
void insert_range(Range &&R)
Inserts range of 'std::pair<KeyT, ValueT>' values into the map.
Definition DenseMap.h:275
Implements a dense probed hash-table based set.
Definition DenseSet.h:279
Analysis pass which computes a DominatorTree.
Definition Dominators.h:284
void changeImmediateDominator(DomTreeNodeBase< NodeT > *N, DomTreeNodeBase< NodeT > *NewIDom)
changeImmediateDominator - This method is used to update the dominator tree information when a node's...
void eraseNode(NodeT *BB)
eraseNode - Removes a node from the dominator tree.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition Dominators.h:165
constexpr bool isVector() const
One or more elements.
Definition TypeSize.h:325
static constexpr ElementCount getScalable(ScalarTy MinVal)
Definition TypeSize.h:313
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition TypeSize.h:310
static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)
Definition TypeSize.h:316
constexpr bool isScalar() const
Exactly one element.
Definition TypeSize.h:321
BasicBlock * createVectorizedLoopSkeleton() final
Implements the interface for creating a vectorized skeleton using the epilogue loop strategy (i....
EpilogueVectorizerEpilogueLoop(Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, const TargetTransformInfo *TTI, AssumptionCache *AC, EpilogueLoopVectorizationInfo &EPI, LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, GeneratedRTChecks &Checks, VPlan &Plan)
void printDebugTracesAtStart() override
Allow subclasses to override and print debug traces before/after vplan execution, when trace informat...
A specialized derived class of inner loop vectorizer that performs vectorization of main loops in the...
void introduceCheckBlockInVPlan(BasicBlock *CheckIRBB)
Introduces a new VPIRBasicBlock for CheckIRBB to Plan between the vector preheader and its predecesso...
BasicBlock * emitIterationCountCheck(BasicBlock *VectorPH, BasicBlock *Bypass, bool ForEpilogue)
Emits an iteration count bypass check once for the main loop (when ForEpilogue is false) and once for...
EpilogueVectorizerMainLoop(Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, const TargetTransformInfo *TTI, AssumptionCache *AC, EpilogueLoopVectorizationInfo &EPI, LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, GeneratedRTChecks &Check, VPlan &Plan)
Value * createIterationCountCheck(BasicBlock *VectorPH, ElementCount VF, unsigned UF) const
void printDebugTracesAtStart() override
Allow subclasses to override and print debug traces before/after vplan execution, when trace informat...
BasicBlock * createVectorizedLoopSkeleton() final
Implements the interface for creating a vectorized skeleton using the main loop strategy (i....
Convenience struct for specifying and reasoning about fast-math flags.
Definition FMF.h:22
Class to represent function types.
param_iterator param_begin() const
param_iterator param_end() const
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition Function.h:209
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition Function.cpp:762
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition Function.cpp:727
Represents flags for the getelementptr instruction/expression.
static GEPNoWrapFlags none()
void applyUpdates(ArrayRef< UpdateT > Updates)
Submit updates to all available trees.
Common base class shared among various IRBuilders.
Definition IRBuilder.h:114
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
Definition IRBuilder.h:345
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2780
A struct for saving information about induction variables.
const SCEV * getStep() const
InductionKind
This enum represents the kinds of inductions that we support.
@ IK_NoInduction
Not an induction variable.
@ IK_FpInduction
Floating point induction variable.
@ IK_PtrInduction
Pointer induction var. Step = C.
@ IK_IntInduction
Integer induction variable. Step = C.
const SmallVectorImpl< Instruction * > & getCastInsts() const
Returns a reference to the type cast instructions in the induction update chain, that are redundant w...
Value * getStartValue() const
InnerLoopAndEpilogueVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, const TargetTransformInfo *TTI, AssumptionCache *AC, EpilogueLoopVectorizationInfo &EPI, LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, GeneratedRTChecks &Checks, VPlan &Plan, ElementCount VecWidth, ElementCount MinProfitableTripCount, unsigned UnrollFactor)
EpilogueLoopVectorizationInfo & EPI
Holds and updates state information required to vectorize the main loop and its epilogue in two separ...
InnerLoopVectorizer vectorizes loops which contain only one basic block to a specified vectorization ...
virtual void printDebugTracesAtStart()
Allow subclasses to override and print debug traces before/after vplan execution, when trace informat...
Value * TripCount
Trip count of the original loop.
const TargetTransformInfo * TTI
Target Transform Info.
LoopVectorizationCostModel * Cost
The profitablity analysis.
BlockFrequencyInfo * BFI
BFI and PSI are used to check for profile guided size optimizations.
Value * getTripCount() const
Returns the original loop trip count.
friend class LoopVectorizationPlanner
PredicatedScalarEvolution & PSE
A wrapper around ScalarEvolution used to add runtime SCEV checks.
LoopInfo * LI
Loop Info.
ProfileSummaryInfo * PSI
DominatorTree * DT
Dominator Tree.
void setTripCount(Value *TC)
Used to set the trip count after ILV's construction and after the preheader block has been executed.
void fixVectorizedLoop(VPTransformState &State)
Fix the vectorized code, taking care of header phi's, and more.
virtual BasicBlock * createVectorizedLoopSkeleton()
Creates a basic block for the scalar preheader.
virtual void printDebugTracesAtEnd()
AssumptionCache * AC
Assumption Cache.
InnerLoopVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, const TargetTransformInfo *TTI, AssumptionCache *AC, ElementCount VecWidth, unsigned UnrollFactor, LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, GeneratedRTChecks &RTChecks, VPlan &Plan)
IRBuilder Builder
The builder that we use.
void fixNonInductionPHIs(VPTransformState &State)
Fix the non-induction PHIs in Plan.
VPBasicBlock * VectorPHVPBB
The vector preheader block of Plan, used as target for check blocks introduced during skeleton creati...
unsigned UF
The vectorization unroll factor to use.
GeneratedRTChecks & RTChecks
Structure to hold information about generated runtime checks, responsible for cleaning the checks,...
virtual ~InnerLoopVectorizer()=default
ElementCount VF
The vectorization SIMD factor to use.
Loop * OrigLoop
The original loop.
BasicBlock * createScalarPreheader(StringRef Prefix)
Create and return a new IR basic block for the scalar preheader whose name is prefixed with Prefix.
InstSimplifyFolder - Use InstructionSimplify to fold operations to existing values.
static InstructionCost getInvalid(CostType Val=0)
static InstructionCost getMax()
CostType getValue() const
This function is intended to be used as sparingly as possible, since the class provides the full rang...
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
LLVM_ABI void moveBefore(InstListType::iterator InsertPos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
bool isBinaryOp() const
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Instruction * user_back()
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
const char * getOpcodeName() const
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Class to represent integer types.
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition Type.cpp:319
LLVM_ABI APInt getMask() const
For example, this is 0xFF for an 8 bit integer, 0xFFFF for i16, etc.
Definition Type.cpp:343
The group of interleaved loads/stores sharing the same stride and close to each other.
uint32_t getFactor() const
InstTy * getMember(uint32_t Index) const
Get the member with the given index Index.
InstTy * getInsertPos() const
uint32_t getNumMembers() const
Drive the analysis of interleaved memory accesses in the loop.
bool requiresScalarEpilogue() const
Returns true if an interleaved group that may access memory out-of-bounds requires a scalar epilogue ...
LLVM_ABI void analyzeInterleaving(bool EnableMaskedInterleavedGroup)
Analyze the interleaved accesses and collect them in interleave groups.
An instruction for reading from memory.
Type * getPointerOperandType() const
This analysis provides dependence information for the memory accesses of a loop.
Drive the analysis of memory accesses in the loop.
const RuntimePointerChecking * getRuntimePointerChecking() const
unsigned getNumRuntimePointerChecks() const
Number of memchecks required to prove independence of otherwise may-alias pointers.
Analysis pass that exposes the LoopInfo for a function.
Definition LoopInfo.h:569
bool contains(const LoopT *L) const
Return true if the specified loop is contained within in this loop.
BlockT * getLoopLatch() const
If there is a single latch block for this loop, return it.
bool isInnermost() const
Return true if the loop does not contain any (natural) loops.
void getExitingBlocks(SmallVectorImpl< BlockT * > &ExitingBlocks) const
Return all blocks inside the loop that have successors outside of the loop.
BlockT * getHeader() const
iterator_range< block_iterator > blocks() const
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
ArrayRef< BlockT * > getBlocks() const
Get a list of the basic blocks which make up this loop.
Store the result of a depth first search within basic blocks contained by a single loop.
RPOIterator beginRPO() const
Reverse iterate over the cached postorder blocks.
void perform(const LoopInfo *LI)
Traverse the loop blocks and store the DFS result.
RPOIterator endRPO() const
Wrapper class to LoopBlocksDFS that provides a standard begin()/end() interface for the DFS reverse p...
void perform(const LoopInfo *LI)
Traverse the loop blocks and store the DFS result.
void removeBlock(BlockT *BB)
This method completely removes BB from all data structures, including all of the Loop objects it is n...
LoopVectorizationCostModel - estimates the expected speedups due to vectorization.
SmallPtrSet< Type *, 16 > ElementTypesInLoop
All element types found in the loop.
bool isLegalMaskedLoad(Type *DataType, Value *Ptr, Align Alignment, unsigned AddressSpace) const
Returns true if the target machine supports masked load operation for the given DataType and kind of ...
LoopVectorizationCostModel(ScalarEpilogueLowering SEL, Loop *L, PredicatedScalarEvolution &PSE, LoopInfo *LI, LoopVectorizationLegality *Legal, const TargetTransformInfo &TTI, const TargetLibraryInfo *TLI, DemandedBits *DB, AssumptionCache *AC, OptimizationRemarkEmitter *ORE, const Function *F, const LoopVectorizeHints *Hints, InterleavedAccessInfo &IAI, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI)
void collectElementTypesForWidening()
Collect all element types in the loop for which widening is needed.
bool canVectorizeReductions(ElementCount VF) const
Returns true if the target machine supports all of the reduction variables found for the given VF.
bool isLegalMaskedStore(Type *DataType, Value *Ptr, Align Alignment, unsigned AddressSpace) const
Returns true if the target machine supports masked store operation for the given DataType and kind of...
bool isEpilogueVectorizationProfitable(const ElementCount VF, const unsigned IC) const
Returns true if epilogue vectorization is considered profitable, and false otherwise.
bool isPredicatedInst(Instruction *I) const
Returns true if I is an instruction that needs to be predicated at runtime.
void collectValuesToIgnore()
Collect values we want to ignore in the cost model.
void collectInLoopReductions()
Split reductions into those that happen in the loop, and those that happen outside.
std::pair< unsigned, unsigned > getSmallestAndWidestTypes()
bool isUniformAfterVectorization(Instruction *I, ElementCount VF) const
Returns true if I is known to be uniform after vectorization.
void collectNonVectorizedAndSetWideningDecisions(ElementCount VF)
Collect values that will not be widened, including Uniforms, Scalars, and Instructions to Scalarize f...
PredicatedScalarEvolution & PSE
Predicated scalar evolution analysis.
const LoopVectorizeHints * Hints
Loop Vectorize Hint.
std::optional< unsigned > getMaxSafeElements() const
Return maximum safe number of elements to be processed per vector iteration, which do not prevent sto...
const TargetTransformInfo & TTI
Vector target information.
LoopVectorizationLegality * Legal
Vectorization legality.
std::optional< InstructionCost > getReductionPatternCost(Instruction *I, ElementCount VF, Type *VectorTy) const
Return the cost of instructions in an inloop reduction pattern, if I is part of that pattern.
InstructionCost getInstructionCost(Instruction *I, ElementCount VF)
Returns the execution time cost of an instruction for a given vector width.
DemandedBits * DB
Demanded bits analysis.
bool interleavedAccessCanBeWidened(Instruction *I, ElementCount VF) const
Returns true if I is a memory instruction in an interleaved-group of memory accesses that can be vect...
const TargetLibraryInfo * TLI
Target Library Info.
bool memoryInstructionCanBeWidened(Instruction *I, ElementCount VF)
Returns true if I is a memory instruction with consecutive memory access that can be widened.
const InterleaveGroup< Instruction > * getInterleavedAccessGroup(Instruction *Instr) const
Get the interleaved access group that Instr belongs to.
InstructionCost getVectorIntrinsicCost(CallInst *CI, ElementCount VF) const
Estimate cost of an intrinsic call instruction CI if it were vectorized with factor VF.
bool OptForSize
Whether this loop should be optimized for size based on function attribute or profile information.
bool useMaxBandwidth(TargetTransformInfo::RegisterKind RegKind)
bool isScalarAfterVectorization(Instruction *I, ElementCount VF) const
Returns true if I is known to be scalar after vectorization.
bool isOptimizableIVTruncate(Instruction *I, ElementCount VF)
Return True if instruction I is an optimizable truncate whose operand is an induction variable.
FixedScalableVFPair computeMaxVF(ElementCount UserVF, unsigned UserIC)
bool shouldConsiderRegPressureForVF(ElementCount VF)
Loop * TheLoop
The loop that we evaluate.
TTI::TargetCostKind CostKind
The kind of cost that we are calculating.
TailFoldingStyle getTailFoldingStyle(bool IVUpdateMayOverflow=true) const
Returns the TailFoldingStyle that is best for the current loop.
InterleavedAccessInfo & InterleaveInfo
The interleave access information contains groups of interleaved accesses with the same stride and cl...
SmallPtrSet< const Value *, 16 > ValuesToIgnore
Values to ignore in the cost model.
void setVectorizedCallDecision(ElementCount VF)
A call may be vectorized in different ways depending on whether we have vectorized variants available...
void invalidateCostModelingDecisions()
Invalidates decisions already taken by the cost model.
bool isAccessInterleaved(Instruction *Instr) const
Check if Instr belongs to any interleaved access group.
bool selectUserVectorizationFactor(ElementCount UserVF)
Setup cost-based decisions for user vectorization factor.
std::optional< unsigned > getVScaleForTuning() const
Return the value of vscale used for tuning the cost model.
OptimizationRemarkEmitter * ORE
Interface to emit optimization remarks.
LoopInfo * LI
Loop Info analysis.
bool requiresScalarEpilogue(bool IsVectorizing) const
Returns true if we're required to use a scalar epilogue for at least the final iteration of the origi...
SmallPtrSet< const Value *, 16 > VecValuesToIgnore
Values to ignore in the cost model when VF > 1.
bool isInLoopReduction(PHINode *Phi) const
Returns true if the Phi is part of an inloop reduction.
bool isProfitableToScalarize(Instruction *I, ElementCount VF) const
void setWideningDecision(const InterleaveGroup< Instruction > *Grp, ElementCount VF, InstWidening W, InstructionCost Cost)
Save vectorization decision W and Cost taken by the cost model for interleaving group Grp and vector ...
const MapVector< Instruction *, uint64_t > & getMinimalBitwidths() const
CallWideningDecision getCallWideningDecision(CallInst *CI, ElementCount VF) const
bool isLegalGatherOrScatter(Value *V, ElementCount VF)
Returns true if the target machine can represent V as a masked gather or scatter operation.
bool canTruncateToMinimalBitwidth(Instruction *I, ElementCount VF) const
bool shouldConsiderInvariant(Value *Op)
Returns true if Op should be considered invariant and if it is trivially hoistable.
bool foldTailByMasking() const
Returns true if all loop blocks should be masked to fold tail loop.
bool foldTailWithEVL() const
Returns true if VP intrinsics with explicit vector length support should be generated in the tail fol...
bool usePredicatedReductionSelect() const
Returns true if the predicated reduction select should be used to set the incoming value for the redu...
bool blockNeedsPredicationForAnyReason(BasicBlock *BB) const
Returns true if the instructions in this block requires predication for any reason,...
void setCallWideningDecision(CallInst *CI, ElementCount VF, InstWidening Kind, Function *Variant, Intrinsic::ID IID, std::optional< unsigned > MaskPos, InstructionCost Cost)
void setTailFoldingStyles(bool IsScalableVF, unsigned UserIC)
Selects and saves TailFoldingStyle for 2 options - if IV update may overflow or not.
AssumptionCache * AC
Assumption cache.
void setWideningDecision(Instruction *I, ElementCount VF, InstWidening W, InstructionCost Cost)
Save vectorization decision W and Cost taken by the cost model for instruction I and vector width VF.
InstWidening
Decision that was taken during cost calculation for memory instruction.
bool isScalarWithPredication(Instruction *I, ElementCount VF) const
Returns true if I is an instruction which requires predication and for which our chosen predication s...
InstructionCost getVectorCallCost(CallInst *CI, ElementCount VF) const
Estimate cost of a call instruction CI if it were vectorized with factor VF.
bool useOrderedReductions(const RecurrenceDescriptor &RdxDesc) const
Returns true if we should use strict in-order reductions for the given RdxDesc.
std::pair< InstructionCost, InstructionCost > getDivRemSpeculationCost(Instruction *I, ElementCount VF) const
Return the costs for our two available strategies for lowering a div/rem operation which requires spe...
bool isDivRemScalarWithPredication(InstructionCost ScalarCost, InstructionCost SafeDivisorCost) const
Given costs for both strategies, return true if the scalar predication lowering should be used for di...
InstructionCost expectedCost(ElementCount VF)
Returns the expected execution cost.
void setCostBasedWideningDecision(ElementCount VF)
Memory access instruction may be vectorized in more than one way.
InstWidening getWideningDecision(Instruction *I, ElementCount VF) const
Return the cost model decision for the given instruction I and vector width VF.
FixedScalableVFPair MaxPermissibleVFWithoutMaxBW
The highest VF possible for this loop, without using MaxBandwidth.
bool isScalarEpilogueAllowed() const
Returns true if a scalar epilogue is not allowed due to optsize or a loop hint annotation.
InstructionCost getWideningCost(Instruction *I, ElementCount VF)
Return the vectorization cost for the given instruction I and vector width VF.
void collectInstsToScalarize(ElementCount VF)
Collects the instructions to scalarize for each predicated instruction in the loop.
LoopVectorizationLegality checks if it is legal to vectorize a loop, and to what vectorization factor...
MapVector< PHINode *, InductionDescriptor > InductionList
InductionList saves induction variables and maps them to the induction descriptor.
const SmallPtrSetImpl< const Instruction * > & getPotentiallyFaultingLoads() const
Returns potentially faulting loads.
bool canVectorize(bool UseVPlanNativePath)
Returns true if it is legal to vectorize this loop.
bool canVectorizeFPMath(bool EnableStrictReductions)
Returns true if it is legal to vectorize the FP math operations in this loop.
PHINode * getPrimaryInduction()
Returns the primary induction variable.
const SmallVector< BasicBlock *, 4 > & getCountableExitingBlocks() const
Returns all exiting blocks with a countable exit, i.e.
const InductionList & getInductionVars() const
Returns the induction variables found in the loop.
bool hasUncountableEarlyExit() const
Returns true if the loop has exactly one uncountable early exit, i.e.
bool hasHistograms() const
Returns a list of all known histogram operations in the loop.
const LoopAccessInfo * getLAI() const
Planner drives the vectorization process after having passed Legality checks.
VectorizationFactor selectEpilogueVectorizationFactor(const ElementCount MaxVF, unsigned IC)
VPlan & getPlanFor(ElementCount VF) const
Return the VPlan for VF.
Definition VPlan.cpp:1614
VectorizationFactor planInVPlanNativePath(ElementCount UserVF)
Use the VPlan-native path to plan how to best vectorize, return the best VF and its cost.
void updateLoopMetadataAndProfileInfo(Loop *VectorLoop, VPBasicBlock *HeaderVPBB, const VPlan &Plan, bool VectorizingEpilogue, MDNode *OrigLoopID, std::optional< unsigned > OrigAverageTripCount, unsigned OrigLoopInvocationWeight, unsigned EstimatedVFxUF, bool DisableRuntimeUnroll)
Update loop metadata and profile info for both the scalar remainder loop and VectorLoop,...
Definition VPlan.cpp:1665
void buildVPlans(ElementCount MinVF, ElementCount MaxVF)
Build VPlans for power-of-2 VF's between MinVF and MaxVF inclusive, according to the information gath...
Definition VPlan.cpp:1598
VectorizationFactor computeBestVF()
Compute and return the most profitable vectorization factor.
DenseMap< const SCEV *, Value * > executePlan(ElementCount VF, unsigned UF, VPlan &BestPlan, InnerLoopVectorizer &LB, DominatorTree *DT, bool VectorizingEpilogue)
Generate the IR code for the vectorized loop captured in VPlan BestPlan according to the best selecte...
unsigned selectInterleaveCount(VPlan &Plan, ElementCount VF, InstructionCost LoopCost)
void emitInvalidCostRemarks(OptimizationRemarkEmitter *ORE)
Emit remarks for recipes with invalid costs in the available VPlans.
static bool getDecisionAndClampRange(const std::function< bool(ElementCount)> &Predicate, VFRange &Range)
Test a Predicate on a Range of VF's.
Definition VPlan.cpp:1579
void printPlans(raw_ostream &O)
Definition VPlan.cpp:1743
void plan(ElementCount UserVF, unsigned UserIC)
Build VPlans for the specified UserVF and UserIC if they are non-zero or all applicable candidate VFs...
void addMinimumIterationCheck(VPlan &Plan, ElementCount VF, unsigned UF, ElementCount MinProfitableTripCount) const
Create a check to Plan to see if the vector loop should be executed based on its trip count.
bool hasPlanWithVF(ElementCount VF) const
Look through the existing plans and return true if we have one with vectorization factor VF.
This holds vectorization requirements that must be verified late in the process.
Utility class for getting and setting loop vectorizer hints in the form of loop metadata.
bool allowVectorization(Function *F, Loop *L, bool VectorizeOnlyWhenForced) const
void emitRemarkWithHints() const
Dumps all the hint information.
const char * vectorizeAnalysisPassName() const
If hints are provided that force vectorization, use the AlwaysPrint pass name to force the frontend t...
This class emits a version of the loop where run-time checks ensure that may-alias pointers can't ove...
Represents a single loop in the control flow graph.
Definition LoopInfo.h:40
bool hasLoopInvariantOperands(const Instruction *I) const
Return true if all the operands of the specified instruction are loop invariant.
Definition LoopInfo.cpp:67
DebugLoc getStartLoc() const
Return the debug location of the start of this loop.
Definition LoopInfo.cpp:632
bool isLoopInvariant(const Value *V) const
Return true if the specified value is loop invariant.
Definition LoopInfo.cpp:61
Metadata node.
Definition Metadata.h:1078
This class implements a map that also provides access to all stored values in a deterministic order.
Definition MapVector.h:36
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition MapVector.h:119
Function * getFunction(StringRef Name) const
Look up the specified function in the module symbol table.
Definition Module.cpp:230
Diagnostic information for optimization analysis remarks related to pointer aliasing.
Diagnostic information for optimization analysis remarks related to floating-point non-commutativity.
Diagnostic information for optimization analysis remarks.
The optimization diagnostic interface.
LLVM_ABI void emit(DiagnosticInfoOptimizationBase &OptDiag)
Output the remark via the diagnostic handler and to the optimization record file.
Diagnostic information for missed-optimization remarks.
Diagnostic information for applied optimization remarks.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
op_range incoming_values()
void setIncomingValueForBlock(const BasicBlock *BB, Value *V)
Set every incoming value(s) for block BB to V.
Value * getIncomingValueForBlock(const BasicBlock *BB) const
unsigned getNumIncomingValues() const
Return the number of incoming edges.
An interface layer with SCEV used to manage how we see SCEV expressions for values in the context of ...
ScalarEvolution * getSE() const
Returns the ScalarEvolution analysis used.
LLVM_ABI const SCEVPredicate & getPredicate() const
LLVM_ABI unsigned getSmallConstantMaxTripCount()
Returns the upper bound of the loop trip count as a normal unsigned value, or 0 if the trip count is ...
LLVM_ABI const SCEV * getBackedgeTakenCount()
Get the (predicated) backedge count for the analyzed loop.
LLVM_ABI const SCEV * getSCEV(Value *V)
Returns the SCEV expression of V, in the context of the current SCEV predicate.
A set of analyses that are preserved following a run of a transformation pass.
Definition Analysis.h:112
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition Analysis.h:118
PreservedAnalyses & preserveSet()
Mark an analysis set as preserved.
Definition Analysis.h:151
PreservedAnalyses & preserve()
Mark an analysis as preserved.
Definition Analysis.h:132
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
Analysis providing profile information.
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
static bool isFMulAddIntrinsic(Instruction *I)
Returns true if the instruction is a call to the llvm.fmuladd intrinsic.
FastMathFlags getFastMathFlags() const
Instruction * getLoopExitInstr() const
static LLVM_ABI unsigned getOpcode(RecurKind Kind)
Returns the opcode corresponding to the RecurrenceKind.
Type * getRecurrenceType() const
Returns the type of the recurrence.
const SmallPtrSet< Instruction *, 8 > & getCastInsts() const
Returns a reference to the instructions used for type-promoting the recurrence.
unsigned getMinWidthCastToRecurrenceTypeInBits() const
Returns the minimum width used by the recurrence in bits.
TrackingVH< Value > getRecurrenceStartValue() const
LLVM_ABI SmallVector< Instruction *, 4 > getReductionOpChain(PHINode *Phi, Loop *L) const
Attempts to find a chain of operations from Phi to LoopExitInst that can be treated as a set of reduc...
static bool isAnyOfRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
bool isSigned() const
Returns true if all source operands of the recurrence are SExtInsts.
RecurKind getRecurrenceKind() const
bool isOrdered() const
Expose an ordered FP reduction to the instance users.
static LLVM_ABI bool isFloatingPointRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is a floating point kind.
static bool isFindIVRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
Value * getSentinelValue() const
Returns the sentinel value for FindFirstIV & FindLastIV recurrences to replace the start value.
static bool isMinMaxRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is any min/max kind.
std::optional< ArrayRef< PointerDiffInfo > > getDiffChecks() const
const SmallVectorImpl< RuntimePointerCheck > & getChecks() const
Returns the checks that generateChecks created.
This class uses information about analyze scalars to rewrite expressions in canonical form.
ScalarEvolution * getSE()
bool isInsertedInstruction(Instruction *I) const
Return true if the specified instruction was inserted by the code rewriter.
LLVM_ABI Value * expandCodeForPredicate(const SCEVPredicate *Pred, Instruction *Loc)
Generates a code sequence that evaluates this predicate.
void eraseDeadInstructions(Value *Root)
Remove inserted instructions that are dead, e.g.
virtual bool isAlwaysTrue() const =0
Returns true if the predicate is always true.
This class represents an analyzed expression in the program.
LLVM_ABI bool isZero() const
Return true if the expression is a constant zero.
LLVM_ABI Type * getType() const
Return the LLVM type of this SCEV expression.
Analysis pass that exposes the ScalarEvolution for a function.
The main scalar evolution driver.
LLVM_ABI const SCEV * getURemExpr(const SCEV *LHS, const SCEV *RHS)
Represents an unsigned remainder expression based on unsigned division.
LLVM_ABI const SCEV * getBackedgeTakenCount(const Loop *L, ExitCountKind Kind=Exact)
If the specified loop has a predictable backedge-taken count, return it, otherwise return a SCEVCould...
LLVM_ABI const SCEV * getConstant(ConstantInt *V)
LLVM_ABI const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
LLVM_ABI const SCEV * getTripCountFromExitCount(const SCEV *ExitCount)
A version of getTripCountFromExitCount below which always picks an evaluation type which can not resu...
const SCEV * getOne(Type *Ty)
Return a SCEV for the constant 1 of a specific type.
LLVM_ABI void forgetLoop(const Loop *L)
This method should be called by the client when it has changed a loop in a way that may effect Scalar...
LLVM_ABI bool isLoopInvariant(const SCEV *S, const Loop *L)
Return true if the value of the given SCEV is unchanging in the specified loop.
LLVM_ABI bool isSCEVable(Type *Ty) const
Test if values of the given type are analyzable within the SCEV framework.
LLVM_ABI const SCEV * getElementCount(Type *Ty, ElementCount EC, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
LLVM_ABI void forgetValue(Value *V)
This method should be called by the client when it has changed a value in a way that may effect its v...
LLVM_ABI void forgetBlockAndLoopDispositions(Value *V=nullptr)
Called when the client has changed the disposition of values in a loop or block.
const SCEV * getMinusOne(Type *Ty)
Return a SCEV for the constant -1 of a specific type.
LLVM_ABI void forgetLcssaPhiWithNewPredecessor(Loop *L, PHINode *V)
Forget LCSSA phi node V of loop L to which a new predecessor was added, such that it may no longer be...
LLVM_ABI unsigned getSmallConstantTripCount(const Loop *L)
Returns the exact trip count of the loop if we can compute it, and the result is a small constant.
APInt getUnsignedRangeMax(const SCEV *S)
Determine the max of the unsigned range for a particular SCEV.
LLVM_ABI const SCEV * applyLoopGuards(const SCEV *Expr, const Loop *L)
Try to apply information from loop guards for L to Expr.
LLVM_ABI const SCEV * getAddExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap, unsigned Depth=0)
Get a canonical add expression, or something simpler if possible.
LLVM_ABI bool isKnownPredicate(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
Test if the given expression is known to satisfy the condition described by Pred, LHS,...
This class represents the LLVM 'select' instruction.
A vector that has set insertion semantics.
Definition SetVector.h:59
size_type size() const
Determine the number of elements in the SetVector.
Definition SetVector.h:102
void insert_range(Range &&R)
Definition SetVector.h:175
size_type count(const key_type &key) const
Count the number of elements of a given key in the SetVector.
Definition SetVector.h:261
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition SetVector.h:150
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
Definition SetVector.h:338
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
Definition StringRef.h:55
Analysis pass providing the TargetTransformInfo.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
LLVM_ABI std::optional< unsigned > getVScaleForTuning() const
LLVM_ABI InstructionCost getScalarizationOverhead(VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract, TTI::TargetCostKind CostKind, bool ForPoisonSrc=true, ArrayRef< Value * > VL={}) const
Estimate the overhead of scalarizing an instruction.
LLVM_ABI bool supportsEfficientVectorElementLoadStore() const
If target has efficient vector element load/store instructions, it can return true here so that inser...
LLVM_ABI bool prefersVectorizedAddressing() const
Return true if target doesn't mind addresses in vectors.
LLVM_ABI TypeSize getRegisterBitWidth(RegisterKind K) const
LLVM_ABI bool preferFixedOverScalableIfEqualCost(bool IsEpilogue) const
LLVM_ABI InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput, OperandValueInfo OpdInfo={OK_AnyValue, OP_None}, const Instruction *I=nullptr) const
LLVM_ABI InstructionCost getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef< unsigned > Indices, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput, bool UseMaskForCond=false, bool UseMaskForGaps=false) const
LLVM_ABI InstructionCost getShuffleCost(ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, ArrayRef< int > Mask={}, TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput, int Index=0, VectorType *SubTp=nullptr, ArrayRef< const Value * > Args={}, const Instruction *CxtI=nullptr) const
static LLVM_ABI PartialReductionExtendKind getPartialReductionExtendKind(Instruction *I)
Get the kind of extension that an instruction represents.
static LLVM_ABI OperandValueInfo getOperandInfo(const Value *V)
Collect properties of V used in cost analysis, e.g. OP_PowerOf2.
LLVM_ABI bool isElementTypeLegalForScalableVector(Type *Ty) const
LLVM_ABI ElementCount getMinimumVF(unsigned ElemWidth, bool IsScalable) const
TargetCostKind
The kind of cost model.
@ TCK_RecipThroughput
Reciprocal throughput.
@ TCK_CodeSize
Instruction code size.
@ TCK_SizeAndLatency
The weighted sum of size and latency.
@ TCK_Latency
The latency of instruction.
LLVM_ABI InstructionCost getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput) const
LLVM_ABI InstructionCost getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE, const SCEV *Ptr, TTI::TargetCostKind CostKind) const
LLVM_ABI InstructionCost getPartialReductionCost(unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType, ElementCount VF, PartialReductionExtendKind OpAExtend, PartialReductionExtendKind OpBExtend, std::optional< unsigned > BinOp, TTI::TargetCostKind CostKind) const
LLVM_ABI InstructionCost getGatherScatterOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask, Align Alignment, TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput, const Instruction *I=nullptr) const
LLVM_ABI bool supportsScalableVectors() const
@ TCC_Free
Expected to fold away in lowering.
LLVM_ABI InstructionCost getInstructionCost(const User *U, ArrayRef< const Value * > Operands, TargetCostKind CostKind) const
Estimate the cost of a given IR user when lowered.
LLVM_ABI InstructionCost getIndexedVectorInstrCostFromEnd(unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index) const
LLVM_ABI InstructionCost getOperandsScalarizationOverhead(ArrayRef< Type * > Tys, TTI::TargetCostKind CostKind) const
Estimate the overhead of scalarizing operands with the given types.
@ SK_Splice
Concatenates elements from the first input vector with elements of the second input vector.
@ SK_Broadcast
Broadcast element 0 to all other elements.
@ SK_Reverse
Reverse the order of the vector.
LLVM_ABI InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind=TTI::TCK_SizeAndLatency, const Instruction *I=nullptr) const
CastContextHint
Represents a hint about the context in which a cast is used.
@ Reversed
The cast is used with a reversed load/store.
@ Masked
The cast is used with a masked load/store.
@ None
The cast is not used with a load/store of any kind.
@ Normal
The cast is used with a normal load/store.
@ Interleave
The cast is used with an interleaved load/store.
@ GatherScatter
The cast is used with a gather/scatter.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
This class implements a switch-like dispatch statement for a value of 'T' using dyn_cast functionalit...
Definition TypeSwitch.h:87
TypeSwitch< T, ResultT > & Case(CallableT &&caseFn)
Add a case on the given type.
Definition TypeSwitch.h:96
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:45
LLVM_ABI unsigned getIntegerBitWidth() const
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:273
static LLVM_ABI Type * getVoidTy(LLVMContext &C)
Definition Type.cpp:281
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:352
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:198
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition Type.h:128
LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.cpp:231
static LLVM_ABI IntegerType * getInt1Ty(LLVMContext &C)
Definition Type.cpp:294
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
Definition Type.h:184
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:240
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:139
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
op_range operands()
Definition User.h:292
LLVM_ABI bool replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
Definition User.cpp:21
Value * getOperand(unsigned i) const
Definition User.h:232
static SmallVector< VFInfo, 8 > getMappings(const CallInst &CI)
Retrieve all the VFInfo instances associated to the CallInst CI.
Definition VectorUtils.h:74
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
Definition VPlan.h:3786
void appendRecipe(VPRecipeBase *Recipe)
Augment the existing recipes of a VPBasicBlock with an additional Recipe as the last recipe.
Definition VPlan.h:3861
RecipeListTy::iterator iterator
Instruction iterators...
Definition VPlan.h:3813
iterator end()
Definition VPlan.h:3823
iterator begin()
Recipe iterator methods.
Definition VPlan.h:3821
iterator_range< iterator > phis()
Returns an iterator range over the PHI-like recipes in the block.
Definition VPlan.h:3874
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
Definition VPlan.cpp:246
VPRegionBlock * getEnclosingLoopRegion()
Definition VPlan.cpp:619
void insert(VPRecipeBase *Recipe, iterator InsertPt)
Definition VPlan.h:3852
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
Definition VPlan.h:82
VPRegionBlock * getParent()
Definition VPlan.h:174
const VPBasicBlock * getExitingBasicBlock() const
Definition VPlan.cpp:190
void setName(const Twine &newName)
Definition VPlan.h:167
size_t getNumSuccessors() const
Definition VPlan.h:220
void swapSuccessors()
Swap successors of the block. The block must have exactly 2 successors.
Definition VPlan.h:323
size_t getNumPredecessors() const
Definition VPlan.h:221
VPlan * getPlan()
Definition VPlan.cpp:165
VPBlockBase * getSinglePredecessor() const
Definition VPlan.h:216
const VPBasicBlock * getEntryBasicBlock() const
Definition VPlan.cpp:170
VPBlockBase * getSingleSuccessor() const
Definition VPlan.h:210
const VPBlocksTy & getSuccessors() const
Definition VPlan.h:199
static auto blocksOnly(const T &Range)
Return an iterator range over Range which only includes BlockTy blocks.
Definition VPlanUtils.h:232
static void insertOnEdge(VPBlockBase *From, VPBlockBase *To, VPBlockBase *BlockPtr)
Inserts BlockPtr on the edge between From and To.
Definition VPlanUtils.h:253
static void connectBlocks(VPBlockBase *From, VPBlockBase *To, unsigned PredIdx=-1u, unsigned SuccIdx=-1u)
Connect VPBlockBases From and To bi-directionally.
Definition VPlanUtils.h:191
static void reassociateBlocks(VPBlockBase *Old, VPBlockBase *New)
Reassociate all the blocks connected to Old so that they now point to New.
Definition VPlanUtils.h:218
VPlan-based builder utility analogous to IRBuilder.
VPDerivedIVRecipe * createDerivedIV(InductionDescriptor::InductionKind Kind, FPMathOperator *FPBinOp, VPValue *Start, VPValue *Current, VPValue *Step, const Twine &Name="")
Convert the input value Current to the corresponding value of an induction with Start and Step values...
VPPhi * createScalarPhi(ArrayRef< VPValue * > IncomingValues, DebugLoc DL, const Twine &Name="")
VPInstruction * createNaryOp(unsigned Opcode, ArrayRef< VPValue * > Operands, Instruction *Inst=nullptr, const Twine &Name="")
Create an N-ary operation with Opcode, Operands and set Inst as its underlying Instruction.
VPInstruction * createScalarCast(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy, DebugLoc DL)
Canonical scalar induction phi of the vector loop.
Definition VPlan.h:3442
unsigned getNumDefinedValues() const
Returns the number of values defined by the VPDef.
Definition VPlanValue.h:424
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
Definition VPlanValue.h:397
void execute(VPTransformState &State) override
Generate the transformed value of the induction at offset StartValue (1.
VPValue * getStepValue() const
Definition VPlan.h:3663
VPValue * getStartValue() const
Definition VPlan.h:3662
A pure virtual base class for all recipes modeling header phis, including phis for first order recurr...
Definition VPlan.h:1978
virtual VPValue * getBackedgeValue()
Returns the incoming value from the loop backedge.
Definition VPlan.h:2026
VPValue * getStartValue()
Returns the start value of the phi, if one is set.
Definition VPlan.h:2015
A special type of VPBasicBlock that wraps an existing IR basic block.
Definition VPlan.h:3939
Helper to manage IR metadata for recipes.
Definition VPlan.h:943
This is a concrete Recipe that models a single VPlan-level instruction.
Definition VPlan.h:984
@ ComputeAnyOfResult
Compute the final result of a AnyOf reduction with select(cmp(),x,y), where one of (x,...
Definition VPlan.h:1017
@ ResumeForEpilogue
Explicit user for the resume phi of the canonical induction in the main VPlan, used by the epilogue v...
Definition VPlan.h:1064
@ FirstOrderRecurrenceSplice
Definition VPlan.h:990
@ ReductionStartVector
Start vector for reductions with 3 operands: the original start value, the identity value for the red...
Definition VPlan.h:1055
unsigned getOpcode() const
Definition VPlan.h:1120
VPInterleaveRecipe is a recipe for transforming an interleave group of load or stores into one wide l...
Definition VPlan.h:2577
In what follows, the term "input IR" refers to code that is fed into the vectorizer whereas the term ...
A recipe for forming partial reductions.
Definition VPlan.h:2754
detail::zippy< llvm::detail::zip_first, VPUser::const_operand_range, const_incoming_blocks_range > incoming_values_and_blocks() const
Returns an iterator range over pairs of incoming values and corresponding incoming blocks.
Definition VPlan.h:1291
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
Definition VPlan.h:395
VPBasicBlock * getParent()
Definition VPlan.h:416
DebugLoc getDebugLoc() const
Returns the debug location of the recipe.
Definition VPlan.h:483
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Helper class to create VPRecipies from IR instructions.
VPRecipeBase * tryToCreateWidenRecipe(VPSingleDefRecipe *R, VFRange &Range)
Create and return a widened recipe for R if one can be created within the given VF Range.
VPValue * getBlockInMask(VPBasicBlock *VPBB) const
Returns the entry mask for block VPBB or null if the mask is all-true.
std::optional< unsigned > getScalingForReduction(const Instruction *ExitInst)
void collectScaledReductions(VFRange &Range)
Find all possible partial reductions in the loop and track all of those that are valid so recipes can...
VPReplicateRecipe * handleReplication(Instruction *I, ArrayRef< VPValue * > Operands, VFRange &Range)
Build a VPReplicationRecipe for I using Operands.
VPRecipeBase * tryToCreatePartialReduction(Instruction *Reduction, ArrayRef< VPValue * > Operands, unsigned ScaleFactor)
Create and return a partial reduction recipe for a reduction instruction along with binary operation ...
A recipe for handling reduction phis.
Definition VPlan.h:2332
bool isInLoop() const
Returns true, if the phi is part of an in-loop reduction.
Definition VPlan.h:2392
RecurKind getRecurrenceKind() const
Returns the recurrence kind of the reduction.
Definition VPlan.h:2386
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
Definition VPlan.h:3974
const VPBlockBase * getEntry() const
Definition VPlan.h:4010
VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...
Definition VPlan.h:2857
VPSingleDef is a base class for recipes for modeling a sequence of one or more output IR that define ...
Definition VPlan.h:522
Instruction * getUnderlyingInstr()
Returns the underlying instruction.
Definition VPlan.h:587
An analysis for type-inference for VPValues.
Type * inferScalarType(const VPValue *V)
Infer the type of V. Returns the scalar type of V.
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
Definition VPlanValue.h:199
void setOperand(unsigned I, VPValue *New)
Definition VPlanValue.h:243
VPValue * getOperand(unsigned N) const
Definition VPlanValue.h:238
void addOperand(VPValue *Operand)
Definition VPlanValue.h:232
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
Definition VPlan.cpp:135
Value * getLiveInIRValue() const
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
Definition VPlanValue.h:176
Value * getUnderlyingValue() const
Return the underlying Value attached to this VPValue.
Definition VPlanValue.h:85
void replaceAllUsesWith(VPValue *New)
Definition VPlan.cpp:1415
user_iterator user_begin()
Definition VPlanValue.h:130
unsigned getNumUsers() const
Definition VPlanValue.h:113
void replaceUsesWithIf(VPValue *New, llvm::function_ref< bool(VPUser &U, unsigned Idx)> ShouldReplace)
Go through the uses list for this VPValue and make each use point to New if the callback ShouldReplac...
Definition VPlan.cpp:1419
user_range users()
Definition VPlanValue.h:134
A recipe to compute a pointer to the last element of each part of a widened memory access for widened...
Definition VPlan.h:1842
VPWidenCastRecipe is a recipe to create vector cast instructions.
Definition VPlan.h:1483
A recipe for handling GEP instructions.
Definition VPlan.h:1770
Base class for widened induction (VPWidenIntOrFpInductionRecipe and VPWidenPointerInductionRecipe),...
Definition VPlan.h:2043
VPValue * getStepValue()
Returns the step value of the induction.
Definition VPlan.h:2071
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
Definition VPlan.h:2088
A recipe for handling phi nodes of integer and floating-point inductions, producing their vector valu...
Definition VPlan.h:2118
A common base class for widening memory operations.
Definition VPlan.h:3155
A recipe for widened phis.
Definition VPlan.h:2254
VPWidenRecipe is a recipe for producing a widened instruction using the opcode and operands of the re...
Definition VPlan.h:1440
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
Definition VPlan.h:4077
bool hasVF(ElementCount VF) const
Definition VPlan.h:4286
VPBasicBlock * getEntry()
Definition VPlan.h:4176
VPValue & getVectorTripCount()
The vector trip count.
Definition VPlan.h:4266
VPValue & getVFxUF()
Returns VF * UF of the vector loop region.
Definition VPlan.h:4272
VPValue & getVF()
Returns the VF of the vector loop region.
Definition VPlan.h:4269
VPValue * getTripCount() const
The trip count of the original loop.
Definition VPlan.h:4238
iterator_range< SmallSetVector< ElementCount, 2 >::iterator > vectorFactors() const
Returns an iterator range over all VFs of the plan.
Definition VPlan.h:4293
bool hasUF(unsigned UF) const
Definition VPlan.h:4304
ArrayRef< VPIRBasicBlock * > getExitBlocks() const
Return an ArrayRef containing VPIRBasicBlocks wrapping the exit blocks of the original scalar loop.
Definition VPlan.h:4228
LLVM_ABI_FOR_TEST VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
Definition VPlan.cpp:1049
bool hasEarlyExit() const
Returns true if the VPlan is based on a loop with an early exit.
Definition VPlan.h:4449
InstructionCost cost(ElementCount VF, VPCostContext &Ctx)
Return the cost of this plan.
Definition VPlan.cpp:1031
void resetTripCount(VPValue *NewTripCount)
Resets the trip count for the VPlan.
Definition VPlan.h:4252
VPBasicBlock * getMiddleBlock()
Returns the 'middle' block of the plan, that is the block that selects whether to execute the scalar ...
Definition VPlan.h:4201
VPValue * getOrAddLiveIn(Value *V)
Gets the live-in VPValue for V or adds a new live-in (if none exists yet) for V.
Definition VPlan.h:4328
VPBasicBlock * getScalarPreheader() const
Return the VPBasicBlock for the preheader of the scalar loop.
Definition VPlan.h:4219
void execute(VPTransformState *State)
Generate the IR code for this VPlan.
Definition VPlan.cpp:943
VPCanonicalIVPHIRecipe * getCanonicalIV()
Returns the canonical induction recipe of the vector loop.
Definition VPlan.h:4382
VPIRBasicBlock * getScalarHeader() const
Return the VPIRBasicBlock wrapping the header of the scalar loop.
Definition VPlan.h:4224
VPBasicBlock * getVectorPreheader()
Returns the preheader of the vector loop region, if one exists, or null otherwise.
Definition VPlan.h:4181
VPlan * duplicate()
Clone the current VPlan, update all VPValues of the new VPlan and cloned recipes to refer to the clon...
Definition VPlan.cpp:1191
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:256
LLVM_ABI bool hasOneUser() const
Return true if there is exactly one user of this value.
Definition Value.cpp:166
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
Definition Value.cpp:390
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition Value.h:439
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:546
iterator_range< user_iterator > users()
Definition Value.h:426
LLVM_ABI LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.cpp:1099
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:322
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
std::pair< iterator, bool > insert(const ValueT &V)
Definition DenseSet.h:202
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
Definition DenseSet.h:175
constexpr bool hasKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns true if there exists a value X where RHS.multiplyCoefficientBy(X) will result in a value whos...
Definition TypeSize.h:270
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:201
static constexpr bool isKnownLE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:231
constexpr bool isNonZero() const
Definition TypeSize.h:156
constexpr ScalarTy getKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns a value X where RHS.multiplyCoefficientBy(X) will result in a value whose quantity matches ou...
Definition TypeSize.h:278
static constexpr bool isKnownLT(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:217
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition TypeSize.h:169
constexpr LeafTy multiplyCoefficientBy(ScalarTy RHS) const
Definition TypeSize.h:257
constexpr bool isFixed() const
Returns true if the quantity is not scaled by vscale.
Definition TypeSize.h:172
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition TypeSize.h:166
constexpr bool isZero() const
Definition TypeSize.h:154
static constexpr bool isKnownGT(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:224
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
Definition TypeSize.h:253
static constexpr bool isKnownGE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:238
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
Definition ilist_node.h:34
self_iterator getIterator()
Definition ilist_node.h:123
IteratorT end() const
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition raw_ostream.h:53
A raw_ostream that writes to an std::string.
Changed
This provides a very simple, boring adaptor for a begin and end iterator into a range type.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Entry
Definition COFF.h:862
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition CallingConv.h:24
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
Definition CallingConv.h:76
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
@ BasicBlock
Various leaf nodes.
Definition ISDOpcodes.h:81
std::variant< std::monostate, Loc::Single, Loc::Multi, Loc::MMI, Loc::EntryValue > Variant
Alias for the std::variant specialization base class of DbgVariable.
Definition DwarfDebug.h:189
SpecificConstantMatch m_ZeroInt()
Convenience matchers for specific integer values.
BinaryOp_match< SpecificConstantMatch, SrcTy, TargetOpcode::G_SUB > m_Neg(const SrcTy &&Src)
Matches a register negated by a G_SUB.
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
class_match< BinaryOperator > m_BinOp()
Match an arbitrary binary operation and ignore it.
OneOps_match< OpTy, Instruction::Freeze > m_Freeze(const OpTy &Op)
Matches FreezeInst.
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
bool match(Val *V, const Pattern &P)
bind_ty< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
BinaryOp_match< LHS, RHS, Instruction::Mul > m_Mul(const LHS &L, const RHS &R)
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
SpecificCmpClass_match< LHS, RHS, ICmpInst > m_SpecificICmp(CmpPredicate MatchPred, const LHS &L, const RHS &R)
class_match< CmpInst > m_Cmp()
Matches any compare instruction and ignore it.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
match_combine_or< CastInst_match< OpTy, ZExtInst >, CastInst_match< OpTy, SExtInst > > m_ZExtOrSExt(const OpTy &Op)
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
MatchFunctor< Val, Pattern > match_fn(const Pattern &P)
A match functor that can be used as a UnaryPredicate in functional algorithms like all_of.
class_match< const SCEVVScale > m_SCEVVScale()
bind_cst_ty m_scev_APInt(const APInt *&C)
Match an SCEV constant and bind it to an APInt.
specificloop_ty m_SpecificLoop(const Loop *L)
cst_pred_ty< is_specific_signed_cst > m_scev_SpecificSInt(int64_t V)
Match an SCEV constant with a plain signed integer (sign-extended value will be matched)
SCEVAffineAddRec_match< Op0_t, Op1_t, class_match< const Loop > > m_scev_AffineAddRec(const Op0_t &Op0, const Op1_t &Op1)
SCEVBinaryExpr_match< SCEVMulExpr, Op0_t, Op1_t > m_scev_Mul(const Op0_t &Op0, const Op1_t &Op1)
bool match(const SCEV *S, const Pattern &P)
class_match< const SCEV > m_SCEV()
match_combine_or< AllRecipe_match< Instruction::ZExt, Op0_t >, AllRecipe_match< Instruction::SExt, Op0_t > > m_ZExtOrSExt(const Op0_t &Op0)
VPInstruction_match< VPInstruction::ExtractLastElement, Op0_t > m_ExtractLastElement(const Op0_t &Op0)
class_match< VPValue > m_VPValue()
Match an arbitrary VPValue and ignore it.
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
Add a small namespace to avoid name clashes with the classes used in the streaming interface.
DiagnosticInfoOptimizationBase::Argument NV
NodeAddr< InstrNode * > Instr
Definition RDFGraph.h:389
NodeAddr< PhiNode * > Phi
Definition RDFGraph.h:390
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
bool isSingleScalar(const VPValue *VPV)
Returns true if VPV is a single scalar, either because it produces the same value for all lanes or on...
Definition VPlanUtils.h:44
VPValue * getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr)
Get or create a VPValue that corresponds to the expansion of Expr.
VPBasicBlock * getFirstLoopHeader(VPlan &Plan, VPDominatorTree &VPDT)
Returns the header block of the first, top-level loop, or null if none exist.
const SCEV * getSCEVExprForVPValue(VPValue *V, ScalarEvolution &SE)
Return the SCEV expression for V.
unsigned getVFScaleFactor(VPRecipeBase *R)
Get the VF scaling factor applied to the recipe's output, if the recipe has one.
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI bool simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI, ScalarEvolution *SE, AssumptionCache *AC, MemorySSAUpdater *MSSAU, bool PreserveLCSSA)
Simplify each loop in a loop nest recursively.
LLVM_ABI void ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI, Instruction *I)
Replace the instruction specified by BI with the instruction specified by I.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:318
@ Offset
Definition DWP.cpp:477
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
Definition STLExtras.h:831
FunctionAddr VTableAddr Value
Definition InstrProf.h:137
LLVM_ABI Value * addRuntimeChecks(Instruction *Loc, Loop *TheLoop, const SmallVectorImpl< RuntimePointerCheck > &PointerChecks, SCEVExpander &Expander, bool HoistRuntimeChecks=false)
Add code that checks at runtime if the accessed arrays in PointerChecks overlap.
auto cast_if_present(const Y &Val)
cast_if_present<X> - Functionally identical to cast, except that a null value is accepted.
Definition Casting.h:684
LLVM_ABI bool RemoveRedundantDbgInstrs(BasicBlock *BB)
Try to remove redundant dbg.value instructions from given basic block.
cl::opt< bool > VerifyEachVPlan
LLVM_ABI std::optional< unsigned > getLoopEstimatedTripCount(Loop *L, unsigned *EstimatedLoopInvocationWeight=nullptr)
Return either:
static void reportVectorization(OptimizationRemarkEmitter *ORE, Loop *TheLoop, VectorizationFactor VF, unsigned IC)
Report successful vectorization of the loop.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1705
unsigned getLoadStoreAddressSpace(const Value *I)
A helper function that returns the address space of the pointer operand of load or store instruction.
LLVM_ABI Intrinsic::ID getMinMaxReductionIntrinsicOp(Intrinsic::ID RdxID)
Returns the min/max intrinsic used when expanding a min/max reduction.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1657
LLVM_ABI_FOR_TEST bool verifyVPlanIsValid(const VPlan &Plan, bool VerifyLate=false)
Verify invariants for general VPlans.
LLVM_ABI Intrinsic::ID getVectorIntrinsicIDForCall(const CallInst *CI, const TargetLibraryInfo *TLI)
Returns intrinsic ID for call.
InstructionCost Cost
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition STLExtras.h:2452
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:644
LLVM_ABI bool verifyFunction(const Function &F, raw_ostream *OS=nullptr)
Check a function for errors, useful for use when debugging a pass.
const Value * getLoadStorePointerOperand(const Value *V)
A helper function that returns the pointer operand of a load or store instruction.
OuterAnalysisManagerProxy< ModuleAnalysisManager, Function > ModuleAnalysisManagerFunctionProxy
Provide the ModuleAnalysisManager to Function proxy.
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
LLVM_ABI bool formLCSSARecursively(Loop &L, const DominatorTree &DT, const LoopInfo *LI, ScalarEvolution *SE)
Put a loop nest into LCSSA form.
Definition LCSSA.cpp:449
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition STLExtras.h:2116
LLVM_ABI bool shouldOptimizeForSize(const MachineFunction *MF, ProfileSummaryInfo *PSI, const MachineBlockFrequencyInfo *BFI, PGSOQueryType QueryType=PGSOQueryType::Other)
Returns true if machine function MF is suggested to be size-optimized based on the profile.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:634
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:293
Align getLoadStoreAlignment(const Value *I)
A helper function that returns the alignment of load or store instruction.
iterator_range< df_iterator< VPBlockShallowTraversalWrapper< VPBlockBase * > > > vp_depth_first_shallow(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order.
Definition VPlanCFG.h:216
LLVM_ABI bool VerifySCEV
LLVM_ABI bool isSafeToSpeculativelyExecute(const Instruction *I, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr, bool UseVariableInfo=true, bool IgnoreUBImplyingAttrs=true)
Return true if the instruction does not have any effects besides calculating the result and does not ...
bool isa_and_nonnull(const Y &Val)
Definition Casting.h:677
iterator_range< df_iterator< VPBlockDeepTraversalWrapper< VPBlockBase * > > > vp_depth_first_deep(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order while traversing t...
Definition VPlanCFG.h:243
SmallVector< VPRegisterUsage, 8 > calculateRegisterUsageForPlan(VPlan &Plan, ArrayRef< ElementCount > VFs, const TargetTransformInfo &TTI, const SmallPtrSetImpl< const Value * > &ValuesToIgnore)
Estimate the register usage for Plan and vectorization factors in VFs by calculating the highest numb...
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:348
LLVM_ABI void setBranchWeights(Instruction &I, ArrayRef< uint32_t > Weights, bool IsExpected, bool ElideAllZero=false)
Create a new branch_weights metadata node and add or overwrite a prof metadata reference to instructi...
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:754
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1712
void collectEphemeralRecipesForVPlan(VPlan &Plan, DenseSet< VPRecipeBase * > &EphRecipes)
auto reverse(ContainerTy &&C)
Definition STLExtras.h:408
bool containsIrreducibleCFG(RPOTraversalT &RPOTraversal, const LoopInfoT &LI)
Return true if the control flow in RPOTraversal is irreducible.
Definition CFG.h:149
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:288
void sort(IteratorTy Start, IteratorTy End)
Definition STLExtras.h:1624
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:207
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1719
LLVM_ABI cl::opt< bool > EnableLoopVectorization
LLVM_ABI bool wouldInstructionBeTriviallyDead(const Instruction *I, const TargetLibraryInfo *TLI=nullptr)
Return true if the result produced by the instruction would have no side effects if it was not used.
Definition Local.cpp:421
FunctionAddr VTableAddr Count
Definition InstrProf.h:139
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
Type * toVectorizedTy(Type *Ty, ElementCount EC)
A helper for converting to vectorized types.
LLVM_ABI void llvm_unreachable_internal(const char *msg=nullptr, const char *file=nullptr, unsigned line=0)
This function calls abort(), and prints the optional message to stderr.
T * find_singleton(R &&Range, Predicate P, bool AllowRepeats=false)
Return the single value in Range that satisfies P(<member of Range> *, AllowRepeats)->T * returning n...
Definition STLExtras.h:1767
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
cl::opt< unsigned > ForceTargetInstructionCost
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:548
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
Definition Format.h:118
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
Definition MathExtras.h:405
bool canVectorizeTy(Type *Ty)
Returns true if Ty is a valid vector element type, void, or an unpacked literal struct where all elem...
TargetTransformInfo TTI
static void reportVectorizationInfo(const StringRef Msg, const StringRef ORETag, OptimizationRemarkEmitter *ORE, Loop *TheLoop, Instruction *I=nullptr, DebugLoc DL={})
Reports an informative message: print Msg for debugging purposes as well as an optimization remark.
LLVM_ABI bool isAssignmentTrackingEnabled(const Module &M)
Return true if assignment tracking is enabled for module M.
RecurKind
These are the kinds of recurrences that we support.
@ Or
Bitwise or logical OR of integers.
@ FMulAdd
Sum of float products with llvm.fmuladd(a * b + sum).
@ Sub
Subtraction of integers.
LLVM_ABI Value * getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF)
Given information about an recurrence kind, return the identity for the @llvm.vector....
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition Alignment.h:144
LLVM_ABI void reportVectorizationFailure(const StringRef DebugMsg, const StringRef OREMsg, const StringRef ORETag, OptimizationRemarkEmitter *ORE, Loop *TheLoop, Instruction *I=nullptr)
Reports a vectorization failure: print DebugMsg for debugging purposes along with the corresponding o...
DWARFExpression::Operation Op
ScalarEpilogueLowering
@ CM_ScalarEpilogueNotAllowedLowTripLoop
@ CM_ScalarEpilogueNotNeededUsePredicate
@ CM_ScalarEpilogueNotAllowedOptSize
@ CM_ScalarEpilogueAllowed
@ CM_ScalarEpilogueNotAllowedUsePredicate
LLVM_ABI bool isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Return true if this function can prove that V does not have undef bits and is never poison.
ArrayRef(const T &OneElt) -> ArrayRef< T >
Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)
Return a value for Step multiplied by VF.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:560
LLVM_ABI BasicBlock * SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="", bool Before=false)
Split the specified block at the specified instruction.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1738
auto predecessors(const MachineBasicBlock *BB)
iterator_range< pointer_iterator< WrappedIteratorT > > make_pointer_range(RangeT &&Range)
Definition iterator.h:363
cl::opt< bool > EnableVPlanNativePath
Definition VPlan.cpp:56
Type * getLoadStoreType(const Value *I)
A helper function that returns the type of a load or store instruction.
ArrayRef< Type * > getContainedTypes(Type *const &Ty)
Returns the types contained in Ty.
LLVM_ABI Value * addDiffRuntimeChecks(Instruction *Loc, ArrayRef< PointerDiffInfo > Checks, SCEVExpander &Expander, function_ref< Value *(IRBuilderBase &, unsigned)> GetVF, unsigned IC)
bool pred_empty(const BasicBlock *BB)
Definition CFG.h:119
@ DataAndControlFlowWithoutRuntimeCheck
Use predicate to control both data and control flow, but modify the trip count so that a runtime over...
@ None
Don't use tail folding.
@ DataWithEVL
Use predicated EVL instructions for tail-folding.
@ DataAndControlFlow
Use predicate to control both data and control flow.
@ DataWithoutLaneMask
Same as Data, but avoids using the get.active.lane.mask intrinsic to calculate the mask and instead i...
@ Data
Use predicate only to mask operations on data in the loop.
unsigned getPredBlockCostDivisor(TargetTransformInfo::TargetCostKind CostKind)
A helper function that returns how much we should divide the cost of a predicated block by.
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
LLVM_ABI bool hasBranchWeightMD(const Instruction &I)
Checks if an instructions has Branch Weight Metadata.
hash_code hash_combine(const Ts &...args)
Combine values into a single hash_code.
Definition Hashing.h:592
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
Definition bit.h:299
Type * toVectorTy(Type *Scalar, ElementCount EC)
A helper function for converting Scalar types to vector types.
std::unique_ptr< VPlan > VPlanPtr
Definition VPlan.h:78
constexpr detail::IsaCheckPredicate< Types... > IsaPred
Function object wrapper for the llvm::isa type check.
Definition Casting.h:831
LLVM_ABI MapVector< Instruction *, uint64_t > computeMinimumValueSizes(ArrayRef< BasicBlock * > Blocks, DemandedBits &DB, const TargetTransformInfo *TTI=nullptr)
Compute a map of integer instructions to their minimum legal type size.
hash_code hash_combine_range(InputIteratorT first, InputIteratorT last)
Compute a hash_code for a sequence of values.
Definition Hashing.h:466
LLVM_ABI cl::opt< bool > EnableLoopInterleaving
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:872
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
A special type used by analysis passes to provide an address that identifies that particular analysis...
Definition Analysis.h:29
static LLVM_ABI void collectEphemeralValues(const Loop *L, AssumptionCache *AC, SmallPtrSetImpl< const Value * > &EphValues)
Collect a loop's ephemeral values (those used only by an assume or similar intrinsics in the loop).
An information struct used to provide DenseMap with the various necessary components for a given valu...
Encapsulate information regarding vectorization of a loop and its epilogue.
EpilogueLoopVectorizationInfo(ElementCount MVF, unsigned MUF, ElementCount EVF, unsigned EUF, VPlan &EpiloguePlan)
A class that represents two vectorization factors (initialized with 0 by default).
static FixedScalableVFPair getNone()
This holds details about a histogram operation – a load -> update -> store sequence where each lane i...
Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...
TargetLibraryInfo * TLI
LLVM_ABI LoopVectorizeResult runImpl(Function &F)
LLVM_ABI bool processLoop(Loop *L)
ProfileSummaryInfo * PSI
LoopAccessInfoManager * LAIs
LLVM_ABI void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
LLVM_ABI LoopVectorizePass(LoopVectorizeOptions Opts={})
BlockFrequencyInfo * BFI
ScalarEvolution * SE
AssumptionCache * AC
LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
OptimizationRemarkEmitter * ORE
TargetTransformInfo * TTI
Storage for information about made changes.
A chain of instructions that form a partial reduction.
Instruction * Reduction
The top-level binary operation that forms the reduction to a scalar after the loop body.
Instruction * ExtendA
The extension of each of the inner binary operation's operands.
A CRTP mix-in to automatically provide informational APIs needed for passes.
Definition PassManager.h:70
A marker analysis to determine if extra passes should be run after loop vectorization.
static LLVM_ABI AnalysisKey Key
Holds the VFShape for a specific scalar to vector function mapping.
std::optional< unsigned > getParamIndexForOptionalMask() const
Instruction Set Architecture.
Encapsulates information needed to describe a parameter.
A range of powers-of-2 vectorization factors with fixed start and adjustable end.
ElementCount End
Struct to hold various analysis needed for cost computations.
LoopVectorizationCostModel & CM
bool isLegacyUniformAfterVectorization(Instruction *I, ElementCount VF) const
Return true if I is considered uniform-after-vectorization in the legacy cost model for VF.
bool skipCostComputation(Instruction *UI, bool IsVector) const
Return true if the cost for UI shouldn't be computed, e.g.
InstructionCost getLegacyCost(Instruction *UI, ElementCount VF) const
Return the cost for UI with VF using the legacy cost model as fallback until computing the cost of al...
SmallPtrSet< Instruction *, 8 > SkipCostComputation
A recipe for handling first-order recurrence phis.
Definition VPlan.h:2297
A struct that represents some properties of the register usage of a loop.
VPTransformState holds information passed down when "executing" a VPlan, needed for generating the ou...
A recipe for widening select instructions.
Definition VPlan.h:1724
static void materializeBroadcasts(VPlan &Plan)
Add explicit broadcasts for live-ins and VPValues defined in Plan's entry block if they are used as v...
static void materializeBackedgeTakenCount(VPlan &Plan, VPBasicBlock *VectorPH)
Materialize the backedge-taken count to be computed explicitly using VPInstructions.
static LLVM_ABI_FOR_TEST std::unique_ptr< VPlan > buildVPlan0(Loop *TheLoop, LoopInfo &LI, Type *InductionTy, DebugLoc IVDL, PredicatedScalarEvolution &PSE)
Create a base VPlan0, serving as the common starting point for all later candidates.
static void optimizeInductionExitUsers(VPlan &Plan, DenseMap< VPValue *, VPValue * > &EndValues, ScalarEvolution &SE)
If there's a single exit block, optimize its phi recipes that use exiting IV values by feeding them p...
static LLVM_ABI_FOR_TEST void handleEarlyExits(VPlan &Plan, bool HasUncountableExit)
Update Plan to account for all early exits.
static void canonicalizeEVLLoops(VPlan &Plan)
Transform EVL loops to use variable-length stepping after region dissolution.
static void dropPoisonGeneratingRecipes(VPlan &Plan, const std::function< bool(BasicBlock *)> &BlockNeedsPredication)
Drop poison flags from recipes that may generate a poison value that is used after vectorization,...
static void createInterleaveGroups(VPlan &Plan, const SmallPtrSetImpl< const InterleaveGroup< Instruction > * > &InterleaveGroups, VPRecipeBuilder &RecipeBuilder, const bool &ScalarEpilogueAllowed)
static bool runPass(bool(*Transform)(VPlan &, ArgsTy...), VPlan &Plan, typename std::remove_reference< ArgsTy >::type &...Args)
Helper to run a VPlan transform Transform on VPlan, forwarding extra arguments to the transform.
static void addBranchWeightToMiddleTerminator(VPlan &Plan, ElementCount VF, std::optional< unsigned > VScaleForTuning)
Add branch weight metadata, if the Plan's middle block is terminated by a BranchOnCond recipe.
static void materializeBuildVectors(VPlan &Plan)
Add explicit Build[Struct]Vector recipes that combine multiple scalar values into single vectors.
static void unrollByUF(VPlan &Plan, unsigned UF)
Explicitly unroll Plan by UF.
static DenseMap< const SCEV *, Value * > expandSCEVs(VPlan &Plan, ScalarEvolution &SE)
Expand VPExpandSCEVRecipes in Plan's entry block.
static void convertToConcreteRecipes(VPlan &Plan)
Lower abstract recipes to concrete ones, that can be codegen'd.
static void addMinimumIterationCheck(VPlan &Plan, ElementCount VF, unsigned UF, ElementCount MinProfitableTripCount, bool RequiresScalarEpilogue, bool TailFolded, bool CheckNeededWithTailFolding, Loop *OrigLoop, const uint32_t *MinItersBypassWeights, DebugLoc DL, ScalarEvolution &SE)
static void convertToAbstractRecipes(VPlan &Plan, VPCostContext &Ctx, VFRange &Range)
This function converts initial recipes to the abstract recipes and clamps Range based on cost model f...
static void materializeConstantVectorTripCount(VPlan &Plan, ElementCount BestVF, unsigned BestUF, PredicatedScalarEvolution &PSE)
static DenseMap< VPBasicBlock *, VPValue * > introduceMasksAndLinearize(VPlan &Plan, bool FoldTail)
Predicate and linearize the control-flow in the only loop region of Plan.
static void addExplicitVectorLength(VPlan &Plan, const std::optional< unsigned > &MaxEVLSafeElements)
Add a VPEVLBasedIVPHIRecipe and related recipes to Plan and replaces all uses except the canonical IV...
static void replaceSymbolicStrides(VPlan &Plan, PredicatedScalarEvolution &PSE, const DenseMap< Value *, const SCEV * > &StridesMap)
Replace symbolic strides from StridesMap in Plan with constants when possible.
static bool handleMaxMinNumReductions(VPlan &Plan)
Check if Plan contains any FMaxNum or FMinNum reductions.
static void removeBranchOnConst(VPlan &Plan)
Remove BranchOnCond recipes with true or false conditions together with removing dead edges to their ...
static LLVM_ABI_FOR_TEST void createLoopRegions(VPlan &Plan)
Replace loops in Plan's flat CFG with VPRegionBlocks, turning Plan's flat CFG into a hierarchical CFG...
static void removeDeadRecipes(VPlan &Plan)
Remove dead recipes from Plan.
static void attachCheckBlock(VPlan &Plan, Value *Cond, BasicBlock *CheckBlock, bool AddBranchWeights)
Wrap runtime check block CheckBlock in a VPIRBB and Cond in a VPValue and connect the block to Plan,...
static void materializeVectorTripCount(VPlan &Plan, VPBasicBlock *VectorPHVPBB, bool TailByMasking, bool RequiresScalarEpilogue)
Materialize vector trip count computations to a set of VPInstructions.
static void simplifyRecipes(VPlan &Plan)
Perform instcombine-like simplifications on recipes in Plan.
static LLVM_ABI_FOR_TEST bool tryToConvertVPInstructionsToVPRecipes(VPlanPtr &Plan, function_ref< const InductionDescriptor *(PHINode *)> GetIntOrFpInductionDescriptor, const TargetLibraryInfo &TLI)
Replaces the VPInstructions in Plan with corresponding widen recipes.
static void replicateByVF(VPlan &Plan, ElementCount VF)
Replace each replicating VPReplicateRecipe and VPInstruction outside of any replicate region in Plan ...
static void clearReductionWrapFlags(VPlan &Plan)
Clear NSW/NUW flags from reduction instructions if necessary.
static void cse(VPlan &Plan)
Perform common-subexpression-elimination on Plan.
static void addActiveLaneMask(VPlan &Plan, bool UseActiveLaneMaskForControlFlow, bool DataAndControlFlowWithoutRuntimeCheck)
Replace (ICMP_ULE, wide canonical IV, backedge-taken-count) checks with an (active-lane-mask recipe,...
static void optimize(VPlan &Plan)
Apply VPlan-to-VPlan optimizations to Plan, including induction recipe optimizations,...
static void dissolveLoopRegions(VPlan &Plan)
Replace loop regions with explicit CFG.
static void narrowInterleaveGroups(VPlan &Plan, ElementCount VF, unsigned VectorRegWidth)
Try to convert a plan with interleave groups with VF elements to a plan with the interleave groups re...
static void truncateToMinimalBitwidths(VPlan &Plan, const MapVector< Instruction *, uint64_t > &MinBWs)
Insert truncates and extends for any truncated recipe.
static bool adjustFixedOrderRecurrences(VPlan &Plan, VPBuilder &Builder)
Try to have all users of fixed-order recurrences appear after the recipe defining their previous valu...
static void optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF, unsigned BestUF, PredicatedScalarEvolution &PSE)
Optimize Plan based on BestVF and BestUF.
static void materializeVFAndVFxUF(VPlan &Plan, VPBasicBlock *VectorPH, ElementCount VF)
Materialize VF and VFxUF to be computed explicitly using VPInstructions.
static void addMinimumVectorEpilogueIterationCheck(VPlan &Plan, Value *TripCount, Value *VectorTripCount, bool RequiresScalarEpilogue, ElementCount EpilogueVF, unsigned EpilogueUF, unsigned MainLoopStep, unsigned EpilogueLoopStep, ScalarEvolution &SE)
Add a check to Plan to see if the epilogue vector loop should be executed.
static LLVM_ABI_FOR_TEST void addMiddleCheck(VPlan &Plan, bool RequiresScalarEpilogueCheck, bool TailFolded)
If a check is needed to guard executing the scalar epilogue loop, it will be added to the middle bloc...
TODO: The following VectorizationFactor was pulled out of LoopVectorizationCostModel class.
InstructionCost Cost
Cost of the loop with that width.
ElementCount MinProfitableTripCount
The minimum trip count required to make vectorization profitable, e.g.
ElementCount Width
Vector width with best cost.
InstructionCost ScalarCost
Cost of the scalar loop.
static VectorizationFactor Disabled()
Width 1 means no vectorization, cost 0 means uncomputed cost.
static LLVM_ABI bool HoistRuntimeChecks