Thanks to visit codestin.com
Credit goes to llvm.org

LLVM 22.0.0git
LoopVectorize.cpp
Go to the documentation of this file.
1//===- LoopVectorize.cpp - A Loop Vectorizer ------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This is the LLVM loop vectorizer. This pass modifies 'vectorizable' loops
10// and generates target-independent LLVM-IR.
11// The vectorizer uses the TargetTransformInfo analysis to estimate the costs
12// of instructions in order to estimate the profitability of vectorization.
13//
14// The loop vectorizer combines consecutive loop iterations into a single
15// 'wide' iteration. After this transformation the index is incremented
16// by the SIMD vector width, and not by one.
17//
18// This pass has three parts:
19// 1. The main loop pass that drives the different parts.
20// 2. LoopVectorizationLegality - A unit that checks for the legality
21// of the vectorization.
22// 3. InnerLoopVectorizer - A unit that performs the actual
23// widening of instructions.
24// 4. LoopVectorizationCostModel - A unit that checks for the profitability
25// of vectorization. It decides on the optimal vector width, which
26// can be one, if vectorization is not profitable.
27//
28// There is a development effort going on to migrate loop vectorizer to the
29// VPlan infrastructure and to introduce outer loop vectorization support (see
30// docs/VectorizationPlan.rst and
31// http://lists.llvm.org/pipermail/llvm-dev/2017-December/119523.html). For this
32// purpose, we temporarily introduced the VPlan-native vectorization path: an
33// alternative vectorization path that is natively implemented on top of the
34// VPlan infrastructure. See EnableVPlanNativePath for enabling.
35//
36//===----------------------------------------------------------------------===//
37//
38// The reduction-variable vectorization is based on the paper:
39// D. Nuzman and R. Henderson. Multi-platform Auto-vectorization.
40//
41// Variable uniformity checks are inspired by:
42// Karrenberg, R. and Hack, S. Whole Function Vectorization.
43//
44// The interleaved access vectorization is based on the paper:
45// Dorit Nuzman, Ira Rosen and Ayal Zaks. Auto-Vectorization of Interleaved
46// Data for SIMD
47//
48// Other ideas/concepts are from:
49// A. Zaks and D. Nuzman. Autovectorization in GCC-two years later.
50//
51// S. Maleki, Y. Gao, M. Garzaran, T. Wong and D. Padua. An Evaluation of
52// Vectorizing Compilers.
53//
54//===----------------------------------------------------------------------===//
55
58#include "VPRecipeBuilder.h"
59#include "VPlan.h"
60#include "VPlanAnalysis.h"
61#include "VPlanCFG.h"
62#include "VPlanHelpers.h"
63#include "VPlanPatternMatch.h"
64#include "VPlanTransforms.h"
65#include "VPlanUtils.h"
66#include "VPlanVerifier.h"
67#include "llvm/ADT/APInt.h"
68#include "llvm/ADT/ArrayRef.h"
69#include "llvm/ADT/DenseMap.h"
71#include "llvm/ADT/Hashing.h"
72#include "llvm/ADT/MapVector.h"
73#include "llvm/ADT/STLExtras.h"
76#include "llvm/ADT/Statistic.h"
77#include "llvm/ADT/StringRef.h"
78#include "llvm/ADT/Twine.h"
79#include "llvm/ADT/TypeSwitch.h"
84#include "llvm/Analysis/CFG.h"
101#include "llvm/IR/Attributes.h"
102#include "llvm/IR/BasicBlock.h"
103#include "llvm/IR/CFG.h"
104#include "llvm/IR/Constant.h"
105#include "llvm/IR/Constants.h"
106#include "llvm/IR/DataLayout.h"
107#include "llvm/IR/DebugInfo.h"
108#include "llvm/IR/DebugLoc.h"
109#include "llvm/IR/DerivedTypes.h"
111#include "llvm/IR/Dominators.h"
112#include "llvm/IR/Function.h"
113#include "llvm/IR/IRBuilder.h"
114#include "llvm/IR/InstrTypes.h"
115#include "llvm/IR/Instruction.h"
116#include "llvm/IR/Instructions.h"
118#include "llvm/IR/Intrinsics.h"
119#include "llvm/IR/MDBuilder.h"
120#include "llvm/IR/Metadata.h"
121#include "llvm/IR/Module.h"
122#include "llvm/IR/Operator.h"
123#include "llvm/IR/PatternMatch.h"
125#include "llvm/IR/Type.h"
126#include "llvm/IR/Use.h"
127#include "llvm/IR/User.h"
128#include "llvm/IR/Value.h"
129#include "llvm/IR/Verifier.h"
130#include "llvm/Support/Casting.h"
132#include "llvm/Support/Debug.h"
147#include <algorithm>
148#include <cassert>
149#include <cstdint>
150#include <functional>
151#include <iterator>
152#include <limits>
153#include <memory>
154#include <string>
155#include <tuple>
156#include <utility>
157
158using namespace llvm;
159using namespace SCEVPatternMatch;
160
161#define LV_NAME "loop-vectorize"
162#define DEBUG_TYPE LV_NAME
163
164#ifndef NDEBUG
165const char VerboseDebug[] = DEBUG_TYPE "-verbose";
166#endif
167
168STATISTIC(LoopsVectorized, "Number of loops vectorized");
169STATISTIC(LoopsAnalyzed, "Number of loops analyzed for vectorization");
170STATISTIC(LoopsEpilogueVectorized, "Number of epilogues vectorized");
171STATISTIC(LoopsEarlyExitVectorized, "Number of early exit loops vectorized");
172
174 "enable-epilogue-vectorization", cl::init(true), cl::Hidden,
175 cl::desc("Enable vectorization of epilogue loops."));
176
178 "epilogue-vectorization-force-VF", cl::init(1), cl::Hidden,
179 cl::desc("When epilogue vectorization is enabled, and a value greater than "
180 "1 is specified, forces the given VF for all applicable epilogue "
181 "loops."));
182
184 "epilogue-vectorization-minimum-VF", cl::Hidden,
185 cl::desc("Only loops with vectorization factor equal to or larger than "
186 "the specified value are considered for epilogue vectorization."));
187
188/// Loops with a known constant trip count below this number are vectorized only
189/// if no scalar iteration overheads are incurred.
191 "vectorizer-min-trip-count", cl::init(16), cl::Hidden,
192 cl::desc("Loops with a constant trip count that is smaller than this "
193 "value are vectorized only if no scalar iteration overheads "
194 "are incurred."));
195
197 "vectorize-memory-check-threshold", cl::init(128), cl::Hidden,
198 cl::desc("The maximum allowed number of runtime memory checks"));
199
200// Option prefer-predicate-over-epilogue indicates that an epilogue is undesired,
201// that predication is preferred, and this lists all options. I.e., the
202// vectorizer will try to fold the tail-loop (epilogue) into the vector body
203// and predicate the instructions accordingly. If tail-folding fails, there are
204// different fallback strategies depending on these values:
211} // namespace PreferPredicateTy
212
214 "prefer-predicate-over-epilogue",
217 cl::desc("Tail-folding and predication preferences over creating a scalar "
218 "epilogue loop."),
220 "scalar-epilogue",
221 "Don't tail-predicate loops, create scalar epilogue"),
223 "predicate-else-scalar-epilogue",
224 "prefer tail-folding, create scalar epilogue if tail "
225 "folding fails."),
227 "predicate-dont-vectorize",
228 "prefers tail-folding, don't attempt vectorization if "
229 "tail-folding fails.")));
230
232 "force-tail-folding-style", cl::desc("Force the tail folding style"),
235 clEnumValN(TailFoldingStyle::None, "none", "Disable tail folding"),
238 "Create lane mask for data only, using active.lane.mask intrinsic"),
240 "data-without-lane-mask",
241 "Create lane mask with compare/stepvector"),
243 "Create lane mask using active.lane.mask intrinsic, and use "
244 "it for both data and control flow"),
246 "data-and-control-without-rt-check",
247 "Similar to data-and-control, but remove the runtime check"),
249 "Use predicated EVL instructions for tail folding. If EVL "
250 "is unsupported, fallback to data-without-lane-mask.")));
251
253 "vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden,
254 cl::desc("Maximize bandwidth when selecting vectorization factor which "
255 "will be determined by the smallest type in loop."));
256
258 "enable-interleaved-mem-accesses", cl::init(false), cl::Hidden,
259 cl::desc("Enable vectorization on interleaved memory accesses in a loop"));
260
261/// An interleave-group may need masking if it resides in a block that needs
262/// predication, or in order to mask away gaps.
264 "enable-masked-interleaved-mem-accesses", cl::init(false), cl::Hidden,
265 cl::desc("Enable vectorization on masked interleaved memory accesses in a loop"));
266
268 "force-target-num-scalar-regs", cl::init(0), cl::Hidden,
269 cl::desc("A flag that overrides the target's number of scalar registers."));
270
272 "force-target-num-vector-regs", cl::init(0), cl::Hidden,
273 cl::desc("A flag that overrides the target's number of vector registers."));
274
276 "force-target-max-scalar-interleave", cl::init(0), cl::Hidden,
277 cl::desc("A flag that overrides the target's max interleave factor for "
278 "scalar loops."));
279
281 "force-target-max-vector-interleave", cl::init(0), cl::Hidden,
282 cl::desc("A flag that overrides the target's max interleave factor for "
283 "vectorized loops."));
284
286 "force-target-instruction-cost", cl::init(0), cl::Hidden,
287 cl::desc("A flag that overrides the target's expected cost for "
288 "an instruction to a single constant value. Mostly "
289 "useful for getting consistent testing."));
290
292 "force-target-supports-scalable-vectors", cl::init(false), cl::Hidden,
293 cl::desc(
294 "Pretend that scalable vectors are supported, even if the target does "
295 "not support them. This flag should only be used for testing."));
296
298 "small-loop-cost", cl::init(20), cl::Hidden,
299 cl::desc(
300 "The cost of a loop that is considered 'small' by the interleaver."));
301
303 "loop-vectorize-with-block-frequency", cl::init(true), cl::Hidden,
304 cl::desc("Enable the use of the block frequency analysis to access PGO "
305 "heuristics minimizing code growth in cold regions and being more "
306 "aggressive in hot regions."));
307
308// Runtime interleave loops for load/store throughput.
310 "enable-loadstore-runtime-interleave", cl::init(true), cl::Hidden,
311 cl::desc(
312 "Enable runtime interleaving until load/store ports are saturated"));
313
314/// The number of stores in a loop that are allowed to need predication.
316 "vectorize-num-stores-pred", cl::init(1), cl::Hidden,
317 cl::desc("Max number of stores to be predicated behind an if."));
318
320 "enable-ind-var-reg-heur", cl::init(true), cl::Hidden,
321 cl::desc("Count the induction variable only once when interleaving"));
322
324 "enable-cond-stores-vec", cl::init(true), cl::Hidden,
325 cl::desc("Enable if predication of stores during vectorization."));
326
328 "max-nested-scalar-reduction-interleave", cl::init(2), cl::Hidden,
329 cl::desc("The maximum interleave count to use when interleaving a scalar "
330 "reduction in a nested loop."));
331
332static cl::opt<bool>
333 PreferInLoopReductions("prefer-inloop-reductions", cl::init(false),
335 cl::desc("Prefer in-loop vector reductions, "
336 "overriding the targets preference."));
337
339 "force-ordered-reductions", cl::init(false), cl::Hidden,
340 cl::desc("Enable the vectorisation of loops with in-order (strict) "
341 "FP reductions"));
342
344 "prefer-predicated-reduction-select", cl::init(false), cl::Hidden,
345 cl::desc(
346 "Prefer predicating a reduction operation over an after loop select."));
347
349 "enable-vplan-native-path", cl::Hidden,
350 cl::desc("Enable VPlan-native vectorization path with "
351 "support for outer loop vectorization."));
352
354 llvm::VerifyEachVPlan("vplan-verify-each",
355#ifdef EXPENSIVE_CHECKS
356 cl::init(true),
357#else
358 cl::init(false),
359#endif
361 cl::desc("Verfiy VPlans after VPlan transforms."));
362
363// This flag enables the stress testing of the VPlan H-CFG construction in the
364// VPlan-native vectorization path. It must be used in conjuction with
365// -enable-vplan-native-path. -vplan-verify-hcfg can also be used to enable the
366// verification of the H-CFGs built.
368 "vplan-build-stress-test", cl::init(false), cl::Hidden,
369 cl::desc(
370 "Build VPlan for every supported loop nest in the function and bail "
371 "out right after the build (stress test the VPlan H-CFG construction "
372 "in the VPlan-native vectorization path)."));
373
375 "interleave-loops", cl::init(true), cl::Hidden,
376 cl::desc("Enable loop interleaving in Loop vectorization passes"));
378 "vectorize-loops", cl::init(true), cl::Hidden,
379 cl::desc("Run the Loop vectorization passes"));
380
382 "force-widen-divrem-via-safe-divisor", cl::Hidden,
383 cl::desc(
384 "Override cost based safe divisor widening for div/rem instructions"));
385
387 "vectorizer-maximize-bandwidth-for-vector-calls", cl::init(true),
389 cl::desc("Try wider VFs if they enable the use of vector variants"));
390
392 "enable-early-exit-vectorization", cl::init(true), cl::Hidden,
393 cl::desc(
394 "Enable vectorization of early exit loops with uncountable exits."));
395
397 "vectorizer-consider-reg-pressure", cl::init(false), cl::Hidden,
398 cl::desc("Discard VFs if their register pressure is too high."));
399
400// Likelyhood of bypassing the vectorized loop because there are zero trips left
401// after prolog. See `emitIterationCountCheck`.
402static constexpr uint32_t MinItersBypassWeights[] = {1, 127};
403
404/// A helper function that returns true if the given type is irregular. The
405/// type is irregular if its allocated size doesn't equal the store size of an
406/// element of the corresponding vector type.
407static bool hasIrregularType(Type *Ty, const DataLayout &DL) {
408 // Determine if an array of N elements of type Ty is "bitcast compatible"
409 // with a <N x Ty> vector.
410 // This is only true if there is no padding between the array elements.
411 return DL.getTypeAllocSizeInBits(Ty) != DL.getTypeSizeInBits(Ty);
412}
413
414/// A version of ScalarEvolution::getSmallConstantTripCount that returns an
415/// ElementCount to include loops whose trip count is a function of vscale.
417 const Loop *L) {
418 if (unsigned ExpectedTC = SE->getSmallConstantTripCount(L))
419 return ElementCount::getFixed(ExpectedTC);
420
421 const SCEV *BTC = SE->getBackedgeTakenCount(L);
423 return ElementCount::getFixed(0);
424
425 const SCEV *ExitCount = SE->getTripCountFromExitCount(BTC, BTC->getType(), L);
426 if (isa<SCEVVScale>(ExitCount))
428
429 const APInt *Scale;
430 if (match(ExitCount, m_scev_Mul(m_scev_APInt(Scale), m_SCEVVScale())))
431 if (cast<SCEVMulExpr>(ExitCount)->hasNoUnsignedWrap())
432 if (Scale->getActiveBits() <= 32)
434
435 return ElementCount::getFixed(0);
436}
437
438/// Returns "best known" trip count, which is either a valid positive trip count
439/// or std::nullopt when an estimate cannot be made (including when the trip
440/// count would overflow), for the specified loop \p L as defined by the
441/// following procedure:
442/// 1) Returns exact trip count if it is known.
443/// 2) Returns expected trip count according to profile data if any.
444/// 3) Returns upper bound estimate if known, and if \p CanUseConstantMax.
445/// 4) Returns std::nullopt if all of the above failed.
446static std::optional<ElementCount>
448 bool CanUseConstantMax = true) {
449 // Check if exact trip count is known.
450 if (auto ExpectedTC = getSmallConstantTripCount(PSE.getSE(), L))
451 return ExpectedTC;
452
453 // Check if there is an expected trip count available from profile data.
455 if (auto EstimatedTC = getLoopEstimatedTripCount(L))
456 return ElementCount::getFixed(*EstimatedTC);
457
458 if (!CanUseConstantMax)
459 return std::nullopt;
460
461 // Check if upper bound estimate is known.
462 if (unsigned ExpectedTC = PSE.getSmallConstantMaxTripCount())
463 return ElementCount::getFixed(ExpectedTC);
464
465 return std::nullopt;
466}
467
468namespace {
469// Forward declare GeneratedRTChecks.
470class GeneratedRTChecks;
471
472using SCEV2ValueTy = DenseMap<const SCEV *, Value *>;
473} // namespace
474
475namespace llvm {
476
478
479/// InnerLoopVectorizer vectorizes loops which contain only one basic
480/// block to a specified vectorization factor (VF).
481/// This class performs the widening of scalars into vectors, or multiple
482/// scalars. This class also implements the following features:
483/// * It inserts an epilogue loop for handling loops that don't have iteration
484/// counts that are known to be a multiple of the vectorization factor.
485/// * It handles the code generation for reduction variables.
486/// * Scalarization (implementation using scalars) of un-vectorizable
487/// instructions.
488/// InnerLoopVectorizer does not perform any vectorization-legality
489/// checks, and relies on the caller to check for the different legality
490/// aspects. The InnerLoopVectorizer relies on the
491/// LoopVectorizationLegality class to provide information about the induction
492/// and reduction variables that were found to a given vectorization factor.
494public:
498 ElementCount VecWidth, unsigned UnrollFactor,
500 ProfileSummaryInfo *PSI, GeneratedRTChecks &RTChecks,
501 VPlan &Plan)
502 : OrigLoop(OrigLoop), PSE(PSE), LI(LI), DT(DT), TTI(TTI), AC(AC),
503 VF(VecWidth), UF(UnrollFactor), Builder(PSE.getSE()->getContext()),
506 Plan.getVectorLoopRegion()->getSinglePredecessor())) {}
507
508 virtual ~InnerLoopVectorizer() = default;
509
510 /// Creates a basic block for the scalar preheader. Both
511 /// EpilogueVectorizerMainLoop and EpilogueVectorizerEpilogueLoop overwrite
512 /// the method to create additional blocks and checks needed for epilogue
513 /// vectorization.
515
516 /// Fix the vectorized code, taking care of header phi's, and more.
518
519 /// Fix the non-induction PHIs in \p Plan.
521
522 /// Returns the original loop trip count.
523 Value *getTripCount() const { return TripCount; }
524
525 /// Used to set the trip count after ILV's construction and after the
526 /// preheader block has been executed. Note that this always holds the trip
527 /// count of the original loop for both main loop and epilogue vectorization.
528 void setTripCount(Value *TC) { TripCount = TC; }
529
530protected:
532
533 /// Create and return a new IR basic block for the scalar preheader whose name
534 /// is prefixed with \p Prefix.
536
537 /// Allow subclasses to override and print debug traces before/after vplan
538 /// execution, when trace information is requested.
539 virtual void printDebugTracesAtStart() {}
540 virtual void printDebugTracesAtEnd() {}
541
542 /// The original loop.
544
545 /// A wrapper around ScalarEvolution used to add runtime SCEV checks. Applies
546 /// dynamic knowledge to simplify SCEV expressions and converts them to a
547 /// more usable form.
549
550 /// Loop Info.
552
553 /// Dominator Tree.
555
556 /// Target Transform Info.
558
559 /// Assumption Cache.
561
562 /// The vectorization SIMD factor to use. Each vector will have this many
563 /// vector elements.
565
566 /// The vectorization unroll factor to use. Each scalar is vectorized to this
567 /// many different vector instructions.
568 unsigned UF;
569
570 /// The builder that we use
572
573 // --- Vectorization state ---
574
575 /// Trip count of the original loop.
576 Value *TripCount = nullptr;
577
578 /// The profitablity analysis.
580
581 /// BFI and PSI are used to check for profile guided size optimizations.
584
585 /// Structure to hold information about generated runtime checks, responsible
586 /// for cleaning the checks, if vectorization turns out unprofitable.
587 GeneratedRTChecks &RTChecks;
588
590
591 /// The vector preheader block of \p Plan, used as target for check blocks
592 /// introduced during skeleton creation.
594};
595
596/// Encapsulate information regarding vectorization of a loop and its epilogue.
597/// This information is meant to be updated and used across two stages of
598/// epilogue vectorization.
601 unsigned MainLoopUF = 0;
603 unsigned EpilogueUF = 0;
606 Value *TripCount = nullptr;
609
611 ElementCount EVF, unsigned EUF,
613 : MainLoopVF(MVF), MainLoopUF(MUF), EpilogueVF(EVF), EpilogueUF(EUF),
615 assert(EUF == 1 &&
616 "A high UF for the epilogue loop is likely not beneficial.");
617 }
618};
619
620/// An extension of the inner loop vectorizer that creates a skeleton for a
621/// vectorized loop that has its epilogue (residual) also vectorized.
622/// The idea is to run the vplan on a given loop twice, firstly to setup the
623/// skeleton and vectorize the main loop, and secondly to complete the skeleton
624/// from the first step and vectorize the epilogue. This is achieved by
625/// deriving two concrete strategy classes from this base class and invoking
626/// them in succession from the loop vectorizer planner.
628public:
639
640 /// Holds and updates state information required to vectorize the main loop
641 /// and its epilogue in two separate passes. This setup helps us avoid
642 /// regenerating and recomputing runtime safety checks. It also helps us to
643 /// shorten the iteration-count-check path length for the cases where the
644 /// iteration count of the loop is so small that the main vector loop is
645 /// completely skipped.
647
648protected:
650};
651
652/// A specialized derived class of inner loop vectorizer that performs
653/// vectorization of *main* loops in the process of vectorizing loops and their
654/// epilogues.
656public:
668 /// Implements the interface for creating a vectorized skeleton using the
669 /// *main loop* strategy (i.e., the first pass of VPlan execution).
671
672protected:
673 /// Introduces a new VPIRBasicBlock for \p CheckIRBB to Plan between the
674 /// vector preheader and its predecessor, also connecting the new block to the
675 /// scalar preheader.
676 void introduceCheckBlockInVPlan(BasicBlock *CheckIRBB);
677
678 // Create a check to see if the main vector loop should be executed
680 unsigned UF) const;
681
682 /// Emits an iteration count bypass check once for the main loop (when \p
683 /// ForEpilogue is false) and once for the epilogue loop (when \p
684 /// ForEpilogue is true).
686 bool ForEpilogue);
687 void printDebugTracesAtStart() override;
688 void printDebugTracesAtEnd() override;
689};
690
691// A specialized derived class of inner loop vectorizer that performs
692// vectorization of *epilogue* loops in the process of vectorizing loops and
693// their epilogues.
695 /// The additional bypass block which conditionally skips over the epilogue
696 /// loop after executing the main loop. Needed to resume inductions and
697 /// reductions during epilogue vectorization.
698 BasicBlock *AdditionalBypassBlock = nullptr;
699
700public:
712 /// Implements the interface for creating a vectorized skeleton using the
713 /// *epilogue loop* strategy (i.e., the second pass of VPlan execution).
715
716 /// Return the additional bypass block which targets the scalar loop by
717 /// skipping the epilogue loop after completing the main loop.
719 assert(AdditionalBypassBlock &&
720 "Trying to access AdditionalBypassBlock but it has not been set");
721 return AdditionalBypassBlock;
722 }
723
724protected:
725 /// Emits an iteration count bypass check after the main vector loop has
726 /// finished to see if there are any iterations left to execute by either
727 /// the vector epilogue or the scalar epilogue.
728 BasicBlock *emitMinimumVectorEpilogueIterCountCheck(BasicBlock *VectorPH,
729 BasicBlock *Bypass,
730 BasicBlock *Insert);
731 void printDebugTracesAtStart() override;
732 void printDebugTracesAtEnd() override;
733};
734} // end namespace llvm
735
736/// Look for a meaningful debug location on the instruction or its operands.
738 if (!I)
739 return DebugLoc::getUnknown();
740
742 if (I->getDebugLoc() != Empty)
743 return I->getDebugLoc();
744
745 for (Use &Op : I->operands()) {
746 if (Instruction *OpInst = dyn_cast<Instruction>(Op))
747 if (OpInst->getDebugLoc() != Empty)
748 return OpInst->getDebugLoc();
749 }
750
751 return I->getDebugLoc();
752}
753
754/// Write a \p DebugMsg about vectorization to the debug output stream. If \p I
755/// is passed, the message relates to that particular instruction.
756#ifndef NDEBUG
757static void debugVectorizationMessage(const StringRef Prefix,
758 const StringRef DebugMsg,
759 Instruction *I) {
760 dbgs() << "LV: " << Prefix << DebugMsg;
761 if (I != nullptr)
762 dbgs() << " " << *I;
763 else
764 dbgs() << '.';
765 dbgs() << '\n';
766}
767#endif
768
769/// Create an analysis remark that explains why vectorization failed
770///
771/// \p PassName is the name of the pass (e.g. can be AlwaysPrint). \p
772/// RemarkName is the identifier for the remark. If \p I is passed it is an
773/// instruction that prevents vectorization. Otherwise \p TheLoop is used for
774/// the location of the remark. If \p DL is passed, use it as debug location for
775/// the remark. \return the remark object that can be streamed to.
777createLVAnalysis(const char *PassName, StringRef RemarkName, Loop *TheLoop,
778 Instruction *I, DebugLoc DL = {}) {
779 BasicBlock *CodeRegion = I ? I->getParent() : TheLoop->getHeader();
780 // If debug location is attached to the instruction, use it. Otherwise if DL
781 // was not provided, use the loop's.
782 if (I && I->getDebugLoc())
783 DL = I->getDebugLoc();
784 else if (!DL)
785 DL = TheLoop->getStartLoc();
786
787 return OptimizationRemarkAnalysis(PassName, RemarkName, DL, CodeRegion);
788}
789
790namespace llvm {
791
792/// Return a value for Step multiplied by VF.
794 int64_t Step) {
795 assert(Ty->isIntegerTy() && "Expected an integer step");
796 ElementCount VFxStep = VF.multiplyCoefficientBy(Step);
797 assert(isPowerOf2_64(VF.getKnownMinValue()) && "must pass power-of-2 VF");
798 if (VF.isScalable() && isPowerOf2_64(Step)) {
799 return B.CreateShl(
800 B.CreateVScale(Ty),
801 ConstantInt::get(Ty, Log2_64(VFxStep.getKnownMinValue())), "", true);
802 }
803 return B.CreateElementCount(Ty, VFxStep);
804}
805
806/// Return the runtime value for VF.
808 return B.CreateElementCount(Ty, VF);
809}
810
812 const StringRef OREMsg, const StringRef ORETag,
813 OptimizationRemarkEmitter *ORE, Loop *TheLoop,
814 Instruction *I) {
815 LLVM_DEBUG(debugVectorizationMessage("Not vectorizing: ", DebugMsg, I));
816 LoopVectorizeHints Hints(TheLoop, true /* doesn't matter */, *ORE);
817 ORE->emit(
818 createLVAnalysis(Hints.vectorizeAnalysisPassName(), ORETag, TheLoop, I)
819 << "loop not vectorized: " << OREMsg);
820}
821
822/// Reports an informative message: print \p Msg for debugging purposes as well
823/// as an optimization remark. Uses either \p I as location of the remark, or
824/// otherwise \p TheLoop. If \p DL is passed, use it as debug location for the
825/// remark. If \p DL is passed, use it as debug location for the remark.
826static void reportVectorizationInfo(const StringRef Msg, const StringRef ORETag,
828 Loop *TheLoop, Instruction *I = nullptr,
829 DebugLoc DL = {}) {
831 LoopVectorizeHints Hints(TheLoop, true /* doesn't matter */, *ORE);
832 ORE->emit(createLVAnalysis(Hints.vectorizeAnalysisPassName(), ORETag, TheLoop,
833 I, DL)
834 << Msg);
835}
836
837/// Report successful vectorization of the loop. In case an outer loop is
838/// vectorized, prepend "outer" to the vectorization remark.
840 VectorizationFactor VF, unsigned IC) {
842 "Vectorizing: ", TheLoop->isInnermost() ? "innermost loop" : "outer loop",
843 nullptr));
844 StringRef LoopType = TheLoop->isInnermost() ? "" : "outer ";
845 ORE->emit([&]() {
846 return OptimizationRemark(LV_NAME, "Vectorized", TheLoop->getStartLoc(),
847 TheLoop->getHeader())
848 << "vectorized " << LoopType << "loop (vectorization width: "
849 << ore::NV("VectorizationFactor", VF.Width)
850 << ", interleaved count: " << ore::NV("InterleaveCount", IC) << ")";
851 });
852}
853
854} // end namespace llvm
855
856namespace llvm {
857
858// Loop vectorization cost-model hints how the scalar epilogue loop should be
859// lowered.
861
862 // The default: allowing scalar epilogues.
864
865 // Vectorization with OptForSize: don't allow epilogues.
867
868 // A special case of vectorisation with OptForSize: loops with a very small
869 // trip count are considered for vectorization under OptForSize, thereby
870 // making sure the cost of their loop body is dominant, free of runtime
871 // guards and scalar iteration overheads.
873
874 // Loop hint predicate indicating an epilogue is undesired.
876
877 // Directive indicating we must either tail fold or not vectorize
879};
880
881/// LoopVectorizationCostModel - estimates the expected speedups due to
882/// vectorization.
883/// In many cases vectorization is not profitable. This can happen because of
884/// a number of reasons. In this class we mainly attempt to predict the
885/// expected speedup/slowdowns due to the supported instruction set. We use the
886/// TargetTransformInfo to query the different backends for the cost of
887/// different operations.
890
891public:
902 : ScalarEpilogueStatus(SEL), TheLoop(L), PSE(PSE), LI(LI), Legal(Legal),
903 TTI(TTI), TLI(TLI), DB(DB), AC(AC), ORE(ORE), TheFunction(F),
904 Hints(Hints), InterleaveInfo(IAI) {
905 if (TTI.supportsScalableVectors() || ForceTargetSupportsScalableVectors)
906 initializeVScaleForTuning();
908 // Query this against the original loop and save it here because the profile
909 // of the original loop header may change as the transformation happens.
910 OptForSize = llvm::shouldOptimizeForSize(L->getHeader(), PSI, BFI,
912 }
913
914 /// \return An upper bound for the vectorization factors (both fixed and
915 /// scalable). If the factors are 0, vectorization and interleaving should be
916 /// avoided up front.
917 FixedScalableVFPair computeMaxVF(ElementCount UserVF, unsigned UserIC);
918
919 /// \return True if runtime checks are required for vectorization, and false
920 /// otherwise.
922
923 /// Setup cost-based decisions for user vectorization factor.
924 /// \return true if the UserVF is a feasible VF to be chosen.
929
930 /// \return True if maximizing vector bandwidth is enabled by the target or
931 /// user options, for the given register kind.
933
934 /// \return True if register pressure should be considered for the given VF.
936
937 /// \return The size (in bits) of the smallest and widest types in the code
938 /// that needs to be vectorized. We ignore values that remain scalar such as
939 /// 64 bit loop indices.
940 std::pair<unsigned, unsigned> getSmallestAndWidestTypes();
941
942 /// Memory access instruction may be vectorized in more than one way.
943 /// Form of instruction after vectorization depends on cost.
944 /// This function takes cost-based decisions for Load/Store instructions
945 /// and collects them in a map. This decisions map is used for building
946 /// the lists of loop-uniform and loop-scalar instructions.
947 /// The calculated cost is saved with widening decision in order to
948 /// avoid redundant calculations.
950
951 /// A call may be vectorized in different ways depending on whether we have
952 /// vectorized variants available and whether the target supports masking.
953 /// This function analyzes all calls in the function at the supplied VF,
954 /// makes a decision based on the costs of available options, and stores that
955 /// decision in a map for use in planning and plan execution.
957
958 /// Collect values we want to ignore in the cost model.
960
961 /// Collect all element types in the loop for which widening is needed.
963
964 /// Split reductions into those that happen in the loop, and those that happen
965 /// outside. In loop reductions are collected into InLoopReductions.
967
968 /// Returns true if we should use strict in-order reductions for the given
969 /// RdxDesc. This is true if the -enable-strict-reductions flag is passed,
970 /// the IsOrdered flag of RdxDesc is set and we do not allow reordering
971 /// of FP operations.
972 bool useOrderedReductions(const RecurrenceDescriptor &RdxDesc) const {
973 return !Hints->allowReordering() && RdxDesc.isOrdered();
974 }
975
976 /// \returns The smallest bitwidth each instruction can be represented with.
977 /// The vector equivalents of these instructions should be truncated to this
978 /// type.
980 return MinBWs;
981 }
982
983 /// \returns True if it is more profitable to scalarize instruction \p I for
984 /// vectorization factor \p VF.
986 assert(VF.isVector() &&
987 "Profitable to scalarize relevant only for VF > 1.");
988 assert(
989 TheLoop->isInnermost() &&
990 "cost-model should not be used for outer loops (in VPlan-native path)");
991
992 auto Scalars = InstsToScalarize.find(VF);
993 assert(Scalars != InstsToScalarize.end() &&
994 "VF not yet analyzed for scalarization profitability");
995 return Scalars->second.contains(I);
996 }
997
998 /// Returns true if \p I is known to be uniform after vectorization.
1000 assert(
1001 TheLoop->isInnermost() &&
1002 "cost-model should not be used for outer loops (in VPlan-native path)");
1003 // Pseudo probe needs to be duplicated for each unrolled iteration and
1004 // vector lane so that profiled loop trip count can be accurately
1005 // accumulated instead of being under counted.
1007 return false;
1008
1009 if (VF.isScalar())
1010 return true;
1011
1012 auto UniformsPerVF = Uniforms.find(VF);
1013 assert(UniformsPerVF != Uniforms.end() &&
1014 "VF not yet analyzed for uniformity");
1015 return UniformsPerVF->second.count(I);
1016 }
1017
1018 /// Returns true if \p I is known to be scalar after vectorization.
1020 assert(
1021 TheLoop->isInnermost() &&
1022 "cost-model should not be used for outer loops (in VPlan-native path)");
1023 if (VF.isScalar())
1024 return true;
1025
1026 auto ScalarsPerVF = Scalars.find(VF);
1027 assert(ScalarsPerVF != Scalars.end() &&
1028 "Scalar values are not calculated for VF");
1029 return ScalarsPerVF->second.count(I);
1030 }
1031
1032 /// \returns True if instruction \p I can be truncated to a smaller bitwidth
1033 /// for vectorization factor \p VF.
1035 return VF.isVector() && MinBWs.contains(I) &&
1036 !isProfitableToScalarize(I, VF) &&
1038 }
1039
1040 /// Decision that was taken during cost calculation for memory instruction.
1043 CM_Widen, // For consecutive accesses with stride +1.
1044 CM_Widen_Reverse, // For consecutive accesses with stride -1.
1050 };
1051
1052 /// Save vectorization decision \p W and \p Cost taken by the cost model for
1053 /// instruction \p I and vector width \p VF.
1056 assert(VF.isVector() && "Expected VF >=2");
1057 WideningDecisions[{I, VF}] = {W, Cost};
1058 }
1059
1060 /// Save vectorization decision \p W and \p Cost taken by the cost model for
1061 /// interleaving group \p Grp and vector width \p VF.
1065 assert(VF.isVector() && "Expected VF >=2");
1066 /// Broadcast this decicion to all instructions inside the group.
1067 /// When interleaving, the cost will only be assigned one instruction, the
1068 /// insert position. For other cases, add the appropriate fraction of the
1069 /// total cost to each instruction. This ensures accurate costs are used,
1070 /// even if the insert position instruction is not used.
1071 InstructionCost InsertPosCost = Cost;
1072 InstructionCost OtherMemberCost = 0;
1073 if (W != CM_Interleave)
1074 OtherMemberCost = InsertPosCost = Cost / Grp->getNumMembers();
1075 ;
1076 for (unsigned Idx = 0; Idx < Grp->getFactor(); ++Idx) {
1077 if (auto *I = Grp->getMember(Idx)) {
1078 if (Grp->getInsertPos() == I)
1079 WideningDecisions[{I, VF}] = {W, InsertPosCost};
1080 else
1081 WideningDecisions[{I, VF}] = {W, OtherMemberCost};
1082 }
1083 }
1084 }
1085
1086 /// Return the cost model decision for the given instruction \p I and vector
1087 /// width \p VF. Return CM_Unknown if this instruction did not pass
1088 /// through the cost modeling.
1090 assert(VF.isVector() && "Expected VF to be a vector VF");
1091 assert(
1092 TheLoop->isInnermost() &&
1093 "cost-model should not be used for outer loops (in VPlan-native path)");
1094
1095 std::pair<Instruction *, ElementCount> InstOnVF(I, VF);
1096 auto Itr = WideningDecisions.find(InstOnVF);
1097 if (Itr == WideningDecisions.end())
1098 return CM_Unknown;
1099 return Itr->second.first;
1100 }
1101
1102 /// Return the vectorization cost for the given instruction \p I and vector
1103 /// width \p VF.
1105 assert(VF.isVector() && "Expected VF >=2");
1106 std::pair<Instruction *, ElementCount> InstOnVF(I, VF);
1107 assert(WideningDecisions.contains(InstOnVF) &&
1108 "The cost is not calculated");
1109 return WideningDecisions[InstOnVF].second;
1110 }
1111
1119
1121 Function *Variant, Intrinsic::ID IID,
1122 std::optional<unsigned> MaskPos,
1124 assert(!VF.isScalar() && "Expected vector VF");
1125 CallWideningDecisions[{CI, VF}] = {Kind, Variant, IID, MaskPos, Cost};
1126 }
1127
1129 ElementCount VF) const {
1130 assert(!VF.isScalar() && "Expected vector VF");
1131 auto I = CallWideningDecisions.find({CI, VF});
1132 if (I == CallWideningDecisions.end())
1133 return {CM_Unknown, nullptr, Intrinsic::not_intrinsic, std::nullopt, 0};
1134 return I->second;
1135 }
1136
1137 /// Return True if instruction \p I is an optimizable truncate whose operand
1138 /// is an induction variable. Such a truncate will be removed by adding a new
1139 /// induction variable with the destination type.
1141 // If the instruction is not a truncate, return false.
1142 auto *Trunc = dyn_cast<TruncInst>(I);
1143 if (!Trunc)
1144 return false;
1145
1146 // Get the source and destination types of the truncate.
1147 Type *SrcTy = toVectorTy(Trunc->getSrcTy(), VF);
1148 Type *DestTy = toVectorTy(Trunc->getDestTy(), VF);
1149
1150 // If the truncate is free for the given types, return false. Replacing a
1151 // free truncate with an induction variable would add an induction variable
1152 // update instruction to each iteration of the loop. We exclude from this
1153 // check the primary induction variable since it will need an update
1154 // instruction regardless.
1155 Value *Op = Trunc->getOperand(0);
1156 if (Op != Legal->getPrimaryInduction() && TTI.isTruncateFree(SrcTy, DestTy))
1157 return false;
1158
1159 // If the truncated value is not an induction variable, return false.
1160 return Legal->isInductionPhi(Op);
1161 }
1162
1163 /// Collects the instructions to scalarize for each predicated instruction in
1164 /// the loop.
1166
1167 /// Collect values that will not be widened, including Uniforms, Scalars, and
1168 /// Instructions to Scalarize for the given \p VF.
1169 /// The sets depend on CM decision for Load/Store instructions
1170 /// that may be vectorized as interleave, gather-scatter or scalarized.
1171 /// Also make a decision on what to do about call instructions in the loop
1172 /// at that VF -- scalarize, call a known vector routine, or call a
1173 /// vector intrinsic.
1175 // Do the analysis once.
1176 if (VF.isScalar() || Uniforms.contains(VF))
1177 return;
1179 collectLoopUniforms(VF);
1181 collectLoopScalars(VF);
1183 }
1184
1185 /// Returns true if the target machine supports masked store operation
1186 /// for the given \p DataType and kind of access to \p Ptr.
1187 bool isLegalMaskedStore(Type *DataType, Value *Ptr, Align Alignment,
1188 unsigned AddressSpace) const {
1189 return Legal->isConsecutivePtr(DataType, Ptr) &&
1190 TTI.isLegalMaskedStore(DataType, Alignment, AddressSpace);
1191 }
1192
1193 /// Returns true if the target machine supports masked load operation
1194 /// for the given \p DataType and kind of access to \p Ptr.
1195 bool isLegalMaskedLoad(Type *DataType, Value *Ptr, Align Alignment,
1196 unsigned AddressSpace) const {
1197 return Legal->isConsecutivePtr(DataType, Ptr) &&
1198 TTI.isLegalMaskedLoad(DataType, Alignment, AddressSpace);
1199 }
1200
1201 /// Returns true if the target machine can represent \p V as a masked gather
1202 /// or scatter operation.
1204 bool LI = isa<LoadInst>(V);
1205 bool SI = isa<StoreInst>(V);
1206 if (!LI && !SI)
1207 return false;
1208 auto *Ty = getLoadStoreType(V);
1210 if (VF.isVector())
1211 Ty = VectorType::get(Ty, VF);
1212 return (LI && TTI.isLegalMaskedGather(Ty, Align)) ||
1213 (SI && TTI.isLegalMaskedScatter(Ty, Align));
1214 }
1215
1216 /// Returns true if the target machine supports all of the reduction
1217 /// variables found for the given VF.
1219 return (all_of(Legal->getReductionVars(), [&](auto &Reduction) -> bool {
1220 const RecurrenceDescriptor &RdxDesc = Reduction.second;
1221 return TTI.isLegalToVectorizeReduction(RdxDesc, VF);
1222 }));
1223 }
1224
1225 /// Given costs for both strategies, return true if the scalar predication
1226 /// lowering should be used for div/rem. This incorporates an override
1227 /// option so it is not simply a cost comparison.
1229 InstructionCost SafeDivisorCost) const {
1230 switch (ForceSafeDivisor) {
1231 case cl::BOU_UNSET:
1232 return ScalarCost < SafeDivisorCost;
1233 case cl::BOU_TRUE:
1234 return false;
1235 case cl::BOU_FALSE:
1236 return true;
1237 }
1238 llvm_unreachable("impossible case value");
1239 }
1240
1241 /// Returns true if \p I is an instruction which requires predication and
1242 /// for which our chosen predication strategy is scalarization (i.e. we
1243 /// don't have an alternate strategy such as masking available).
1244 /// \p VF is the vectorization factor that will be used to vectorize \p I.
1246
1247 /// Returns true if \p I is an instruction that needs to be predicated
1248 /// at runtime. The result is independent of the predication mechanism.
1249 /// Superset of instructions that return true for isScalarWithPredication.
1250 bool isPredicatedInst(Instruction *I) const;
1251
1252 /// Return the costs for our two available strategies for lowering a
1253 /// div/rem operation which requires speculating at least one lane.
1254 /// First result is for scalarization (will be invalid for scalable
1255 /// vectors); second is for the safe-divisor strategy.
1256 std::pair<InstructionCost, InstructionCost>
1258 ElementCount VF) const;
1259
1260 /// Returns true if \p I is a memory instruction with consecutive memory
1261 /// access that can be widened.
1263
1264 /// Returns true if \p I is a memory instruction in an interleaved-group
1265 /// of memory accesses that can be vectorized with wide vector loads/stores
1266 /// and shuffles.
1268
1269 /// Check if \p Instr belongs to any interleaved access group.
1271 return InterleaveInfo.isInterleaved(Instr);
1272 }
1273
1274 /// Get the interleaved access group that \p Instr belongs to.
1277 return InterleaveInfo.getInterleaveGroup(Instr);
1278 }
1279
1280 /// Returns true if we're required to use a scalar epilogue for at least
1281 /// the final iteration of the original loop.
1282 bool requiresScalarEpilogue(bool IsVectorizing) const {
1283 if (!isScalarEpilogueAllowed()) {
1284 LLVM_DEBUG(dbgs() << "LV: Loop does not require scalar epilogue\n");
1285 return false;
1286 }
1287 // If we might exit from anywhere but the latch and early exit vectorization
1288 // is disabled, we must run the exiting iteration in scalar form.
1289 if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch() &&
1290 !(EnableEarlyExitVectorization && Legal->hasUncountableEarlyExit())) {
1291 LLVM_DEBUG(dbgs() << "LV: Loop requires scalar epilogue: not exiting "
1292 "from latch block\n");
1293 return true;
1294 }
1295 if (IsVectorizing && InterleaveInfo.requiresScalarEpilogue()) {
1296 LLVM_DEBUG(dbgs() << "LV: Loop requires scalar epilogue: "
1297 "interleaved group requires scalar epilogue\n");
1298 return true;
1299 }
1300 LLVM_DEBUG(dbgs() << "LV: Loop does not require scalar epilogue\n");
1301 return false;
1302 }
1303
1304 /// Returns true if a scalar epilogue is not allowed due to optsize or a
1305 /// loop hint annotation.
1307 return ScalarEpilogueStatus == CM_ScalarEpilogueAllowed;
1308 }
1309
1310 /// Returns the TailFoldingStyle that is best for the current loop.
1311 TailFoldingStyle getTailFoldingStyle(bool IVUpdateMayOverflow = true) const {
1312 if (!ChosenTailFoldingStyle)
1314 return IVUpdateMayOverflow ? ChosenTailFoldingStyle->first
1315 : ChosenTailFoldingStyle->second;
1316 }
1317
1318 /// Selects and saves TailFoldingStyle for 2 options - if IV update may
1319 /// overflow or not.
1320 /// \param IsScalableVF true if scalable vector factors enabled.
1321 /// \param UserIC User specific interleave count.
1322 void setTailFoldingStyles(bool IsScalableVF, unsigned UserIC) {
1323 assert(!ChosenTailFoldingStyle && "Tail folding must not be selected yet.");
1324 if (!Legal->canFoldTailByMasking()) {
1325 ChosenTailFoldingStyle = {TailFoldingStyle::None, TailFoldingStyle::None};
1326 return;
1327 }
1328
1329 // Default to TTI preference, but allow command line override.
1330 ChosenTailFoldingStyle = {
1331 TTI.getPreferredTailFoldingStyle(/*IVUpdateMayOverflow=*/true),
1332 TTI.getPreferredTailFoldingStyle(/*IVUpdateMayOverflow=*/false)};
1333 if (ForceTailFoldingStyle.getNumOccurrences())
1334 ChosenTailFoldingStyle = {ForceTailFoldingStyle.getValue(),
1335 ForceTailFoldingStyle.getValue()};
1336
1337 if (ChosenTailFoldingStyle->first != TailFoldingStyle::DataWithEVL &&
1338 ChosenTailFoldingStyle->second != TailFoldingStyle::DataWithEVL)
1339 return;
1340 // Override EVL styles if needed.
1341 // FIXME: Investigate opportunity for fixed vector factor.
1342 bool EVLIsLegal = UserIC <= 1 && IsScalableVF &&
1343 TTI.hasActiveVectorLength() && !EnableVPlanNativePath;
1344 if (EVLIsLegal)
1345 return;
1346 // If for some reason EVL mode is unsupported, fallback to a scalar epilogue
1347 // if it's allowed, or DataWithoutLaneMask otherwise.
1348 if (ScalarEpilogueStatus == CM_ScalarEpilogueAllowed ||
1349 ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUsePredicate)
1350 ChosenTailFoldingStyle = {TailFoldingStyle::None, TailFoldingStyle::None};
1351 else
1352 ChosenTailFoldingStyle = {TailFoldingStyle::DataWithoutLaneMask,
1354
1355 LLVM_DEBUG(
1356 dbgs() << "LV: Preference for VP intrinsics indicated. Will "
1357 "not try to generate VP Intrinsics "
1358 << (UserIC > 1
1359 ? "since interleave count specified is greater than 1.\n"
1360 : "due to non-interleaving reasons.\n"));
1361 }
1362
1363 /// Returns true if all loop blocks should be masked to fold tail loop.
1364 bool foldTailByMasking() const {
1365 // TODO: check if it is possible to check for None style independent of
1366 // IVUpdateMayOverflow flag in getTailFoldingStyle.
1368 }
1369
1370 /// Return maximum safe number of elements to be processed per vector
1371 /// iteration, which do not prevent store-load forwarding and are safe with
1372 /// regard to the memory dependencies. Required for EVL-based VPlans to
1373 /// correctly calculate AVL (application vector length) as min(remaining AVL,
1374 /// MaxSafeElements).
1375 /// TODO: need to consider adjusting cost model to use this value as a
1376 /// vectorization factor for EVL-based vectorization.
1377 std::optional<unsigned> getMaxSafeElements() const { return MaxSafeElements; }
1378
1379 /// Returns true if the instructions in this block requires predication
1380 /// for any reason, e.g. because tail folding now requires a predicate
1381 /// or because the block in the original loop was predicated.
1383 return foldTailByMasking() || Legal->blockNeedsPredication(BB);
1384 }
1385
1386 /// Returns true if VP intrinsics with explicit vector length support should
1387 /// be generated in the tail folded loop.
1391
1392 /// Returns true if the Phi is part of an inloop reduction.
1393 bool isInLoopReduction(PHINode *Phi) const {
1394 return InLoopReductions.contains(Phi);
1395 }
1396
1397 /// Returns true if the predicated reduction select should be used to set the
1398 /// incoming value for the reduction phi.
1400 // Force to use predicated reduction select since the EVL of the
1401 // second-to-last iteration might not be VF*UF.
1402 if (foldTailWithEVL())
1403 return true;
1405 TTI.preferPredicatedReductionSelect();
1406 }
1407
1408 /// Estimate cost of an intrinsic call instruction CI if it were vectorized
1409 /// with factor VF. Return the cost of the instruction, including
1410 /// scalarization overhead if it's needed.
1412
1413 /// Estimate cost of a call instruction CI if it were vectorized with factor
1414 /// VF. Return the cost of the instruction, including scalarization overhead
1415 /// if it's needed.
1417
1418 /// Invalidates decisions already taken by the cost model.
1420 WideningDecisions.clear();
1421 CallWideningDecisions.clear();
1422 Uniforms.clear();
1423 Scalars.clear();
1424 }
1425
1426 /// Returns the expected execution cost. The unit of the cost does
1427 /// not matter because we use the 'cost' units to compare different
1428 /// vector widths. The cost that is returned is *not* normalized by
1429 /// the factor width.
1431
1432 bool hasPredStores() const { return NumPredStores > 0; }
1433
1434 /// Returns true if epilogue vectorization is considered profitable, and
1435 /// false otherwise.
1436 /// \p VF is the vectorization factor chosen for the original loop.
1437 /// \p Multiplier is an aditional scaling factor applied to VF before
1438 /// comparing to EpilogueVectorizationMinVF.
1440 const unsigned IC) const;
1441
1442 /// Returns the execution time cost of an instruction for a given vector
1443 /// width. Vector width of one means scalar.
1445
1446 /// Return the cost of instructions in an inloop reduction pattern, if I is
1447 /// part of that pattern.
1448 std::optional<InstructionCost> getReductionPatternCost(Instruction *I,
1449 ElementCount VF,
1450 Type *VectorTy) const;
1451
1452 /// Returns true if \p Op should be considered invariant and if it is
1453 /// trivially hoistable.
1455
1456 /// Return the value of vscale used for tuning the cost model.
1457 std::optional<unsigned> getVScaleForTuning() const { return VScaleForTuning; }
1458
1459private:
1460 unsigned NumPredStores = 0;
1461
1462 /// Used to store the value of vscale used for tuning the cost model. It is
1463 /// initialized during object construction.
1464 std::optional<unsigned> VScaleForTuning;
1465
1466 /// Initializes the value of vscale used for tuning the cost model. If
1467 /// vscale_range.min == vscale_range.max then return vscale_range.max, else
1468 /// return the value returned by the corresponding TTI method.
1469 void initializeVScaleForTuning() {
1470 const Function *Fn = TheLoop->getHeader()->getParent();
1471 if (Fn->hasFnAttribute(Attribute::VScaleRange)) {
1472 auto Attr = Fn->getFnAttribute(Attribute::VScaleRange);
1473 auto Min = Attr.getVScaleRangeMin();
1474 auto Max = Attr.getVScaleRangeMax();
1475 if (Max && Min == Max) {
1476 VScaleForTuning = Max;
1477 return;
1478 }
1479 }
1480
1481 VScaleForTuning = TTI.getVScaleForTuning();
1482 }
1483
1484 /// \return An upper bound for the vectorization factors for both
1485 /// fixed and scalable vectorization, where the minimum-known number of
1486 /// elements is a power-of-2 larger than zero. If scalable vectorization is
1487 /// disabled or unsupported, then the scalable part will be equal to
1488 /// ElementCount::getScalable(0).
1489 FixedScalableVFPair computeFeasibleMaxVF(unsigned MaxTripCount,
1490 ElementCount UserVF,
1491 bool FoldTailByMasking);
1492
1493 /// If \p VF > MaxTripcount, clamps it to the next lower VF that is <=
1494 /// MaxTripCount.
1495 ElementCount clampVFByMaxTripCount(ElementCount VF, unsigned MaxTripCount,
1496 bool FoldTailByMasking) const;
1497
1498 /// \return the maximized element count based on the targets vector
1499 /// registers and the loop trip-count, but limited to a maximum safe VF.
1500 /// This is a helper function of computeFeasibleMaxVF.
1501 ElementCount getMaximizedVFForTarget(unsigned MaxTripCount,
1502 unsigned SmallestType,
1503 unsigned WidestType,
1504 ElementCount MaxSafeVF,
1505 bool FoldTailByMasking);
1506
1507 /// Checks if scalable vectorization is supported and enabled. Caches the
1508 /// result to avoid repeated debug dumps for repeated queries.
1509 bool isScalableVectorizationAllowed();
1510
1511 /// \return the maximum legal scalable VF, based on the safe max number
1512 /// of elements.
1513 ElementCount getMaxLegalScalableVF(unsigned MaxSafeElements);
1514
1515 /// Calculate vectorization cost of memory instruction \p I.
1516 InstructionCost getMemoryInstructionCost(Instruction *I, ElementCount VF);
1517
1518 /// The cost computation for scalarized memory instruction.
1519 InstructionCost getMemInstScalarizationCost(Instruction *I, ElementCount VF);
1520
1521 /// The cost computation for interleaving group of memory instructions.
1522 InstructionCost getInterleaveGroupCost(Instruction *I, ElementCount VF);
1523
1524 /// The cost computation for Gather/Scatter instruction.
1525 InstructionCost getGatherScatterCost(Instruction *I, ElementCount VF);
1526
1527 /// The cost computation for widening instruction \p I with consecutive
1528 /// memory access.
1529 InstructionCost getConsecutiveMemOpCost(Instruction *I, ElementCount VF);
1530
1531 /// The cost calculation for Load/Store instruction \p I with uniform pointer -
1532 /// Load: scalar load + broadcast.
1533 /// Store: scalar store + (loop invariant value stored? 0 : extract of last
1534 /// element)
1535 InstructionCost getUniformMemOpCost(Instruction *I, ElementCount VF);
1536
1537 /// Estimate the overhead of scalarizing an instruction. This is a
1538 /// convenience wrapper for the type-based getScalarizationOverhead API.
1539 InstructionCost getScalarizationOverhead(Instruction *I,
1540 ElementCount VF) const;
1541
1542 /// Returns true if an artificially high cost for emulated masked memrefs
1543 /// should be used.
1544 bool useEmulatedMaskMemRefHack(Instruction *I, ElementCount VF);
1545
1546 /// Map of scalar integer values to the smallest bitwidth they can be legally
1547 /// represented as. The vector equivalents of these values should be truncated
1548 /// to this type.
1549 MapVector<Instruction *, uint64_t> MinBWs;
1550
1551 /// A type representing the costs for instructions if they were to be
1552 /// scalarized rather than vectorized. The entries are Instruction-Cost
1553 /// pairs.
1554 using ScalarCostsTy = MapVector<Instruction *, InstructionCost>;
1555
1556 /// A set containing all BasicBlocks that are known to present after
1557 /// vectorization as a predicated block.
1558 DenseMap<ElementCount, SmallPtrSet<BasicBlock *, 4>>
1559 PredicatedBBsAfterVectorization;
1560
1561 /// Records whether it is allowed to have the original scalar loop execute at
1562 /// least once. This may be needed as a fallback loop in case runtime
1563 /// aliasing/dependence checks fail, or to handle the tail/remainder
1564 /// iterations when the trip count is unknown or doesn't divide by the VF,
1565 /// or as a peel-loop to handle gaps in interleave-groups.
1566 /// Under optsize and when the trip count is very small we don't allow any
1567 /// iterations to execute in the scalar loop.
1568 ScalarEpilogueLowering ScalarEpilogueStatus = CM_ScalarEpilogueAllowed;
1569
1570 /// Control finally chosen tail folding style. The first element is used if
1571 /// the IV update may overflow, the second element - if it does not.
1572 std::optional<std::pair<TailFoldingStyle, TailFoldingStyle>>
1573 ChosenTailFoldingStyle;
1574
1575 /// true if scalable vectorization is supported and enabled.
1576 std::optional<bool> IsScalableVectorizationAllowed;
1577
1578 /// Maximum safe number of elements to be processed per vector iteration,
1579 /// which do not prevent store-load forwarding and are safe with regard to the
1580 /// memory dependencies. Required for EVL-based veectorization, where this
1581 /// value is used as the upper bound of the safe AVL.
1582 std::optional<unsigned> MaxSafeElements;
1583
1584 /// A map holding scalar costs for different vectorization factors. The
1585 /// presence of a cost for an instruction in the mapping indicates that the
1586 /// instruction will be scalarized when vectorizing with the associated
1587 /// vectorization factor. The entries are VF-ScalarCostTy pairs.
1588 MapVector<ElementCount, ScalarCostsTy> InstsToScalarize;
1589
1590 /// Holds the instructions known to be uniform after vectorization.
1591 /// The data is collected per VF.
1592 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Uniforms;
1593
1594 /// Holds the instructions known to be scalar after vectorization.
1595 /// The data is collected per VF.
1596 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Scalars;
1597
1598 /// Holds the instructions (address computations) that are forced to be
1599 /// scalarized.
1600 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> ForcedScalars;
1601
1602 /// PHINodes of the reductions that should be expanded in-loop.
1603 SmallPtrSet<PHINode *, 4> InLoopReductions;
1604
1605 /// A Map of inloop reduction operations and their immediate chain operand.
1606 /// FIXME: This can be removed once reductions can be costed correctly in
1607 /// VPlan. This was added to allow quick lookup of the inloop operations.
1608 DenseMap<Instruction *, Instruction *> InLoopReductionImmediateChains;
1609
1610 /// Returns the expected difference in cost from scalarizing the expression
1611 /// feeding a predicated instruction \p PredInst. The instructions to
1612 /// scalarize and their scalar costs are collected in \p ScalarCosts. A
1613 /// non-negative return value implies the expression will be scalarized.
1614 /// Currently, only single-use chains are considered for scalarization.
1615 InstructionCost computePredInstDiscount(Instruction *PredInst,
1616 ScalarCostsTy &ScalarCosts,
1617 ElementCount VF);
1618
1619 /// Collect the instructions that are uniform after vectorization. An
1620 /// instruction is uniform if we represent it with a single scalar value in
1621 /// the vectorized loop corresponding to each vector iteration. Examples of
1622 /// uniform instructions include pointer operands of consecutive or
1623 /// interleaved memory accesses. Note that although uniformity implies an
1624 /// instruction will be scalar, the reverse is not true. In general, a
1625 /// scalarized instruction will be represented by VF scalar values in the
1626 /// vectorized loop, each corresponding to an iteration of the original
1627 /// scalar loop.
1628 void collectLoopUniforms(ElementCount VF);
1629
1630 /// Collect the instructions that are scalar after vectorization. An
1631 /// instruction is scalar if it is known to be uniform or will be scalarized
1632 /// during vectorization. collectLoopScalars should only add non-uniform nodes
1633 /// to the list if they are used by a load/store instruction that is marked as
1634 /// CM_Scalarize. Non-uniform scalarized instructions will be represented by
1635 /// VF values in the vectorized loop, each corresponding to an iteration of
1636 /// the original scalar loop.
1637 void collectLoopScalars(ElementCount VF);
1638
1639 /// Keeps cost model vectorization decision and cost for instructions.
1640 /// Right now it is used for memory instructions only.
1641 using DecisionList = DenseMap<std::pair<Instruction *, ElementCount>,
1642 std::pair<InstWidening, InstructionCost>>;
1643
1644 DecisionList WideningDecisions;
1645
1646 using CallDecisionList =
1647 DenseMap<std::pair<CallInst *, ElementCount>, CallWideningDecision>;
1648
1649 CallDecisionList CallWideningDecisions;
1650
1651 /// Returns true if \p V is expected to be vectorized and it needs to be
1652 /// extracted.
1653 bool needsExtract(Value *V, ElementCount VF) const {
1655 if (VF.isScalar() || !I || !TheLoop->contains(I) ||
1656 TheLoop->isLoopInvariant(I) ||
1658 (isa<CallInst>(I) &&
1660 return false;
1661
1662 // Assume we can vectorize V (and hence we need extraction) if the
1663 // scalars are not computed yet. This can happen, because it is called
1664 // via getScalarizationOverhead from setCostBasedWideningDecision, before
1665 // the scalars are collected. That should be a safe assumption in most
1666 // cases, because we check if the operands have vectorizable types
1667 // beforehand in LoopVectorizationLegality.
1668 return !Scalars.contains(VF) || !isScalarAfterVectorization(I, VF);
1669 };
1670
1671 /// Returns a range containing only operands needing to be extracted.
1672 SmallVector<Value *, 4> filterExtractingOperands(Instruction::op_range Ops,
1673 ElementCount VF) const {
1674
1675 SmallPtrSet<const Value *, 4> UniqueOperands;
1677 for (Value *Op : Ops) {
1678 if (isa<Constant>(Op) || !UniqueOperands.insert(Op).second ||
1679 !needsExtract(Op, VF))
1680 continue;
1681 Res.push_back(Op);
1682 }
1683 return Res;
1684 }
1685
1686public:
1687 /// The loop that we evaluate.
1689
1690 /// Predicated scalar evolution analysis.
1692
1693 /// Loop Info analysis.
1695
1696 /// Vectorization legality.
1698
1699 /// Vector target information.
1701
1702 /// Target Library Info.
1704
1705 /// Demanded bits analysis.
1707
1708 /// Assumption cache.
1710
1711 /// Interface to emit optimization remarks.
1713
1715
1716 /// Loop Vectorize Hint.
1718
1719 /// The interleave access information contains groups of interleaved accesses
1720 /// with the same stride and close to each other.
1722
1723 /// Values to ignore in the cost model.
1725
1726 /// Values to ignore in the cost model when VF > 1.
1728
1729 /// All element types found in the loop.
1731
1732 /// The kind of cost that we are calculating
1734
1735 /// Whether this loop should be optimized for size based on function attribute
1736 /// or profile information.
1738
1739 /// The highest VF possible for this loop, without using MaxBandwidth.
1741};
1742} // end namespace llvm
1743
1744namespace {
1745/// Helper struct to manage generating runtime checks for vectorization.
1746///
1747/// The runtime checks are created up-front in temporary blocks to allow better
1748/// estimating the cost and un-linked from the existing IR. After deciding to
1749/// vectorize, the checks are moved back. If deciding not to vectorize, the
1750/// temporary blocks are completely removed.
1751class GeneratedRTChecks {
1752 /// Basic block which contains the generated SCEV checks, if any.
1753 BasicBlock *SCEVCheckBlock = nullptr;
1754
1755 /// The value representing the result of the generated SCEV checks. If it is
1756 /// nullptr no SCEV checks have been generated.
1757 Value *SCEVCheckCond = nullptr;
1758
1759 /// Basic block which contains the generated memory runtime checks, if any.
1760 BasicBlock *MemCheckBlock = nullptr;
1761
1762 /// The value representing the result of the generated memory runtime checks.
1763 /// If it is nullptr no memory runtime checks have been generated.
1764 Value *MemRuntimeCheckCond = nullptr;
1765
1766 DominatorTree *DT;
1767 LoopInfo *LI;
1769
1770 SCEVExpander SCEVExp;
1771 SCEVExpander MemCheckExp;
1772
1773 bool CostTooHigh = false;
1774
1775 Loop *OuterLoop = nullptr;
1776
1778
1779 /// The kind of cost that we are calculating
1781
1782public:
1783 GeneratedRTChecks(PredicatedScalarEvolution &PSE, DominatorTree *DT,
1786 : DT(DT), LI(LI), TTI(TTI), SCEVExp(*PSE.getSE(), DL, "scev.check"),
1787 MemCheckExp(*PSE.getSE(), DL, "scev.check"), PSE(PSE),
1788 CostKind(CostKind) {}
1789
1790 /// Generate runtime checks in SCEVCheckBlock and MemCheckBlock, so we can
1791 /// accurately estimate the cost of the runtime checks. The blocks are
1792 /// un-linked from the IR and are added back during vector code generation. If
1793 /// there is no vector code generation, the check blocks are removed
1794 /// completely.
1795 void create(Loop *L, const LoopAccessInfo &LAI,
1796 const SCEVPredicate &UnionPred, ElementCount VF, unsigned IC) {
1797
1798 // Hard cutoff to limit compile-time increase in case a very large number of
1799 // runtime checks needs to be generated.
1800 // TODO: Skip cutoff if the loop is guaranteed to execute, e.g. due to
1801 // profile info.
1802 CostTooHigh =
1804 if (CostTooHigh)
1805 return;
1806
1807 BasicBlock *LoopHeader = L->getHeader();
1808 BasicBlock *Preheader = L->getLoopPreheader();
1809
1810 // Use SplitBlock to create blocks for SCEV & memory runtime checks to
1811 // ensure the blocks are properly added to LoopInfo & DominatorTree. Those
1812 // may be used by SCEVExpander. The blocks will be un-linked from their
1813 // predecessors and removed from LI & DT at the end of the function.
1814 if (!UnionPred.isAlwaysTrue()) {
1815 SCEVCheckBlock = SplitBlock(Preheader, Preheader->getTerminator(), DT, LI,
1816 nullptr, "vector.scevcheck");
1817
1818 SCEVCheckCond = SCEVExp.expandCodeForPredicate(
1819 &UnionPred, SCEVCheckBlock->getTerminator());
1820 if (isa<Constant>(SCEVCheckCond)) {
1821 // Clean up directly after expanding the predicate to a constant, to
1822 // avoid further expansions re-using anything left over from SCEVExp.
1823 SCEVExpanderCleaner SCEVCleaner(SCEVExp);
1824 SCEVCleaner.cleanup();
1825 }
1826 }
1827
1828 const auto &RtPtrChecking = *LAI.getRuntimePointerChecking();
1829 if (RtPtrChecking.Need) {
1830 auto *Pred = SCEVCheckBlock ? SCEVCheckBlock : Preheader;
1831 MemCheckBlock = SplitBlock(Pred, Pred->getTerminator(), DT, LI, nullptr,
1832 "vector.memcheck");
1833
1834 auto DiffChecks = RtPtrChecking.getDiffChecks();
1835 if (DiffChecks) {
1836 Value *RuntimeVF = nullptr;
1837 MemRuntimeCheckCond = addDiffRuntimeChecks(
1838 MemCheckBlock->getTerminator(), *DiffChecks, MemCheckExp,
1839 [VF, &RuntimeVF](IRBuilderBase &B, unsigned Bits) {
1840 if (!RuntimeVF)
1841 RuntimeVF = getRuntimeVF(B, B.getIntNTy(Bits), VF);
1842 return RuntimeVF;
1843 },
1844 IC);
1845 } else {
1846 MemRuntimeCheckCond = addRuntimeChecks(
1847 MemCheckBlock->getTerminator(), L, RtPtrChecking.getChecks(),
1849 }
1850 assert(MemRuntimeCheckCond &&
1851 "no RT checks generated although RtPtrChecking "
1852 "claimed checks are required");
1853 }
1854
1855 SCEVExp.eraseDeadInstructions(SCEVCheckCond);
1856
1857 if (!MemCheckBlock && !SCEVCheckBlock)
1858 return;
1859
1860 // Unhook the temporary block with the checks, update various places
1861 // accordingly.
1862 if (SCEVCheckBlock)
1863 SCEVCheckBlock->replaceAllUsesWith(Preheader);
1864 if (MemCheckBlock)
1865 MemCheckBlock->replaceAllUsesWith(Preheader);
1866
1867 if (SCEVCheckBlock) {
1868 SCEVCheckBlock->getTerminator()->moveBefore(
1869 Preheader->getTerminator()->getIterator());
1870 auto *UI = new UnreachableInst(Preheader->getContext(), SCEVCheckBlock);
1871 UI->setDebugLoc(DebugLoc::getTemporary());
1872 Preheader->getTerminator()->eraseFromParent();
1873 }
1874 if (MemCheckBlock) {
1875 MemCheckBlock->getTerminator()->moveBefore(
1876 Preheader->getTerminator()->getIterator());
1877 auto *UI = new UnreachableInst(Preheader->getContext(), MemCheckBlock);
1878 UI->setDebugLoc(DebugLoc::getTemporary());
1879 Preheader->getTerminator()->eraseFromParent();
1880 }
1881
1882 DT->changeImmediateDominator(LoopHeader, Preheader);
1883 if (MemCheckBlock) {
1884 DT->eraseNode(MemCheckBlock);
1885 LI->removeBlock(MemCheckBlock);
1886 }
1887 if (SCEVCheckBlock) {
1888 DT->eraseNode(SCEVCheckBlock);
1889 LI->removeBlock(SCEVCheckBlock);
1890 }
1891
1892 // Outer loop is used as part of the later cost calculations.
1893 OuterLoop = L->getParentLoop();
1894 }
1895
1897 if (SCEVCheckBlock || MemCheckBlock)
1898 LLVM_DEBUG(dbgs() << "Calculating cost of runtime checks:\n");
1899
1900 if (CostTooHigh) {
1902 Cost.setInvalid();
1903 LLVM_DEBUG(dbgs() << " number of checks exceeded threshold\n");
1904 return Cost;
1905 }
1906
1907 InstructionCost RTCheckCost = 0;
1908 if (SCEVCheckBlock)
1909 for (Instruction &I : *SCEVCheckBlock) {
1910 if (SCEVCheckBlock->getTerminator() == &I)
1911 continue;
1913 LLVM_DEBUG(dbgs() << " " << C << " for " << I << "\n");
1914 RTCheckCost += C;
1915 }
1916 if (MemCheckBlock) {
1917 InstructionCost MemCheckCost = 0;
1918 for (Instruction &I : *MemCheckBlock) {
1919 if (MemCheckBlock->getTerminator() == &I)
1920 continue;
1922 LLVM_DEBUG(dbgs() << " " << C << " for " << I << "\n");
1923 MemCheckCost += C;
1924 }
1925
1926 // If the runtime memory checks are being created inside an outer loop
1927 // we should find out if these checks are outer loop invariant. If so,
1928 // the checks will likely be hoisted out and so the effective cost will
1929 // reduce according to the outer loop trip count.
1930 if (OuterLoop) {
1931 ScalarEvolution *SE = MemCheckExp.getSE();
1932 // TODO: If profitable, we could refine this further by analysing every
1933 // individual memory check, since there could be a mixture of loop
1934 // variant and invariant checks that mean the final condition is
1935 // variant.
1936 const SCEV *Cond = SE->getSCEV(MemRuntimeCheckCond);
1937 if (SE->isLoopInvariant(Cond, OuterLoop)) {
1938 // It seems reasonable to assume that we can reduce the effective
1939 // cost of the checks even when we know nothing about the trip
1940 // count. Assume that the outer loop executes at least twice.
1941 unsigned BestTripCount = 2;
1942
1943 // Get the best known TC estimate.
1944 if (auto EstimatedTC = getSmallBestKnownTC(
1945 PSE, OuterLoop, /* CanUseConstantMax = */ false))
1946 if (EstimatedTC->isFixed())
1947 BestTripCount = EstimatedTC->getFixedValue();
1948
1949 InstructionCost NewMemCheckCost = MemCheckCost / BestTripCount;
1950
1951 // Let's ensure the cost is always at least 1.
1952 NewMemCheckCost = std::max(NewMemCheckCost.getValue(),
1953 (InstructionCost::CostType)1);
1954
1955 if (BestTripCount > 1)
1957 << "We expect runtime memory checks to be hoisted "
1958 << "out of the outer loop. Cost reduced from "
1959 << MemCheckCost << " to " << NewMemCheckCost << '\n');
1960
1961 MemCheckCost = NewMemCheckCost;
1962 }
1963 }
1964
1965 RTCheckCost += MemCheckCost;
1966 }
1967
1968 if (SCEVCheckBlock || MemCheckBlock)
1969 LLVM_DEBUG(dbgs() << "Total cost of runtime checks: " << RTCheckCost
1970 << "\n");
1971
1972 return RTCheckCost;
1973 }
1974
1975 /// Remove the created SCEV & memory runtime check blocks & instructions, if
1976 /// unused.
1977 ~GeneratedRTChecks() {
1978 SCEVExpanderCleaner SCEVCleaner(SCEVExp);
1979 SCEVExpanderCleaner MemCheckCleaner(MemCheckExp);
1980 bool SCEVChecksUsed = !SCEVCheckBlock || !pred_empty(SCEVCheckBlock);
1981 bool MemChecksUsed = !MemCheckBlock || !pred_empty(MemCheckBlock);
1982 if (SCEVChecksUsed)
1983 SCEVCleaner.markResultUsed();
1984
1985 if (MemChecksUsed) {
1986 MemCheckCleaner.markResultUsed();
1987 } else {
1988 auto &SE = *MemCheckExp.getSE();
1989 // Memory runtime check generation creates compares that use expanded
1990 // values. Remove them before running the SCEVExpanderCleaners.
1991 for (auto &I : make_early_inc_range(reverse(*MemCheckBlock))) {
1992 if (MemCheckExp.isInsertedInstruction(&I))
1993 continue;
1994 SE.forgetValue(&I);
1995 I.eraseFromParent();
1996 }
1997 }
1998 MemCheckCleaner.cleanup();
1999 SCEVCleaner.cleanup();
2000
2001 if (!SCEVChecksUsed)
2002 SCEVCheckBlock->eraseFromParent();
2003 if (!MemChecksUsed)
2004 MemCheckBlock->eraseFromParent();
2005 }
2006
2007 /// Retrieves the SCEVCheckCond and SCEVCheckBlock that were generated as IR
2008 /// outside VPlan.
2009 std::pair<Value *, BasicBlock *> getSCEVChecks() const {
2010 using namespace llvm::PatternMatch;
2011 if (!SCEVCheckCond || match(SCEVCheckCond, m_ZeroInt()))
2012 return {nullptr, nullptr};
2013
2014 return {SCEVCheckCond, SCEVCheckBlock};
2015 }
2016
2017 /// Retrieves the MemCheckCond and MemCheckBlock that were generated as IR
2018 /// outside VPlan.
2019 std::pair<Value *, BasicBlock *> getMemRuntimeChecks() const {
2020 using namespace llvm::PatternMatch;
2021 if (MemRuntimeCheckCond && match(MemRuntimeCheckCond, m_ZeroInt()))
2022 return {nullptr, nullptr};
2023 return {MemRuntimeCheckCond, MemCheckBlock};
2024 }
2025
2026 /// Return true if any runtime checks have been added
2027 bool hasChecks() const {
2028 return getSCEVChecks().first || getMemRuntimeChecks().first;
2029 }
2030};
2031} // namespace
2032
2038
2043
2044// Return true if \p OuterLp is an outer loop annotated with hints for explicit
2045// vectorization. The loop needs to be annotated with #pragma omp simd
2046// simdlen(#) or #pragma clang vectorize(enable) vectorize_width(#). If the
2047// vector length information is not provided, vectorization is not considered
2048// explicit. Interleave hints are not allowed either. These limitations will be
2049// relaxed in the future.
2050// Please, note that we are currently forced to abuse the pragma 'clang
2051// vectorize' semantics. This pragma provides *auto-vectorization hints*
2052// (i.e., LV must check that vectorization is legal) whereas pragma 'omp simd'
2053// provides *explicit vectorization hints* (LV can bypass legal checks and
2054// assume that vectorization is legal). However, both hints are implemented
2055// using the same metadata (llvm.loop.vectorize, processed by
2056// LoopVectorizeHints). This will be fixed in the future when the native IR
2057// representation for pragma 'omp simd' is introduced.
2058static bool isExplicitVecOuterLoop(Loop *OuterLp,
2060 assert(!OuterLp->isInnermost() && "This is not an outer loop");
2061 LoopVectorizeHints Hints(OuterLp, true /*DisableInterleaving*/, *ORE);
2062
2063 // Only outer loops with an explicit vectorization hint are supported.
2064 // Unannotated outer loops are ignored.
2066 return false;
2067
2068 Function *Fn = OuterLp->getHeader()->getParent();
2069 if (!Hints.allowVectorization(Fn, OuterLp,
2070 true /*VectorizeOnlyWhenForced*/)) {
2071 LLVM_DEBUG(dbgs() << "LV: Loop hints prevent outer loop vectorization.\n");
2072 return false;
2073 }
2074
2075 if (Hints.getInterleave() > 1) {
2076 // TODO: Interleave support is future work.
2077 LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Interleave is not supported for "
2078 "outer loops.\n");
2079 Hints.emitRemarkWithHints();
2080 return false;
2081 }
2082
2083 return true;
2084}
2085
2089 // Collect inner loops and outer loops without irreducible control flow. For
2090 // now, only collect outer loops that have explicit vectorization hints. If we
2091 // are stress testing the VPlan H-CFG construction, we collect the outermost
2092 // loop of every loop nest.
2093 if (L.isInnermost() || VPlanBuildStressTest ||
2095 LoopBlocksRPO RPOT(&L);
2096 RPOT.perform(LI);
2098 V.push_back(&L);
2099 // TODO: Collect inner loops inside marked outer loops in case
2100 // vectorization fails for the outer loop. Do not invoke
2101 // 'containsIrreducibleCFG' again for inner loops when the outer loop is
2102 // already known to be reducible. We can use an inherited attribute for
2103 // that.
2104 return;
2105 }
2106 }
2107 for (Loop *InnerL : L)
2108 collectSupportedLoops(*InnerL, LI, ORE, V);
2109}
2110
2111//===----------------------------------------------------------------------===//
2112// Implementation of LoopVectorizationLegality, InnerLoopVectorizer and
2113// LoopVectorizationCostModel and LoopVectorizationPlanner.
2114//===----------------------------------------------------------------------===//
2115
2116/// Compute the transformed value of Index at offset StartValue using step
2117/// StepValue.
2118/// For integer induction, returns StartValue + Index * StepValue.
2119/// For pointer induction, returns StartValue[Index * StepValue].
2120/// FIXME: The newly created binary instructions should contain nsw/nuw
2121/// flags, which can be found from the original scalar operations.
2122static Value *
2124 Value *Step,
2126 const BinaryOperator *InductionBinOp) {
2127 using namespace llvm::PatternMatch;
2128 Type *StepTy = Step->getType();
2129 Value *CastedIndex = StepTy->isIntegerTy()
2130 ? B.CreateSExtOrTrunc(Index, StepTy)
2131 : B.CreateCast(Instruction::SIToFP, Index, StepTy);
2132 if (CastedIndex != Index) {
2133 CastedIndex->setName(CastedIndex->getName() + ".cast");
2134 Index = CastedIndex;
2135 }
2136
2137 // Note: the IR at this point is broken. We cannot use SE to create any new
2138 // SCEV and then expand it, hoping that SCEV's simplification will give us
2139 // a more optimal code. Unfortunately, attempt of doing so on invalid IR may
2140 // lead to various SCEV crashes. So all we can do is to use builder and rely
2141 // on InstCombine for future simplifications. Here we handle some trivial
2142 // cases only.
2143 auto CreateAdd = [&B](Value *X, Value *Y) {
2144 assert(X->getType() == Y->getType() && "Types don't match!");
2145 if (match(X, m_ZeroInt()))
2146 return Y;
2147 if (match(Y, m_ZeroInt()))
2148 return X;
2149 return B.CreateAdd(X, Y);
2150 };
2151
2152 // We allow X to be a vector type, in which case Y will potentially be
2153 // splatted into a vector with the same element count.
2154 auto CreateMul = [&B](Value *X, Value *Y) {
2155 assert(X->getType()->getScalarType() == Y->getType() &&
2156 "Types don't match!");
2157 if (match(X, m_One()))
2158 return Y;
2159 if (match(Y, m_One()))
2160 return X;
2161 VectorType *XVTy = dyn_cast<VectorType>(X->getType());
2162 if (XVTy && !isa<VectorType>(Y->getType()))
2163 Y = B.CreateVectorSplat(XVTy->getElementCount(), Y);
2164 return B.CreateMul(X, Y);
2165 };
2166
2167 switch (InductionKind) {
2169 assert(!isa<VectorType>(Index->getType()) &&
2170 "Vector indices not supported for integer inductions yet");
2171 assert(Index->getType() == StartValue->getType() &&
2172 "Index type does not match StartValue type");
2173 if (isa<ConstantInt>(Step) && cast<ConstantInt>(Step)->isMinusOne())
2174 return B.CreateSub(StartValue, Index);
2175 auto *Offset = CreateMul(Index, Step);
2176 return CreateAdd(StartValue, Offset);
2177 }
2179 return B.CreatePtrAdd(StartValue, CreateMul(Index, Step));
2181 assert(!isa<VectorType>(Index->getType()) &&
2182 "Vector indices not supported for FP inductions yet");
2183 assert(Step->getType()->isFloatingPointTy() && "Expected FP Step value");
2184 assert(InductionBinOp &&
2185 (InductionBinOp->getOpcode() == Instruction::FAdd ||
2186 InductionBinOp->getOpcode() == Instruction::FSub) &&
2187 "Original bin op should be defined for FP induction");
2188
2189 Value *MulExp = B.CreateFMul(Step, Index);
2190 return B.CreateBinOp(InductionBinOp->getOpcode(), StartValue, MulExp,
2191 "induction");
2192 }
2194 return nullptr;
2195 }
2196 llvm_unreachable("invalid enum");
2197}
2198
2199static std::optional<unsigned> getMaxVScale(const Function &F,
2200 const TargetTransformInfo &TTI) {
2201 if (std::optional<unsigned> MaxVScale = TTI.getMaxVScale())
2202 return MaxVScale;
2203
2204 if (F.hasFnAttribute(Attribute::VScaleRange))
2205 return F.getFnAttribute(Attribute::VScaleRange).getVScaleRangeMax();
2206
2207 return std::nullopt;
2208}
2209
2210/// For the given VF and UF and maximum trip count computed for the loop, return
2211/// whether the induction variable might overflow in the vectorized loop. If not,
2212/// then we know a runtime overflow check always evaluates to false and can be
2213/// removed.
2215 const LoopVectorizationCostModel *Cost,
2216 ElementCount VF, std::optional<unsigned> UF = std::nullopt) {
2217 // Always be conservative if we don't know the exact unroll factor.
2218 unsigned MaxUF = UF ? *UF : Cost->TTI.getMaxInterleaveFactor(VF);
2219
2220 IntegerType *IdxTy = Cost->Legal->getWidestInductionType();
2221 APInt MaxUIntTripCount = IdxTy->getMask();
2222
2223 // We know the runtime overflow check is known false iff the (max) trip-count
2224 // is known and (max) trip-count + (VF * UF) does not overflow in the type of
2225 // the vector loop induction variable.
2226 if (unsigned TC = Cost->PSE.getSmallConstantMaxTripCount()) {
2227 uint64_t MaxVF = VF.getKnownMinValue();
2228 if (VF.isScalable()) {
2229 std::optional<unsigned> MaxVScale =
2230 getMaxVScale(*Cost->TheFunction, Cost->TTI);
2231 if (!MaxVScale)
2232 return false;
2233 MaxVF *= *MaxVScale;
2234 }
2235
2236 return (MaxUIntTripCount - TC).ugt(MaxVF * MaxUF);
2237 }
2238
2239 return false;
2240}
2241
2242// Return whether we allow using masked interleave-groups (for dealing with
2243// strided loads/stores that reside in predicated blocks, or for dealing
2244// with gaps).
2246 // If an override option has been passed in for interleaved accesses, use it.
2247 if (EnableMaskedInterleavedMemAccesses.getNumOccurrences() > 0)
2249
2250 return TTI.enableMaskedInterleavedAccessVectorization();
2251}
2252
2254 BasicBlock *CheckIRBB) {
2255 // Note: The block with the minimum trip-count check is already connected
2256 // during earlier VPlan construction.
2257 VPBlockBase *ScalarPH = Plan.getScalarPreheader();
2258 VPBlockBase *PreVectorPH = VectorPHVPBB->getSinglePredecessor();
2259 assert(PreVectorPH->getNumSuccessors() == 2 && "Expected 2 successors");
2260 assert(PreVectorPH->getSuccessors()[0] == ScalarPH && "Unexpected successor");
2261 VPIRBasicBlock *CheckVPIRBB = Plan.createVPIRBasicBlock(CheckIRBB);
2262 VPBlockUtils::insertOnEdge(PreVectorPH, VectorPHVPBB, CheckVPIRBB);
2263 PreVectorPH = CheckVPIRBB;
2264 VPBlockUtils::connectBlocks(PreVectorPH, ScalarPH);
2265 PreVectorPH->swapSuccessors();
2266
2267 // We just connected a new block to the scalar preheader. Update all
2268 // VPPhis by adding an incoming value for it, replicating the last value.
2269 unsigned NumPredecessors = ScalarPH->getNumPredecessors();
2270 for (VPRecipeBase &R : cast<VPBasicBlock>(ScalarPH)->phis()) {
2271 assert(isa<VPPhi>(&R) && "Phi expected to be VPPhi");
2272 assert(cast<VPPhi>(&R)->getNumIncoming() == NumPredecessors - 1 &&
2273 "must have incoming values for all operands");
2274 R.addOperand(R.getOperand(NumPredecessors - 2));
2275 }
2276}
2277
2279 BasicBlock *VectorPH, ElementCount VF, unsigned UF) const {
2280 // Generate code to check if the loop's trip count is less than VF * UF, or
2281 // equal to it in case a scalar epilogue is required; this implies that the
2282 // vector trip count is zero. This check also covers the case where adding one
2283 // to the backedge-taken count overflowed leading to an incorrect trip count
2284 // of zero. In this case we will also jump to the scalar loop.
2285 auto P = Cost->requiresScalarEpilogue(VF.isVector()) ? ICmpInst::ICMP_ULE
2287
2288 // Reuse existing vector loop preheader for TC checks.
2289 // Note that new preheader block is generated for vector loop.
2290 BasicBlock *const TCCheckBlock = VectorPH;
2292 TCCheckBlock->getContext(),
2293 InstSimplifyFolder(TCCheckBlock->getDataLayout()));
2294 Builder.SetInsertPoint(TCCheckBlock->getTerminator());
2295
2296 // If tail is to be folded, vector loop takes care of all iterations.
2298 Type *CountTy = Count->getType();
2299 Value *CheckMinIters = Builder.getFalse();
2300 auto CreateStep = [&]() -> Value * {
2301 // Create step with max(MinProTripCount, UF * VF).
2302 if (UF * VF.getKnownMinValue() >= MinProfitableTripCount.getKnownMinValue())
2303 return createStepForVF(Builder, CountTy, VF, UF);
2304
2305 Value *MinProfTC =
2306 Builder.CreateElementCount(CountTy, MinProfitableTripCount);
2307 if (!VF.isScalable())
2308 return MinProfTC;
2309 return Builder.CreateBinaryIntrinsic(
2310 Intrinsic::umax, MinProfTC, createStepForVF(Builder, CountTy, VF, UF));
2311 };
2312
2313 TailFoldingStyle Style = Cost->getTailFoldingStyle();
2314 if (Style == TailFoldingStyle::None) {
2315 Value *Step = CreateStep();
2316 ScalarEvolution &SE = *PSE.getSE();
2317 // TODO: Emit unconditional branch to vector preheader instead of
2318 // conditional branch with known condition.
2319 const SCEV *TripCountSCEV = SE.applyLoopGuards(SE.getSCEV(Count), OrigLoop);
2320 // Check if the trip count is < the step.
2321 if (SE.isKnownPredicate(P, TripCountSCEV, SE.getSCEV(Step))) {
2322 // TODO: Ensure step is at most the trip count when determining max VF and
2323 // UF, w/o tail folding.
2324 CheckMinIters = Builder.getTrue();
2326 TripCountSCEV, SE.getSCEV(Step))) {
2327 // Generate the minimum iteration check only if we cannot prove the
2328 // check is known to be true, or known to be false.
2329 CheckMinIters = Builder.CreateICmp(P, Count, Step, "min.iters.check");
2330 } // else step known to be < trip count, use CheckMinIters preset to false.
2331 } else if (VF.isScalable() && !TTI->isVScaleKnownToBeAPowerOfTwo() &&
2334 // vscale is not necessarily a power-of-2, which means we cannot guarantee
2335 // an overflow to zero when updating induction variables and so an
2336 // additional overflow check is required before entering the vector loop.
2337
2338 // Get the maximum unsigned value for the type.
2339 Value *MaxUIntTripCount =
2340 ConstantInt::get(CountTy, cast<IntegerType>(CountTy)->getMask());
2341 Value *LHS = Builder.CreateSub(MaxUIntTripCount, Count);
2342
2343 // Don't execute the vector loop if (UMax - n) < (VF * UF).
2344 CheckMinIters = Builder.CreateICmp(ICmpInst::ICMP_ULT, LHS, CreateStep());
2345 }
2346 return CheckMinIters;
2347}
2348
2349/// Replace \p VPBB with a VPIRBasicBlock wrapping \p IRBB. All recipes from \p
2350/// VPBB are moved to the end of the newly created VPIRBasicBlock. All
2351/// predecessors and successors of VPBB, if any, are rewired to the new
2352/// VPIRBasicBlock. If \p VPBB may be unreachable, \p Plan must be passed.
2354 BasicBlock *IRBB,
2355 VPlan *Plan = nullptr) {
2356 if (!Plan)
2357 Plan = VPBB->getPlan();
2358 VPIRBasicBlock *IRVPBB = Plan->createVPIRBasicBlock(IRBB);
2359 auto IP = IRVPBB->begin();
2360 for (auto &R : make_early_inc_range(VPBB->phis()))
2361 R.moveBefore(*IRVPBB, IP);
2362
2363 for (auto &R :
2365 R.moveBefore(*IRVPBB, IRVPBB->end());
2366
2367 VPBlockUtils::reassociateBlocks(VPBB, IRVPBB);
2368 // VPBB is now dead and will be cleaned up when the plan gets destroyed.
2369 return IRVPBB;
2370}
2371
2373 BasicBlock *VectorPH = OrigLoop->getLoopPreheader();
2374 assert(VectorPH && "Invalid loop structure");
2375 assert((OrigLoop->getUniqueLatchExitBlock() ||
2376 Cost->requiresScalarEpilogue(VF.isVector())) &&
2377 "loops not exiting via the latch without required epilogue?");
2378
2379 // NOTE: The Plan's scalar preheader VPBB isn't replaced with a VPIRBasicBlock
2380 // wrapping the newly created scalar preheader here at the moment, because the
2381 // Plan's scalar preheader may be unreachable at this point. Instead it is
2382 // replaced in executePlan.
2383 return SplitBlock(VectorPH, VectorPH->getTerminator(), DT, LI, nullptr,
2384 Twine(Prefix) + "scalar.ph");
2385}
2386
2387/// Return the expanded step for \p ID using \p ExpandedSCEVs to look up SCEV
2388/// expansion results.
2390 const SCEV2ValueTy &ExpandedSCEVs) {
2391 const SCEV *Step = ID.getStep();
2392 if (auto *C = dyn_cast<SCEVConstant>(Step))
2393 return C->getValue();
2394 if (auto *U = dyn_cast<SCEVUnknown>(Step))
2395 return U->getValue();
2396 Value *V = ExpandedSCEVs.lookup(Step);
2397 assert(V && "SCEV must be expanded at this point");
2398 return V;
2399}
2400
2401/// Knowing that loop \p L executes a single vector iteration, add instructions
2402/// that will get simplified and thus should not have any cost to \p
2403/// InstsToIgnore.
2406 SmallPtrSetImpl<Instruction *> &InstsToIgnore) {
2407 auto *Cmp = L->getLatchCmpInst();
2408 if (Cmp)
2409 InstsToIgnore.insert(Cmp);
2410 for (const auto &KV : IL) {
2411 // Extract the key by hand so that it can be used in the lambda below. Note
2412 // that captured structured bindings are a C++20 extension.
2413 const PHINode *IV = KV.first;
2414
2415 // Get next iteration value of the induction variable.
2416 Instruction *IVInst =
2417 cast<Instruction>(IV->getIncomingValueForBlock(L->getLoopLatch()));
2418 if (all_of(IVInst->users(),
2419 [&](const User *U) { return U == IV || U == Cmp; }))
2420 InstsToIgnore.insert(IVInst);
2421 }
2422}
2423
2425 // Create a new IR basic block for the scalar preheader.
2426 BasicBlock *ScalarPH = createScalarPreheader("");
2427 return ScalarPH->getSinglePredecessor();
2428}
2429
2430namespace {
2431
2432struct CSEDenseMapInfo {
2433 static bool canHandle(const Instruction *I) {
2436 }
2437
2438 static inline Instruction *getEmptyKey() {
2440 }
2441
2442 static inline Instruction *getTombstoneKey() {
2443 return DenseMapInfo<Instruction *>::getTombstoneKey();
2444 }
2445
2446 static unsigned getHashValue(const Instruction *I) {
2447 assert(canHandle(I) && "Unknown instruction!");
2448 return hash_combine(I->getOpcode(),
2449 hash_combine_range(I->operand_values()));
2450 }
2451
2452 static bool isEqual(const Instruction *LHS, const Instruction *RHS) {
2453 if (LHS == getEmptyKey() || RHS == getEmptyKey() ||
2454 LHS == getTombstoneKey() || RHS == getTombstoneKey())
2455 return LHS == RHS;
2456 return LHS->isIdenticalTo(RHS);
2457 }
2458};
2459
2460} // end anonymous namespace
2461
2462///Perform cse of induction variable instructions.
2463static void cse(BasicBlock *BB) {
2464 // Perform simple cse.
2466 for (Instruction &In : llvm::make_early_inc_range(*BB)) {
2467 if (!CSEDenseMapInfo::canHandle(&In))
2468 continue;
2469
2470 // Check if we can replace this instruction with any of the
2471 // visited instructions.
2472 if (Instruction *V = CSEMap.lookup(&In)) {
2473 In.replaceAllUsesWith(V);
2474 In.eraseFromParent();
2475 continue;
2476 }
2477
2478 CSEMap[&In] = &In;
2479 }
2480}
2481
2482/// This function attempts to return a value that represents the ElementCount
2483/// at runtime. For fixed-width VFs we know this precisely at compile
2484/// time, but for scalable VFs we calculate it based on an estimate of the
2485/// vscale value.
2487 std::optional<unsigned> VScale) {
2488 unsigned EstimatedVF = VF.getKnownMinValue();
2489 if (VF.isScalable())
2490 if (VScale)
2491 EstimatedVF *= *VScale;
2492 assert(EstimatedVF >= 1 && "Estimated VF shouldn't be less than 1");
2493 return EstimatedVF;
2494}
2495
2498 ElementCount VF) const {
2499 // We only need to calculate a cost if the VF is scalar; for actual vectors
2500 // we should already have a pre-calculated cost at each VF.
2501 if (!VF.isScalar())
2502 return getCallWideningDecision(CI, VF).Cost;
2503
2504 Type *RetTy = CI->getType();
2506 if (auto RedCost = getReductionPatternCost(CI, VF, RetTy))
2507 return *RedCost;
2508
2510 for (auto &ArgOp : CI->args())
2511 Tys.push_back(ArgOp->getType());
2512
2513 InstructionCost ScalarCallCost =
2514 TTI.getCallInstrCost(CI->getCalledFunction(), RetTy, Tys, CostKind);
2515
2516 // If this is an intrinsic we may have a lower cost for it.
2519 return std::min(ScalarCallCost, IntrinsicCost);
2520 }
2521 return ScalarCallCost;
2522}
2523
2525 if (VF.isScalar() || !canVectorizeTy(Ty))
2526 return Ty;
2527 return toVectorizedTy(Ty, VF);
2528}
2529
2532 ElementCount VF) const {
2534 assert(ID && "Expected intrinsic call!");
2535 Type *RetTy = maybeVectorizeType(CI->getType(), VF);
2536 FastMathFlags FMF;
2537 if (auto *FPMO = dyn_cast<FPMathOperator>(CI))
2538 FMF = FPMO->getFastMathFlags();
2539
2542 SmallVector<Type *> ParamTys;
2543 std::transform(FTy->param_begin(), FTy->param_end(),
2544 std::back_inserter(ParamTys),
2545 [&](Type *Ty) { return maybeVectorizeType(Ty, VF); });
2546
2547 IntrinsicCostAttributes CostAttrs(ID, RetTy, Arguments, ParamTys, FMF,
2550 return TTI.getIntrinsicInstrCost(CostAttrs, CostKind);
2551}
2552
2554 // Fix widened non-induction PHIs by setting up the PHI operands.
2555 fixNonInductionPHIs(State);
2556
2557 // Don't apply optimizations below when no (vector) loop remains, as they all
2558 // require one at the moment.
2559 VPBasicBlock *HeaderVPBB =
2560 vputils::getFirstLoopHeader(*State.Plan, State.VPDT);
2561 if (!HeaderVPBB)
2562 return;
2563
2564 BasicBlock *HeaderBB = State.CFG.VPBB2IRBB[HeaderVPBB];
2565
2566 // Remove redundant induction instructions.
2567 cse(HeaderBB);
2568}
2569
2571 auto Iter = vp_depth_first_shallow(Plan.getEntry());
2573 for (VPRecipeBase &P : VPBB->phis()) {
2575 if (!VPPhi)
2576 continue;
2577 PHINode *NewPhi = cast<PHINode>(State.get(VPPhi));
2578 // Make sure the builder has a valid insert point.
2579 Builder.SetInsertPoint(NewPhi);
2580 for (const auto &[Inc, VPBB] : VPPhi->incoming_values_and_blocks())
2581 NewPhi->addIncoming(State.get(Inc), State.CFG.VPBB2IRBB[VPBB]);
2582 }
2583 }
2584}
2585
2586void LoopVectorizationCostModel::collectLoopScalars(ElementCount VF) {
2587 // We should not collect Scalars more than once per VF. Right now, this
2588 // function is called from collectUniformsAndScalars(), which already does
2589 // this check. Collecting Scalars for VF=1 does not make any sense.
2590 assert(VF.isVector() && !Scalars.contains(VF) &&
2591 "This function should not be visited twice for the same VF");
2592
2593 // This avoids any chances of creating a REPLICATE recipe during planning
2594 // since that would result in generation of scalarized code during execution,
2595 // which is not supported for scalable vectors.
2596 if (VF.isScalable()) {
2597 Scalars[VF].insert_range(Uniforms[VF]);
2598 return;
2599 }
2600
2602
2603 // These sets are used to seed the analysis with pointers used by memory
2604 // accesses that will remain scalar.
2606 SmallPtrSet<Instruction *, 8> PossibleNonScalarPtrs;
2607 auto *Latch = TheLoop->getLoopLatch();
2608
2609 // A helper that returns true if the use of Ptr by MemAccess will be scalar.
2610 // The pointer operands of loads and stores will be scalar as long as the
2611 // memory access is not a gather or scatter operation. The value operand of a
2612 // store will remain scalar if the store is scalarized.
2613 auto IsScalarUse = [&](Instruction *MemAccess, Value *Ptr) {
2614 InstWidening WideningDecision = getWideningDecision(MemAccess, VF);
2615 assert(WideningDecision != CM_Unknown &&
2616 "Widening decision should be ready at this moment");
2617 if (auto *Store = dyn_cast<StoreInst>(MemAccess))
2618 if (Ptr == Store->getValueOperand())
2619 return WideningDecision == CM_Scalarize;
2620 assert(Ptr == getLoadStorePointerOperand(MemAccess) &&
2621 "Ptr is neither a value or pointer operand");
2622 return WideningDecision != CM_GatherScatter;
2623 };
2624
2625 // A helper that returns true if the given value is a getelementptr
2626 // instruction contained in the loop.
2627 auto IsLoopVaryingGEP = [&](Value *V) {
2628 return isa<GetElementPtrInst>(V) && !TheLoop->isLoopInvariant(V);
2629 };
2630
2631 // A helper that evaluates a memory access's use of a pointer. If the use will
2632 // be a scalar use and the pointer is only used by memory accesses, we place
2633 // the pointer in ScalarPtrs. Otherwise, the pointer is placed in
2634 // PossibleNonScalarPtrs.
2635 auto EvaluatePtrUse = [&](Instruction *MemAccess, Value *Ptr) {
2636 // We only care about bitcast and getelementptr instructions contained in
2637 // the loop.
2638 if (!IsLoopVaryingGEP(Ptr))
2639 return;
2640
2641 // If the pointer has already been identified as scalar (e.g., if it was
2642 // also identified as uniform), there's nothing to do.
2643 auto *I = cast<Instruction>(Ptr);
2644 if (Worklist.count(I))
2645 return;
2646
2647 // If the use of the pointer will be a scalar use, and all users of the
2648 // pointer are memory accesses, place the pointer in ScalarPtrs. Otherwise,
2649 // place the pointer in PossibleNonScalarPtrs.
2650 if (IsScalarUse(MemAccess, Ptr) &&
2652 ScalarPtrs.insert(I);
2653 else
2654 PossibleNonScalarPtrs.insert(I);
2655 };
2656
2657 // We seed the scalars analysis with three classes of instructions: (1)
2658 // instructions marked uniform-after-vectorization and (2) bitcast,
2659 // getelementptr and (pointer) phi instructions used by memory accesses
2660 // requiring a scalar use.
2661 //
2662 // (1) Add to the worklist all instructions that have been identified as
2663 // uniform-after-vectorization.
2664 Worklist.insert_range(Uniforms[VF]);
2665
2666 // (2) Add to the worklist all bitcast and getelementptr instructions used by
2667 // memory accesses requiring a scalar use. The pointer operands of loads and
2668 // stores will be scalar unless the operation is a gather or scatter.
2669 // The value operand of a store will remain scalar if the store is scalarized.
2670 for (auto *BB : TheLoop->blocks())
2671 for (auto &I : *BB) {
2672 if (auto *Load = dyn_cast<LoadInst>(&I)) {
2673 EvaluatePtrUse(Load, Load->getPointerOperand());
2674 } else if (auto *Store = dyn_cast<StoreInst>(&I)) {
2675 EvaluatePtrUse(Store, Store->getPointerOperand());
2676 EvaluatePtrUse(Store, Store->getValueOperand());
2677 }
2678 }
2679 for (auto *I : ScalarPtrs)
2680 if (!PossibleNonScalarPtrs.count(I)) {
2681 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *I << "\n");
2682 Worklist.insert(I);
2683 }
2684
2685 // Insert the forced scalars.
2686 // FIXME: Currently VPWidenPHIRecipe() often creates a dead vector
2687 // induction variable when the PHI user is scalarized.
2688 auto ForcedScalar = ForcedScalars.find(VF);
2689 if (ForcedScalar != ForcedScalars.end())
2690 for (auto *I : ForcedScalar->second) {
2691 LLVM_DEBUG(dbgs() << "LV: Found (forced) scalar instruction: " << *I << "\n");
2692 Worklist.insert(I);
2693 }
2694
2695 // Expand the worklist by looking through any bitcasts and getelementptr
2696 // instructions we've already identified as scalar. This is similar to the
2697 // expansion step in collectLoopUniforms(); however, here we're only
2698 // expanding to include additional bitcasts and getelementptr instructions.
2699 unsigned Idx = 0;
2700 while (Idx != Worklist.size()) {
2701 Instruction *Dst = Worklist[Idx++];
2702 if (!IsLoopVaryingGEP(Dst->getOperand(0)))
2703 continue;
2704 auto *Src = cast<Instruction>(Dst->getOperand(0));
2705 if (llvm::all_of(Src->users(), [&](User *U) -> bool {
2706 auto *J = cast<Instruction>(U);
2707 return !TheLoop->contains(J) || Worklist.count(J) ||
2708 ((isa<LoadInst>(J) || isa<StoreInst>(J)) &&
2709 IsScalarUse(J, Src));
2710 })) {
2711 Worklist.insert(Src);
2712 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Src << "\n");
2713 }
2714 }
2715
2716 // An induction variable will remain scalar if all users of the induction
2717 // variable and induction variable update remain scalar.
2718 for (const auto &Induction : Legal->getInductionVars()) {
2719 auto *Ind = Induction.first;
2720 auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch));
2721
2722 // If tail-folding is applied, the primary induction variable will be used
2723 // to feed a vector compare.
2724 if (Ind == Legal->getPrimaryInduction() && foldTailByMasking())
2725 continue;
2726
2727 // Returns true if \p Indvar is a pointer induction that is used directly by
2728 // load/store instruction \p I.
2729 auto IsDirectLoadStoreFromPtrIndvar = [&](Instruction *Indvar,
2730 Instruction *I) {
2731 return Induction.second.getKind() ==
2734 Indvar == getLoadStorePointerOperand(I) && IsScalarUse(I, Indvar);
2735 };
2736
2737 // Determine if all users of the induction variable are scalar after
2738 // vectorization.
2739 bool ScalarInd = all_of(Ind->users(), [&](User *U) -> bool {
2740 auto *I = cast<Instruction>(U);
2741 return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I) ||
2742 IsDirectLoadStoreFromPtrIndvar(Ind, I);
2743 });
2744 if (!ScalarInd)
2745 continue;
2746
2747 // If the induction variable update is a fixed-order recurrence, neither the
2748 // induction variable or its update should be marked scalar after
2749 // vectorization.
2750 auto *IndUpdatePhi = dyn_cast<PHINode>(IndUpdate);
2751 if (IndUpdatePhi && Legal->isFixedOrderRecurrence(IndUpdatePhi))
2752 continue;
2753
2754 // Determine if all users of the induction variable update instruction are
2755 // scalar after vectorization.
2756 bool ScalarIndUpdate = all_of(IndUpdate->users(), [&](User *U) -> bool {
2757 auto *I = cast<Instruction>(U);
2758 return I == Ind || !TheLoop->contains(I) || Worklist.count(I) ||
2759 IsDirectLoadStoreFromPtrIndvar(IndUpdate, I);
2760 });
2761 if (!ScalarIndUpdate)
2762 continue;
2763
2764 // The induction variable and its update instruction will remain scalar.
2765 Worklist.insert(Ind);
2766 Worklist.insert(IndUpdate);
2767 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Ind << "\n");
2768 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *IndUpdate
2769 << "\n");
2770 }
2771
2772 Scalars[VF].insert_range(Worklist);
2773}
2774
2776 Instruction *I, ElementCount VF) const {
2777 if (!isPredicatedInst(I))
2778 return false;
2779
2780 // Do we have a non-scalar lowering for this predicated
2781 // instruction? No - it is scalar with predication.
2782 switch(I->getOpcode()) {
2783 default:
2784 return true;
2785 case Instruction::Call:
2786 if (VF.isScalar())
2787 return true;
2789 case Instruction::Load:
2790 case Instruction::Store: {
2792 auto *Ty = getLoadStoreType(I);
2793 unsigned AS = getLoadStoreAddressSpace(I);
2794 Type *VTy = Ty;
2795 if (VF.isVector())
2796 VTy = VectorType::get(Ty, VF);
2797 const Align Alignment = getLoadStoreAlignment(I);
2798 return isa<LoadInst>(I) ? !(isLegalMaskedLoad(Ty, Ptr, Alignment, AS) ||
2799 TTI.isLegalMaskedGather(VTy, Alignment))
2800 : !(isLegalMaskedStore(Ty, Ptr, Alignment, AS) ||
2801 TTI.isLegalMaskedScatter(VTy, Alignment));
2802 }
2803 case Instruction::UDiv:
2804 case Instruction::SDiv:
2805 case Instruction::SRem:
2806 case Instruction::URem: {
2807 // We have the option to use the safe-divisor idiom to avoid predication.
2808 // The cost based decision here will always select safe-divisor for
2809 // scalable vectors as scalarization isn't legal.
2810 const auto [ScalarCost, SafeDivisorCost] = getDivRemSpeculationCost(I, VF);
2811 return isDivRemScalarWithPredication(ScalarCost, SafeDivisorCost);
2812 }
2813 }
2814}
2815
2816// TODO: Fold into LoopVectorizationLegality::isMaskRequired.
2818 // TODO: We can use the loop-preheader as context point here and get
2819 // context sensitive reasoning for isSafeToSpeculativelyExecute.
2821 (isa<LoadInst, StoreInst, CallInst>(I) && !Legal->isMaskRequired(I)) ||
2823 return false;
2824
2825 // If the instruction was executed conditionally in the original scalar loop,
2826 // predication is needed with a mask whose lanes are all possibly inactive.
2827 if (Legal->blockNeedsPredication(I->getParent()))
2828 return true;
2829
2830 // If we're not folding the tail by masking, predication is unnecessary.
2831 if (!foldTailByMasking())
2832 return false;
2833
2834 // All that remain are instructions with side-effects originally executed in
2835 // the loop unconditionally, but now execute under a tail-fold mask (only)
2836 // having at least one active lane (the first). If the side-effects of the
2837 // instruction are invariant, executing it w/o (the tail-folding) mask is safe
2838 // - it will cause the same side-effects as when masked.
2839 switch(I->getOpcode()) {
2840 default:
2842 "instruction should have been considered by earlier checks");
2843 case Instruction::Call:
2844 // Side-effects of a Call are assumed to be non-invariant, needing a
2845 // (fold-tail) mask.
2846 assert(Legal->isMaskRequired(I) &&
2847 "should have returned earlier for calls not needing a mask");
2848 return true;
2849 case Instruction::Load:
2850 // If the address is loop invariant no predication is needed.
2851 return !Legal->isInvariant(getLoadStorePointerOperand(I));
2852 case Instruction::Store: {
2853 // For stores, we need to prove both speculation safety (which follows from
2854 // the same argument as loads), but also must prove the value being stored
2855 // is correct. The easiest form of the later is to require that all values
2856 // stored are the same.
2857 return !(Legal->isInvariant(getLoadStorePointerOperand(I)) &&
2858 TheLoop->isLoopInvariant(cast<StoreInst>(I)->getValueOperand()));
2859 }
2860 case Instruction::UDiv:
2861 case Instruction::SDiv:
2862 case Instruction::SRem:
2863 case Instruction::URem:
2864 // If the divisor is loop-invariant no predication is needed.
2865 return !Legal->isInvariant(I->getOperand(1));
2866 }
2867}
2868
2869std::pair<InstructionCost, InstructionCost>
2871 ElementCount VF) const {
2872 assert(I->getOpcode() == Instruction::UDiv ||
2873 I->getOpcode() == Instruction::SDiv ||
2874 I->getOpcode() == Instruction::SRem ||
2875 I->getOpcode() == Instruction::URem);
2877
2878 // Scalarization isn't legal for scalable vector types
2879 InstructionCost ScalarizationCost = InstructionCost::getInvalid();
2880 if (!VF.isScalable()) {
2881 // Get the scalarization cost and scale this amount by the probability of
2882 // executing the predicated block. If the instruction is not predicated,
2883 // we fall through to the next case.
2884 ScalarizationCost = 0;
2885
2886 // These instructions have a non-void type, so account for the phi nodes
2887 // that we will create. This cost is likely to be zero. The phi node
2888 // cost, if any, should be scaled by the block probability because it
2889 // models a copy at the end of each predicated block.
2890 ScalarizationCost +=
2891 VF.getFixedValue() * TTI.getCFInstrCost(Instruction::PHI, CostKind);
2892
2893 // The cost of the non-predicated instruction.
2894 ScalarizationCost +=
2895 VF.getFixedValue() *
2896 TTI.getArithmeticInstrCost(I->getOpcode(), I->getType(), CostKind);
2897
2898 // The cost of insertelement and extractelement instructions needed for
2899 // scalarization.
2900 ScalarizationCost += getScalarizationOverhead(I, VF);
2901
2902 // Scale the cost by the probability of executing the predicated blocks.
2903 // This assumes the predicated block for each vector lane is equally
2904 // likely.
2905 ScalarizationCost = ScalarizationCost / getPredBlockCostDivisor(CostKind);
2906 }
2907
2908 InstructionCost SafeDivisorCost = 0;
2909 auto *VecTy = toVectorTy(I->getType(), VF);
2910 auto *DivisorI = dyn_cast<Instruction>(I->getOperand(1));
2911 if (DivisorI && !Legal->isInvariant(DivisorI)) {
2912 // The cost of the select guard to ensure all lanes are well defined
2913 // after we speculate above any internal control flow.
2914 SafeDivisorCost +=
2915 TTI.getCmpSelInstrCost(Instruction::Select, VecTy,
2916 toVectorTy(Type::getInt1Ty(I->getContext()), VF),
2918 }
2919
2920 SmallVector<const Value *, 4> Operands(I->operand_values());
2921 SafeDivisorCost += TTI.getArithmeticInstrCost(
2922 I->getOpcode(), VecTy, CostKind,
2923 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
2924 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
2925 Operands, I);
2926 return {ScalarizationCost, SafeDivisorCost};
2927}
2928
2930 Instruction *I, ElementCount VF) const {
2931 assert(isAccessInterleaved(I) && "Expecting interleaved access.");
2933 "Decision should not be set yet.");
2934 auto *Group = getInterleavedAccessGroup(I);
2935 assert(Group && "Must have a group.");
2936 unsigned InterleaveFactor = Group->getFactor();
2937
2938 // If the instruction's allocated size doesn't equal its type size, it
2939 // requires padding and will be scalarized.
2940 auto &DL = I->getDataLayout();
2941 auto *ScalarTy = getLoadStoreType(I);
2942 if (hasIrregularType(ScalarTy, DL))
2943 return false;
2944
2945 // For scalable vectors, the interleave factors must be <= 8 since we require
2946 // the (de)interleaveN intrinsics instead of shufflevectors.
2947 if (VF.isScalable() && InterleaveFactor > 8)
2948 return false;
2949
2950 // If the group involves a non-integral pointer, we may not be able to
2951 // losslessly cast all values to a common type.
2952 bool ScalarNI = DL.isNonIntegralPointerType(ScalarTy);
2953 for (unsigned Idx = 0; Idx < InterleaveFactor; Idx++) {
2954 Instruction *Member = Group->getMember(Idx);
2955 if (!Member)
2956 continue;
2957 auto *MemberTy = getLoadStoreType(Member);
2958 bool MemberNI = DL.isNonIntegralPointerType(MemberTy);
2959 // Don't coerce non-integral pointers to integers or vice versa.
2960 if (MemberNI != ScalarNI)
2961 // TODO: Consider adding special nullptr value case here
2962 return false;
2963 if (MemberNI && ScalarNI &&
2964 ScalarTy->getPointerAddressSpace() !=
2965 MemberTy->getPointerAddressSpace())
2966 return false;
2967 }
2968
2969 // Check if masking is required.
2970 // A Group may need masking for one of two reasons: it resides in a block that
2971 // needs predication, or it was decided to use masking to deal with gaps
2972 // (either a gap at the end of a load-access that may result in a speculative
2973 // load, or any gaps in a store-access).
2974 bool PredicatedAccessRequiresMasking =
2975 blockNeedsPredicationForAnyReason(I->getParent()) &&
2976 Legal->isMaskRequired(I);
2977 bool LoadAccessWithGapsRequiresEpilogMasking =
2978 isa<LoadInst>(I) && Group->requiresScalarEpilogue() &&
2980 bool StoreAccessWithGapsRequiresMasking =
2981 isa<StoreInst>(I) && !Group->isFull();
2982 if (!PredicatedAccessRequiresMasking &&
2983 !LoadAccessWithGapsRequiresEpilogMasking &&
2984 !StoreAccessWithGapsRequiresMasking)
2985 return true;
2986
2987 // If masked interleaving is required, we expect that the user/target had
2988 // enabled it, because otherwise it either wouldn't have been created or
2989 // it should have been invalidated by the CostModel.
2991 "Masked interleave-groups for predicated accesses are not enabled.");
2992
2993 if (Group->isReverse())
2994 return false;
2995
2996 // TODO: Support interleaved access that requires a gap mask for scalable VFs.
2997 bool NeedsMaskForGaps = LoadAccessWithGapsRequiresEpilogMasking ||
2998 StoreAccessWithGapsRequiresMasking;
2999 if (VF.isScalable() && NeedsMaskForGaps)
3000 return false;
3001
3002 auto *Ty = getLoadStoreType(I);
3003 const Align Alignment = getLoadStoreAlignment(I);
3004 unsigned AS = getLoadStoreAddressSpace(I);
3005 return isa<LoadInst>(I) ? TTI.isLegalMaskedLoad(Ty, Alignment, AS)
3006 : TTI.isLegalMaskedStore(Ty, Alignment, AS);
3007}
3008
3010 Instruction *I, ElementCount VF) {
3011 // Get and ensure we have a valid memory instruction.
3012 assert((isa<LoadInst, StoreInst>(I)) && "Invalid memory instruction");
3013
3015 auto *ScalarTy = getLoadStoreType(I);
3016
3017 // In order to be widened, the pointer should be consecutive, first of all.
3018 if (!Legal->isConsecutivePtr(ScalarTy, Ptr))
3019 return false;
3020
3021 // If the instruction is a store located in a predicated block, it will be
3022 // scalarized.
3023 if (isScalarWithPredication(I, VF))
3024 return false;
3025
3026 // If the instruction's allocated size doesn't equal it's type size, it
3027 // requires padding and will be scalarized.
3028 auto &DL = I->getDataLayout();
3029 if (hasIrregularType(ScalarTy, DL))
3030 return false;
3031
3032 return true;
3033}
3034
3035void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
3036 // We should not collect Uniforms more than once per VF. Right now,
3037 // this function is called from collectUniformsAndScalars(), which
3038 // already does this check. Collecting Uniforms for VF=1 does not make any
3039 // sense.
3040
3041 assert(VF.isVector() && !Uniforms.contains(VF) &&
3042 "This function should not be visited twice for the same VF");
3043
3044 // Visit the list of Uniforms. If we find no uniform value, we won't
3045 // analyze again. Uniforms.count(VF) will return 1.
3046 Uniforms[VF].clear();
3047
3048 // Now we know that the loop is vectorizable!
3049 // Collect instructions inside the loop that will remain uniform after
3050 // vectorization.
3051
3052 // Global values, params and instructions outside of current loop are out of
3053 // scope.
3054 auto IsOutOfScope = [&](Value *V) -> bool {
3056 return (!I || !TheLoop->contains(I));
3057 };
3058
3059 // Worklist containing uniform instructions demanding lane 0.
3060 SetVector<Instruction *> Worklist;
3061
3062 // Add uniform instructions demanding lane 0 to the worklist. Instructions
3063 // that require predication must not be considered uniform after
3064 // vectorization, because that would create an erroneous replicating region
3065 // where only a single instance out of VF should be formed.
3066 auto AddToWorklistIfAllowed = [&](Instruction *I) -> void {
3067 if (IsOutOfScope(I)) {
3068 LLVM_DEBUG(dbgs() << "LV: Found not uniform due to scope: "
3069 << *I << "\n");
3070 return;
3071 }
3072 if (isPredicatedInst(I)) {
3073 LLVM_DEBUG(
3074 dbgs() << "LV: Found not uniform due to requiring predication: " << *I
3075 << "\n");
3076 return;
3077 }
3078 LLVM_DEBUG(dbgs() << "LV: Found uniform instruction: " << *I << "\n");
3079 Worklist.insert(I);
3080 };
3081
3082 // Start with the conditional branches exiting the loop. If the branch
3083 // condition is an instruction contained in the loop that is only used by the
3084 // branch, it is uniform. Note conditions from uncountable early exits are not
3085 // uniform.
3087 TheLoop->getExitingBlocks(Exiting);
3088 for (BasicBlock *E : Exiting) {
3089 if (Legal->hasUncountableEarlyExit() && TheLoop->getLoopLatch() != E)
3090 continue;
3091 auto *Cmp = dyn_cast<Instruction>(E->getTerminator()->getOperand(0));
3092 if (Cmp && TheLoop->contains(Cmp) && Cmp->hasOneUse())
3093 AddToWorklistIfAllowed(Cmp);
3094 }
3095
3096 auto PrevVF = VF.divideCoefficientBy(2);
3097 // Return true if all lanes perform the same memory operation, and we can
3098 // thus choose to execute only one.
3099 auto IsUniformMemOpUse = [&](Instruction *I) {
3100 // If the value was already known to not be uniform for the previous
3101 // (smaller VF), it cannot be uniform for the larger VF.
3102 if (PrevVF.isVector()) {
3103 auto Iter = Uniforms.find(PrevVF);
3104 if (Iter != Uniforms.end() && !Iter->second.contains(I))
3105 return false;
3106 }
3107 if (!Legal->isUniformMemOp(*I, VF))
3108 return false;
3109 if (isa<LoadInst>(I))
3110 // Loading the same address always produces the same result - at least
3111 // assuming aliasing and ordering which have already been checked.
3112 return true;
3113 // Storing the same value on every iteration.
3114 return TheLoop->isLoopInvariant(cast<StoreInst>(I)->getValueOperand());
3115 };
3116
3117 auto IsUniformDecision = [&](Instruction *I, ElementCount VF) {
3118 InstWidening WideningDecision = getWideningDecision(I, VF);
3119 assert(WideningDecision != CM_Unknown &&
3120 "Widening decision should be ready at this moment");
3121
3122 if (IsUniformMemOpUse(I))
3123 return true;
3124
3125 return (WideningDecision == CM_Widen ||
3126 WideningDecision == CM_Widen_Reverse ||
3127 WideningDecision == CM_Interleave);
3128 };
3129
3130 // Returns true if Ptr is the pointer operand of a memory access instruction
3131 // I, I is known to not require scalarization, and the pointer is not also
3132 // stored.
3133 auto IsVectorizedMemAccessUse = [&](Instruction *I, Value *Ptr) -> bool {
3134 if (isa<StoreInst>(I) && I->getOperand(0) == Ptr)
3135 return false;
3136 return getLoadStorePointerOperand(I) == Ptr &&
3137 (IsUniformDecision(I, VF) || Legal->isInvariant(Ptr));
3138 };
3139
3140 // Holds a list of values which are known to have at least one uniform use.
3141 // Note that there may be other uses which aren't uniform. A "uniform use"
3142 // here is something which only demands lane 0 of the unrolled iterations;
3143 // it does not imply that all lanes produce the same value (e.g. this is not
3144 // the usual meaning of uniform)
3145 SetVector<Value *> HasUniformUse;
3146
3147 // Scan the loop for instructions which are either a) known to have only
3148 // lane 0 demanded or b) are uses which demand only lane 0 of their operand.
3149 for (auto *BB : TheLoop->blocks())
3150 for (auto &I : *BB) {
3151 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(&I)) {
3152 switch (II->getIntrinsicID()) {
3153 case Intrinsic::sideeffect:
3154 case Intrinsic::experimental_noalias_scope_decl:
3155 case Intrinsic::assume:
3156 case Intrinsic::lifetime_start:
3157 case Intrinsic::lifetime_end:
3158 if (TheLoop->hasLoopInvariantOperands(&I))
3159 AddToWorklistIfAllowed(&I);
3160 break;
3161 default:
3162 break;
3163 }
3164 }
3165
3166 if (auto *EVI = dyn_cast<ExtractValueInst>(&I)) {
3167 if (IsOutOfScope(EVI->getAggregateOperand())) {
3168 AddToWorklistIfAllowed(EVI);
3169 continue;
3170 }
3171 // Only ExtractValue instructions where the aggregate value comes from a
3172 // call are allowed to be non-uniform.
3173 assert(isa<CallInst>(EVI->getAggregateOperand()) &&
3174 "Expected aggregate value to be call return value");
3175 }
3176
3177 // If there's no pointer operand, there's nothing to do.
3179 if (!Ptr)
3180 continue;
3181
3182 // If the pointer can be proven to be uniform, always add it to the
3183 // worklist.
3184 if (isa<Instruction>(Ptr) && Legal->isUniform(Ptr, VF))
3185 AddToWorklistIfAllowed(cast<Instruction>(Ptr));
3186
3187 if (IsUniformMemOpUse(&I))
3188 AddToWorklistIfAllowed(&I);
3189
3190 if (IsVectorizedMemAccessUse(&I, Ptr))
3191 HasUniformUse.insert(Ptr);
3192 }
3193
3194 // Add to the worklist any operands which have *only* uniform (e.g. lane 0
3195 // demanding) users. Since loops are assumed to be in LCSSA form, this
3196 // disallows uses outside the loop as well.
3197 for (auto *V : HasUniformUse) {
3198 if (IsOutOfScope(V))
3199 continue;
3200 auto *I = cast<Instruction>(V);
3201 bool UsersAreMemAccesses = all_of(I->users(), [&](User *U) -> bool {
3202 auto *UI = cast<Instruction>(U);
3203 return TheLoop->contains(UI) && IsVectorizedMemAccessUse(UI, V);
3204 });
3205 if (UsersAreMemAccesses)
3206 AddToWorklistIfAllowed(I);
3207 }
3208
3209 // Expand Worklist in topological order: whenever a new instruction
3210 // is added , its users should be already inside Worklist. It ensures
3211 // a uniform instruction will only be used by uniform instructions.
3212 unsigned Idx = 0;
3213 while (Idx != Worklist.size()) {
3214 Instruction *I = Worklist[Idx++];
3215
3216 for (auto *OV : I->operand_values()) {
3217 // isOutOfScope operands cannot be uniform instructions.
3218 if (IsOutOfScope(OV))
3219 continue;
3220 // First order recurrence Phi's should typically be considered
3221 // non-uniform.
3222 auto *OP = dyn_cast<PHINode>(OV);
3223 if (OP && Legal->isFixedOrderRecurrence(OP))
3224 continue;
3225 // If all the users of the operand are uniform, then add the
3226 // operand into the uniform worklist.
3227 auto *OI = cast<Instruction>(OV);
3228 if (llvm::all_of(OI->users(), [&](User *U) -> bool {
3229 auto *J = cast<Instruction>(U);
3230 return Worklist.count(J) || IsVectorizedMemAccessUse(J, OI);
3231 }))
3232 AddToWorklistIfAllowed(OI);
3233 }
3234 }
3235
3236 // For an instruction to be added into Worklist above, all its users inside
3237 // the loop should also be in Worklist. However, this condition cannot be
3238 // true for phi nodes that form a cyclic dependence. We must process phi
3239 // nodes separately. An induction variable will remain uniform if all users
3240 // of the induction variable and induction variable update remain uniform.
3241 // The code below handles both pointer and non-pointer induction variables.
3242 BasicBlock *Latch = TheLoop->getLoopLatch();
3243 for (const auto &Induction : Legal->getInductionVars()) {
3244 auto *Ind = Induction.first;
3245 auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch));
3246
3247 // Determine if all users of the induction variable are uniform after
3248 // vectorization.
3249 bool UniformInd = all_of(Ind->users(), [&](User *U) -> bool {
3250 auto *I = cast<Instruction>(U);
3251 return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I) ||
3252 IsVectorizedMemAccessUse(I, Ind);
3253 });
3254 if (!UniformInd)
3255 continue;
3256
3257 // Determine if all users of the induction variable update instruction are
3258 // uniform after vectorization.
3259 bool UniformIndUpdate = all_of(IndUpdate->users(), [&](User *U) -> bool {
3260 auto *I = cast<Instruction>(U);
3261 return I == Ind || Worklist.count(I) ||
3262 IsVectorizedMemAccessUse(I, IndUpdate);
3263 });
3264 if (!UniformIndUpdate)
3265 continue;
3266
3267 // The induction variable and its update instruction will remain uniform.
3268 AddToWorklistIfAllowed(Ind);
3269 AddToWorklistIfAllowed(IndUpdate);
3270 }
3271
3272 Uniforms[VF].insert_range(Worklist);
3273}
3274
3276 LLVM_DEBUG(dbgs() << "LV: Performing code size checks.\n");
3277
3278 if (Legal->getRuntimePointerChecking()->Need) {
3279 reportVectorizationFailure("Runtime ptr check is required with -Os/-Oz",
3280 "runtime pointer checks needed. Enable vectorization of this "
3281 "loop with '#pragma clang loop vectorize(enable)' when "
3282 "compiling with -Os/-Oz",
3283 "CantVersionLoopWithOptForSize", ORE, TheLoop);
3284 return true;
3285 }
3286
3287 if (!PSE.getPredicate().isAlwaysTrue()) {
3288 reportVectorizationFailure("Runtime SCEV check is required with -Os/-Oz",
3289 "runtime SCEV checks needed. Enable vectorization of this "
3290 "loop with '#pragma clang loop vectorize(enable)' when "
3291 "compiling with -Os/-Oz",
3292 "CantVersionLoopWithOptForSize", ORE, TheLoop);
3293 return true;
3294 }
3295
3296 // FIXME: Avoid specializing for stride==1 instead of bailing out.
3297 if (!Legal->getLAI()->getSymbolicStrides().empty()) {
3298 reportVectorizationFailure("Runtime stride check for small trip count",
3299 "runtime stride == 1 checks needed. Enable vectorization of "
3300 "this loop without such check by compiling with -Os/-Oz",
3301 "CantVersionLoopWithOptForSize", ORE, TheLoop);
3302 return true;
3303 }
3304
3305 return false;
3306}
3307
3308bool LoopVectorizationCostModel::isScalableVectorizationAllowed() {
3309 if (IsScalableVectorizationAllowed)
3310 return *IsScalableVectorizationAllowed;
3311
3312 IsScalableVectorizationAllowed = false;
3313 if (!TTI.supportsScalableVectors() && !ForceTargetSupportsScalableVectors)
3314 return false;
3315
3317 reportVectorizationInfo("Scalable vectorization is explicitly disabled",
3318 "ScalableVectorizationDisabled", ORE, TheLoop);
3319 return false;
3320 }
3321
3322 LLVM_DEBUG(dbgs() << "LV: Scalable vectorization is available\n");
3323
3324 auto MaxScalableVF = ElementCount::getScalable(
3325 std::numeric_limits<ElementCount::ScalarTy>::max());
3326
3327 // Test that the loop-vectorizer can legalize all operations for this MaxVF.
3328 // FIXME: While for scalable vectors this is currently sufficient, this should
3329 // be replaced by a more detailed mechanism that filters out specific VFs,
3330 // instead of invalidating vectorization for a whole set of VFs based on the
3331 // MaxVF.
3332
3333 // Disable scalable vectorization if the loop contains unsupported reductions.
3334 if (!canVectorizeReductions(MaxScalableVF)) {
3336 "Scalable vectorization not supported for the reduction "
3337 "operations found in this loop.",
3338 "ScalableVFUnfeasible", ORE, TheLoop);
3339 return false;
3340 }
3341
3342 // Disable scalable vectorization if the loop contains any instructions
3343 // with element types not supported for scalable vectors.
3344 if (any_of(ElementTypesInLoop, [&](Type *Ty) {
3345 return !Ty->isVoidTy() &&
3346 !this->TTI.isElementTypeLegalForScalableVector(Ty);
3347 })) {
3348 reportVectorizationInfo("Scalable vectorization is not supported "
3349 "for all element types found in this loop.",
3350 "ScalableVFUnfeasible", ORE, TheLoop);
3351 return false;
3352 }
3353
3354 if (!Legal->isSafeForAnyVectorWidth() && !getMaxVScale(*TheFunction, TTI)) {
3355 reportVectorizationInfo("The target does not provide maximum vscale value "
3356 "for safe distance analysis.",
3357 "ScalableVFUnfeasible", ORE, TheLoop);
3358 return false;
3359 }
3360
3361 IsScalableVectorizationAllowed = true;
3362 return true;
3363}
3364
3366LoopVectorizationCostModel::getMaxLegalScalableVF(unsigned MaxSafeElements) {
3367 if (!isScalableVectorizationAllowed())
3368 return ElementCount::getScalable(0);
3369
3370 auto MaxScalableVF = ElementCount::getScalable(
3371 std::numeric_limits<ElementCount::ScalarTy>::max());
3372 if (Legal->isSafeForAnyVectorWidth())
3373 return MaxScalableVF;
3374
3375 std::optional<unsigned> MaxVScale = getMaxVScale(*TheFunction, TTI);
3376 // Limit MaxScalableVF by the maximum safe dependence distance.
3377 MaxScalableVF = ElementCount::getScalable(MaxSafeElements / *MaxVScale);
3378
3379 if (!MaxScalableVF)
3381 "Max legal vector width too small, scalable vectorization "
3382 "unfeasible.",
3383 "ScalableVFUnfeasible", ORE, TheLoop);
3384
3385 return MaxScalableVF;
3386}
3387
3388FixedScalableVFPair LoopVectorizationCostModel::computeFeasibleMaxVF(
3389 unsigned MaxTripCount, ElementCount UserVF, bool FoldTailByMasking) {
3390 MinBWs = computeMinimumValueSizes(TheLoop->getBlocks(), *DB, &TTI);
3391 unsigned SmallestType, WidestType;
3392 std::tie(SmallestType, WidestType) = getSmallestAndWidestTypes();
3393
3394 // Get the maximum safe dependence distance in bits computed by LAA.
3395 // It is computed by MaxVF * sizeOf(type) * 8, where type is taken from
3396 // the memory accesses that is most restrictive (involved in the smallest
3397 // dependence distance).
3398 unsigned MaxSafeElementsPowerOf2 =
3399 bit_floor(Legal->getMaxSafeVectorWidthInBits() / WidestType);
3400 if (!Legal->isSafeForAnyStoreLoadForwardDistances()) {
3401 unsigned SLDist = Legal->getMaxStoreLoadForwardSafeDistanceInBits();
3402 MaxSafeElementsPowerOf2 =
3403 std::min(MaxSafeElementsPowerOf2, SLDist / WidestType);
3404 }
3405 auto MaxSafeFixedVF = ElementCount::getFixed(MaxSafeElementsPowerOf2);
3406 auto MaxSafeScalableVF = getMaxLegalScalableVF(MaxSafeElementsPowerOf2);
3407
3408 if (!Legal->isSafeForAnyVectorWidth())
3409 this->MaxSafeElements = MaxSafeElementsPowerOf2;
3410
3411 LLVM_DEBUG(dbgs() << "LV: The max safe fixed VF is: " << MaxSafeFixedVF
3412 << ".\n");
3413 LLVM_DEBUG(dbgs() << "LV: The max safe scalable VF is: " << MaxSafeScalableVF
3414 << ".\n");
3415
3416 // First analyze the UserVF, fall back if the UserVF should be ignored.
3417 if (UserVF) {
3418 auto MaxSafeUserVF =
3419 UserVF.isScalable() ? MaxSafeScalableVF : MaxSafeFixedVF;
3420
3421 if (ElementCount::isKnownLE(UserVF, MaxSafeUserVF)) {
3422 // If `VF=vscale x N` is safe, then so is `VF=N`
3423 if (UserVF.isScalable())
3424 return FixedScalableVFPair(
3425 ElementCount::getFixed(UserVF.getKnownMinValue()), UserVF);
3426
3427 return UserVF;
3428 }
3429
3430 assert(ElementCount::isKnownGT(UserVF, MaxSafeUserVF));
3431
3432 // Only clamp if the UserVF is not scalable. If the UserVF is scalable, it
3433 // is better to ignore the hint and let the compiler choose a suitable VF.
3434 if (!UserVF.isScalable()) {
3435 LLVM_DEBUG(dbgs() << "LV: User VF=" << UserVF
3436 << " is unsafe, clamping to max safe VF="
3437 << MaxSafeFixedVF << ".\n");
3438 ORE->emit([&]() {
3439 return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationFactor",
3440 TheLoop->getStartLoc(),
3441 TheLoop->getHeader())
3442 << "User-specified vectorization factor "
3443 << ore::NV("UserVectorizationFactor", UserVF)
3444 << " is unsafe, clamping to maximum safe vectorization factor "
3445 << ore::NV("VectorizationFactor", MaxSafeFixedVF);
3446 });
3447 return MaxSafeFixedVF;
3448 }
3449
3450 if (!TTI.supportsScalableVectors() && !ForceTargetSupportsScalableVectors) {
3451 LLVM_DEBUG(dbgs() << "LV: User VF=" << UserVF
3452 << " is ignored because scalable vectors are not "
3453 "available.\n");
3454 ORE->emit([&]() {
3455 return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationFactor",
3456 TheLoop->getStartLoc(),
3457 TheLoop->getHeader())
3458 << "User-specified vectorization factor "
3459 << ore::NV("UserVectorizationFactor", UserVF)
3460 << " is ignored because the target does not support scalable "
3461 "vectors. The compiler will pick a more suitable value.";
3462 });
3463 } else {
3464 LLVM_DEBUG(dbgs() << "LV: User VF=" << UserVF
3465 << " is unsafe. Ignoring scalable UserVF.\n");
3466 ORE->emit([&]() {
3467 return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationFactor",
3468 TheLoop->getStartLoc(),
3469 TheLoop->getHeader())
3470 << "User-specified vectorization factor "
3471 << ore::NV("UserVectorizationFactor", UserVF)
3472 << " is unsafe. Ignoring the hint to let the compiler pick a "
3473 "more suitable value.";
3474 });
3475 }
3476 }
3477
3478 LLVM_DEBUG(dbgs() << "LV: The Smallest and Widest types: " << SmallestType
3479 << " / " << WidestType << " bits.\n");
3480
3481 FixedScalableVFPair Result(ElementCount::getFixed(1),
3483 if (auto MaxVF =
3484 getMaximizedVFForTarget(MaxTripCount, SmallestType, WidestType,
3485 MaxSafeFixedVF, FoldTailByMasking))
3486 Result.FixedVF = MaxVF;
3487
3488 if (auto MaxVF =
3489 getMaximizedVFForTarget(MaxTripCount, SmallestType, WidestType,
3490 MaxSafeScalableVF, FoldTailByMasking))
3491 if (MaxVF.isScalable()) {
3492 Result.ScalableVF = MaxVF;
3493 LLVM_DEBUG(dbgs() << "LV: Found feasible scalable VF = " << MaxVF
3494 << "\n");
3495 }
3496
3497 return Result;
3498}
3499
3502 if (Legal->getRuntimePointerChecking()->Need && TTI.hasBranchDivergence()) {
3503 // TODO: It may be useful to do since it's still likely to be dynamically
3504 // uniform if the target can skip.
3506 "Not inserting runtime ptr check for divergent target",
3507 "runtime pointer checks needed. Not enabled for divergent target",
3508 "CantVersionLoopWithDivergentTarget", ORE, TheLoop);
3510 }
3511
3512 ScalarEvolution *SE = PSE.getSE();
3514 unsigned MaxTC = PSE.getSmallConstantMaxTripCount();
3515 LLVM_DEBUG(dbgs() << "LV: Found trip count: " << TC << '\n');
3516 if (TC != ElementCount::getFixed(MaxTC))
3517 LLVM_DEBUG(dbgs() << "LV: Found maximum trip count: " << MaxTC << '\n');
3518 if (TC.isScalar()) {
3519 reportVectorizationFailure("Single iteration (non) loop",
3520 "loop trip count is one, irrelevant for vectorization",
3521 "SingleIterationLoop", ORE, TheLoop);
3523 }
3524
3525 // If BTC matches the widest induction type and is -1 then the trip count
3526 // computation will wrap to 0 and the vector trip count will be 0. Do not try
3527 // to vectorize.
3528 const SCEV *BTC = SE->getBackedgeTakenCount(TheLoop);
3529 if (!isa<SCEVCouldNotCompute>(BTC) &&
3530 BTC->getType()->getScalarSizeInBits() >=
3531 Legal->getWidestInductionType()->getScalarSizeInBits() &&
3533 SE->getMinusOne(BTC->getType()))) {
3535 "Trip count computation wrapped",
3536 "backedge-taken count is -1, loop trip count wrapped to 0",
3537 "TripCountWrapped", ORE, TheLoop);
3539 }
3540
3541 switch (ScalarEpilogueStatus) {
3543 return computeFeasibleMaxVF(MaxTC, UserVF, false);
3545 [[fallthrough]];
3547 LLVM_DEBUG(
3548 dbgs() << "LV: vector predicate hint/switch found.\n"
3549 << "LV: Not allowing scalar epilogue, creating predicated "
3550 << "vector loop.\n");
3551 break;
3553 // fallthrough as a special case of OptForSize
3555 if (ScalarEpilogueStatus == CM_ScalarEpilogueNotAllowedOptSize)
3556 LLVM_DEBUG(
3557 dbgs() << "LV: Not allowing scalar epilogue due to -Os/-Oz.\n");
3558 else
3559 LLVM_DEBUG(dbgs() << "LV: Not allowing scalar epilogue due to low trip "
3560 << "count.\n");
3561
3562 // Bail if runtime checks are required, which are not good when optimising
3563 // for size.
3566
3567 break;
3568 }
3569
3570 // Now try the tail folding
3571
3572 // Invalidate interleave groups that require an epilogue if we can't mask
3573 // the interleave-group.
3575 assert(WideningDecisions.empty() && Uniforms.empty() && Scalars.empty() &&
3576 "No decisions should have been taken at this point");
3577 // Note: There is no need to invalidate any cost modeling decisions here, as
3578 // none were taken so far.
3579 InterleaveInfo.invalidateGroupsRequiringScalarEpilogue();
3580 }
3581
3582 FixedScalableVFPair MaxFactors = computeFeasibleMaxVF(MaxTC, UserVF, true);
3583
3584 // Avoid tail folding if the trip count is known to be a multiple of any VF
3585 // we choose.
3586 std::optional<unsigned> MaxPowerOf2RuntimeVF =
3587 MaxFactors.FixedVF.getFixedValue();
3588 if (MaxFactors.ScalableVF) {
3589 std::optional<unsigned> MaxVScale = getMaxVScale(*TheFunction, TTI);
3590 if (MaxVScale && TTI.isVScaleKnownToBeAPowerOfTwo()) {
3591 MaxPowerOf2RuntimeVF = std::max<unsigned>(
3592 *MaxPowerOf2RuntimeVF,
3593 *MaxVScale * MaxFactors.ScalableVF.getKnownMinValue());
3594 } else
3595 MaxPowerOf2RuntimeVF = std::nullopt; // Stick with tail-folding for now.
3596 }
3597
3598 auto NoScalarEpilogueNeeded = [this, &UserIC](unsigned MaxVF) {
3599 // Return false if the loop is neither a single-latch-exit loop nor an
3600 // early-exit loop as tail-folding is not supported in that case.
3601 if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch() &&
3602 !Legal->hasUncountableEarlyExit())
3603 return false;
3604 unsigned MaxVFtimesIC = UserIC ? MaxVF * UserIC : MaxVF;
3605 ScalarEvolution *SE = PSE.getSE();
3606 // Calling getSymbolicMaxBackedgeTakenCount enables support for loops
3607 // with uncountable exits. For countable loops, the symbolic maximum must
3608 // remain identical to the known back-edge taken count.
3609 const SCEV *BackedgeTakenCount = PSE.getSymbolicMaxBackedgeTakenCount();
3610 assert((Legal->hasUncountableEarlyExit() ||
3611 BackedgeTakenCount == PSE.getBackedgeTakenCount()) &&
3612 "Invalid loop count");
3613 const SCEV *ExitCount = SE->getAddExpr(
3614 BackedgeTakenCount, SE->getOne(BackedgeTakenCount->getType()));
3615 const SCEV *Rem = SE->getURemExpr(
3616 SE->applyLoopGuards(ExitCount, TheLoop),
3617 SE->getConstant(BackedgeTakenCount->getType(), MaxVFtimesIC));
3618 return Rem->isZero();
3619 };
3620
3621 if (MaxPowerOf2RuntimeVF > 0u) {
3622 assert((UserVF.isNonZero() || isPowerOf2_32(*MaxPowerOf2RuntimeVF)) &&
3623 "MaxFixedVF must be a power of 2");
3624 if (NoScalarEpilogueNeeded(*MaxPowerOf2RuntimeVF)) {
3625 // Accept MaxFixedVF if we do not have a tail.
3626 LLVM_DEBUG(dbgs() << "LV: No tail will remain for any chosen VF.\n");
3627 return MaxFactors;
3628 }
3629 }
3630
3631 auto ExpectedTC = getSmallBestKnownTC(PSE, TheLoop);
3632 if (ExpectedTC && ExpectedTC->isFixed() &&
3633 ExpectedTC->getFixedValue() <=
3634 TTI.getMinTripCountTailFoldingThreshold()) {
3635 if (MaxPowerOf2RuntimeVF > 0u) {
3636 // If we have a low-trip-count, and the fixed-width VF is known to divide
3637 // the trip count but the scalable factor does not, use the fixed-width
3638 // factor in preference to allow the generation of a non-predicated loop.
3639 if (ScalarEpilogueStatus == CM_ScalarEpilogueNotAllowedLowTripLoop &&
3640 NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue())) {
3641 LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width so that no tail will "
3642 "remain for any chosen VF.\n");
3643 MaxFactors.ScalableVF = ElementCount::getScalable(0);
3644 return MaxFactors;
3645 }
3646 }
3647
3649 "The trip count is below the minial threshold value.",
3650 "loop trip count is too low, avoiding vectorization", "LowTripCount",
3651 ORE, TheLoop);
3653 }
3654
3655 // If we don't know the precise trip count, or if the trip count that we
3656 // found modulo the vectorization factor is not zero, try to fold the tail
3657 // by masking.
3658 // FIXME: look for a smaller MaxVF that does divide TC rather than masking.
3659 bool ContainsScalableVF = MaxFactors.ScalableVF.isNonZero();
3660 setTailFoldingStyles(ContainsScalableVF, UserIC);
3661 if (foldTailByMasking()) {
3663 LLVM_DEBUG(
3664 dbgs()
3665 << "LV: tail is folded with EVL, forcing unroll factor to be 1. Will "
3666 "try to generate VP Intrinsics with scalable vector "
3667 "factors only.\n");
3668 // Tail folded loop using VP intrinsics restricts the VF to be scalable
3669 // for now.
3670 // TODO: extend it for fixed vectors, if required.
3671 assert(ContainsScalableVF && "Expected scalable vector factor.");
3672
3673 MaxFactors.FixedVF = ElementCount::getFixed(1);
3674 }
3675 return MaxFactors;
3676 }
3677
3678 // If there was a tail-folding hint/switch, but we can't fold the tail by
3679 // masking, fallback to a vectorization with a scalar epilogue.
3680 if (ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUsePredicate) {
3681 LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking: vectorize with a "
3682 "scalar epilogue instead.\n");
3683 ScalarEpilogueStatus = CM_ScalarEpilogueAllowed;
3684 return MaxFactors;
3685 }
3686
3687 if (ScalarEpilogueStatus == CM_ScalarEpilogueNotAllowedUsePredicate) {
3688 LLVM_DEBUG(dbgs() << "LV: Can't fold tail by masking: don't vectorize\n");
3690 }
3691
3692 if (TC.isZero()) {
3694 "unable to calculate the loop count due to complex control flow",
3695 "UnknownLoopCountComplexCFG", ORE, TheLoop);
3697 }
3698
3700 "Cannot optimize for size and vectorize at the same time.",
3701 "cannot optimize for size and vectorize at the same time. "
3702 "Enable vectorization of this loop with '#pragma clang loop "
3703 "vectorize(enable)' when compiling with -Os/-Oz",
3704 "NoTailLoopWithOptForSize", ORE, TheLoop);
3706}
3707
3709 ElementCount VF) {
3710 if (ConsiderRegPressure.getNumOccurrences())
3711 return ConsiderRegPressure;
3712
3713 // TODO: We should eventually consider register pressure for all targets. The
3714 // TTI hook is temporary whilst target-specific issues are being fixed.
3715 if (TTI.shouldConsiderVectorizationRegPressure())
3716 return true;
3717
3718 if (!useMaxBandwidth(VF.isScalable()
3721 return false;
3722 // Only calculate register pressure for VFs enabled by MaxBandwidth.
3724 VF, VF.isScalable() ? MaxPermissibleVFWithoutMaxBW.ScalableVF
3726}
3727
3730 return MaximizeBandwidth || (MaximizeBandwidth.getNumOccurrences() == 0 &&
3731 (TTI.shouldMaximizeVectorBandwidth(RegKind) ||
3733 Legal->hasVectorCallVariants())));
3734}
3735
3736ElementCount LoopVectorizationCostModel::clampVFByMaxTripCount(
3737 ElementCount VF, unsigned MaxTripCount, bool FoldTailByMasking) const {
3738 unsigned EstimatedVF = VF.getKnownMinValue();
3739 if (VF.isScalable() && TheFunction->hasFnAttribute(Attribute::VScaleRange)) {
3740 auto Attr = TheFunction->getFnAttribute(Attribute::VScaleRange);
3741 auto Min = Attr.getVScaleRangeMin();
3742 EstimatedVF *= Min;
3743 }
3744
3745 // When a scalar epilogue is required, at least one iteration of the scalar
3746 // loop has to execute. Adjust MaxTripCount accordingly to avoid picking a
3747 // max VF that results in a dead vector loop.
3748 if (MaxTripCount > 0 && requiresScalarEpilogue(true))
3749 MaxTripCount -= 1;
3750
3751 if (MaxTripCount && MaxTripCount <= EstimatedVF &&
3752 (!FoldTailByMasking || isPowerOf2_32(MaxTripCount))) {
3753 // If upper bound loop trip count (TC) is known at compile time there is no
3754 // point in choosing VF greater than TC (as done in the loop below). Select
3755 // maximum power of two which doesn't exceed TC. If VF is
3756 // scalable, we only fall back on a fixed VF when the TC is less than or
3757 // equal to the known number of lanes.
3758 auto ClampedUpperTripCount = llvm::bit_floor(MaxTripCount);
3759 LLVM_DEBUG(dbgs() << "LV: Clamping the MaxVF to maximum power of two not "
3760 "exceeding the constant trip count: "
3761 << ClampedUpperTripCount << "\n");
3762 return ElementCount::get(ClampedUpperTripCount,
3763 FoldTailByMasking ? VF.isScalable() : false);
3764 }
3765 return VF;
3766}
3767
3768ElementCount LoopVectorizationCostModel::getMaximizedVFForTarget(
3769 unsigned MaxTripCount, unsigned SmallestType, unsigned WidestType,
3770 ElementCount MaxSafeVF, bool FoldTailByMasking) {
3771 bool ComputeScalableMaxVF = MaxSafeVF.isScalable();
3772 const TypeSize WidestRegister = TTI.getRegisterBitWidth(
3773 ComputeScalableMaxVF ? TargetTransformInfo::RGK_ScalableVector
3775
3776 // Convenience function to return the minimum of two ElementCounts.
3777 auto MinVF = [](const ElementCount &LHS, const ElementCount &RHS) {
3778 assert((LHS.isScalable() == RHS.isScalable()) &&
3779 "Scalable flags must match");
3780 return ElementCount::isKnownLT(LHS, RHS) ? LHS : RHS;
3781 };
3782
3783 // Ensure MaxVF is a power of 2; the dependence distance bound may not be.
3784 // Note that both WidestRegister and WidestType may not be a powers of 2.
3785 auto MaxVectorElementCount = ElementCount::get(
3786 llvm::bit_floor(WidestRegister.getKnownMinValue() / WidestType),
3787 ComputeScalableMaxVF);
3788 MaxVectorElementCount = MinVF(MaxVectorElementCount, MaxSafeVF);
3789 LLVM_DEBUG(dbgs() << "LV: The Widest register safe to use is: "
3790 << (MaxVectorElementCount * WidestType) << " bits.\n");
3791
3792 if (!MaxVectorElementCount) {
3793 LLVM_DEBUG(dbgs() << "LV: The target has no "
3794 << (ComputeScalableMaxVF ? "scalable" : "fixed")
3795 << " vector registers.\n");
3796 return ElementCount::getFixed(1);
3797 }
3798
3799 ElementCount MaxVF = clampVFByMaxTripCount(MaxVectorElementCount,
3800 MaxTripCount, FoldTailByMasking);
3801 // If the MaxVF was already clamped, there's no point in trying to pick a
3802 // larger one.
3803 if (MaxVF != MaxVectorElementCount)
3804 return MaxVF;
3805
3807 ComputeScalableMaxVF ? TargetTransformInfo::RGK_ScalableVector
3809
3810 if (MaxVF.isScalable())
3811 MaxPermissibleVFWithoutMaxBW.ScalableVF = MaxVF;
3812 else
3813 MaxPermissibleVFWithoutMaxBW.FixedVF = MaxVF;
3814
3815 if (useMaxBandwidth(RegKind)) {
3816 auto MaxVectorElementCountMaxBW = ElementCount::get(
3817 llvm::bit_floor(WidestRegister.getKnownMinValue() / SmallestType),
3818 ComputeScalableMaxVF);
3819 MaxVF = MinVF(MaxVectorElementCountMaxBW, MaxSafeVF);
3820
3821 if (ElementCount MinVF =
3822 TTI.getMinimumVF(SmallestType, ComputeScalableMaxVF)) {
3823 if (ElementCount::isKnownLT(MaxVF, MinVF)) {
3824 LLVM_DEBUG(dbgs() << "LV: Overriding calculated MaxVF(" << MaxVF
3825 << ") with target's minimum: " << MinVF << '\n');
3826 MaxVF = MinVF;
3827 }
3828 }
3829
3830 MaxVF = clampVFByMaxTripCount(MaxVF, MaxTripCount, FoldTailByMasking);
3831
3832 if (MaxVectorElementCount != MaxVF) {
3833 // Invalidate any widening decisions we might have made, in case the loop
3834 // requires prediction (decided later), but we have already made some
3835 // load/store widening decisions.
3837 }
3838 }
3839 return MaxVF;
3840}
3841
3842bool LoopVectorizationPlanner::isMoreProfitable(const VectorizationFactor &A,
3843 const VectorizationFactor &B,
3844 const unsigned MaxTripCount,
3845 bool HasTail,
3846 bool IsEpilogue) const {
3847 InstructionCost CostA = A.Cost;
3848 InstructionCost CostB = B.Cost;
3849
3850 // Improve estimate for the vector width if it is scalable.
3851 unsigned EstimatedWidthA = A.Width.getKnownMinValue();
3852 unsigned EstimatedWidthB = B.Width.getKnownMinValue();
3853 if (std::optional<unsigned> VScale = CM.getVScaleForTuning()) {
3854 if (A.Width.isScalable())
3855 EstimatedWidthA *= *VScale;
3856 if (B.Width.isScalable())
3857 EstimatedWidthB *= *VScale;
3858 }
3859
3860 // When optimizing for size choose whichever is smallest, which will be the
3861 // one with the smallest cost for the whole loop. On a tie pick the larger
3862 // vector width, on the assumption that throughput will be greater.
3863 if (CM.CostKind == TTI::TCK_CodeSize)
3864 return CostA < CostB ||
3865 (CostA == CostB && EstimatedWidthA > EstimatedWidthB);
3866
3867 // Assume vscale may be larger than 1 (or the value being tuned for),
3868 // so that scalable vectorization is slightly favorable over fixed-width
3869 // vectorization.
3870 bool PreferScalable = !TTI.preferFixedOverScalableIfEqualCost(IsEpilogue) &&
3871 A.Width.isScalable() && !B.Width.isScalable();
3872
3873 auto CmpFn = [PreferScalable](const InstructionCost &LHS,
3874 const InstructionCost &RHS) {
3875 return PreferScalable ? LHS <= RHS : LHS < RHS;
3876 };
3877
3878 // To avoid the need for FP division:
3879 // (CostA / EstimatedWidthA) < (CostB / EstimatedWidthB)
3880 // <=> (CostA * EstimatedWidthB) < (CostB * EstimatedWidthA)
3881 if (!MaxTripCount)
3882 return CmpFn(CostA * EstimatedWidthB, CostB * EstimatedWidthA);
3883
3884 auto GetCostForTC = [MaxTripCount, HasTail](unsigned VF,
3885 InstructionCost VectorCost,
3886 InstructionCost ScalarCost) {
3887 // If the trip count is a known (possibly small) constant, the trip count
3888 // will be rounded up to an integer number of iterations under
3889 // FoldTailByMasking. The total cost in that case will be
3890 // VecCost*ceil(TripCount/VF). When not folding the tail, the total
3891 // cost will be VecCost*floor(TC/VF) + ScalarCost*(TC%VF). There will be
3892 // some extra overheads, but for the purpose of comparing the costs of
3893 // different VFs we can use this to compare the total loop-body cost
3894 // expected after vectorization.
3895 if (HasTail)
3896 return VectorCost * (MaxTripCount / VF) +
3897 ScalarCost * (MaxTripCount % VF);
3898 return VectorCost * divideCeil(MaxTripCount, VF);
3899 };
3900
3901 auto RTCostA = GetCostForTC(EstimatedWidthA, CostA, A.ScalarCost);
3902 auto RTCostB = GetCostForTC(EstimatedWidthB, CostB, B.ScalarCost);
3903 return CmpFn(RTCostA, RTCostB);
3904}
3905
3906bool LoopVectorizationPlanner::isMoreProfitable(const VectorizationFactor &A,
3907 const VectorizationFactor &B,
3908 bool HasTail,
3909 bool IsEpilogue) const {
3910 const unsigned MaxTripCount = PSE.getSmallConstantMaxTripCount();
3911 return LoopVectorizationPlanner::isMoreProfitable(A, B, MaxTripCount, HasTail,
3912 IsEpilogue);
3913}
3914
3917 using RecipeVFPair = std::pair<VPRecipeBase *, ElementCount>;
3918 SmallVector<RecipeVFPair> InvalidCosts;
3919 for (const auto &Plan : VPlans) {
3920 for (ElementCount VF : Plan->vectorFactors()) {
3921 // The VPlan-based cost model is designed for computing vector cost.
3922 // Querying VPlan-based cost model with a scarlar VF will cause some
3923 // errors because we expect the VF is vector for most of the widen
3924 // recipes.
3925 if (VF.isScalar())
3926 continue;
3927
3928 VPCostContext CostCtx(CM.TTI, *CM.TLI, *Plan, CM, CM.CostKind);
3929 precomputeCosts(*Plan, VF, CostCtx);
3930 auto Iter = vp_depth_first_deep(Plan->getVectorLoopRegion()->getEntry());
3932 for (auto &R : *VPBB) {
3933 if (!R.cost(VF, CostCtx).isValid())
3934 InvalidCosts.emplace_back(&R, VF);
3935 }
3936 }
3937 }
3938 }
3939 if (InvalidCosts.empty())
3940 return;
3941
3942 // Emit a report of VFs with invalid costs in the loop.
3943
3944 // Group the remarks per recipe, keeping the recipe order from InvalidCosts.
3946 unsigned I = 0;
3947 for (auto &Pair : InvalidCosts)
3948 if (Numbering.try_emplace(Pair.first, I).second)
3949 ++I;
3950
3951 // Sort the list, first on recipe(number) then on VF.
3952 sort(InvalidCosts, [&Numbering](RecipeVFPair &A, RecipeVFPair &B) {
3953 unsigned NA = Numbering[A.first];
3954 unsigned NB = Numbering[B.first];
3955 if (NA != NB)
3956 return NA < NB;
3957 return ElementCount::isKnownLT(A.second, B.second);
3958 });
3959
3960 // For a list of ordered recipe-VF pairs:
3961 // [(load, VF1), (load, VF2), (store, VF1)]
3962 // group the recipes together to emit separate remarks for:
3963 // load (VF1, VF2)
3964 // store (VF1)
3965 auto Tail = ArrayRef<RecipeVFPair>(InvalidCosts);
3966 auto Subset = ArrayRef<RecipeVFPair>();
3967 do {
3968 if (Subset.empty())
3969 Subset = Tail.take_front(1);
3970
3971 VPRecipeBase *R = Subset.front().first;
3972
3973 unsigned Opcode =
3976 [](const auto *R) { return Instruction::PHI; })
3977 .Case<VPWidenSelectRecipe>(
3978 [](const auto *R) { return Instruction::Select; })
3979 .Case<VPWidenStoreRecipe>(
3980 [](const auto *R) { return Instruction::Store; })
3981 .Case<VPWidenLoadRecipe>(
3982 [](const auto *R) { return Instruction::Load; })
3983 .Case<VPWidenCallRecipe, VPWidenIntrinsicRecipe>(
3984 [](const auto *R) { return Instruction::Call; })
3987 [](const auto *R) { return R->getOpcode(); })
3988 .Case<VPInterleaveRecipe>([](const VPInterleaveRecipe *R) {
3989 return R->getStoredValues().empty() ? Instruction::Load
3990 : Instruction::Store;
3991 });
3992
3993 // If the next recipe is different, or if there are no other pairs,
3994 // emit a remark for the collated subset. e.g.
3995 // [(load, VF1), (load, VF2))]
3996 // to emit:
3997 // remark: invalid costs for 'load' at VF=(VF1, VF2)
3998 if (Subset == Tail || Tail[Subset.size()].first != R) {
3999 std::string OutString;
4000 raw_string_ostream OS(OutString);
4001 assert(!Subset.empty() && "Unexpected empty range");
4002 OS << "Recipe with invalid costs prevented vectorization at VF=(";
4003 for (const auto &Pair : Subset)
4004 OS << (Pair.second == Subset.front().second ? "" : ", ") << Pair.second;
4005 OS << "):";
4006 if (Opcode == Instruction::Call) {
4007 StringRef Name = "";
4008 if (auto *Int = dyn_cast<VPWidenIntrinsicRecipe>(R)) {
4009 Name = Int->getIntrinsicName();
4010 } else {
4011 auto *WidenCall = dyn_cast<VPWidenCallRecipe>(R);
4012 Function *CalledFn =
4013 WidenCall ? WidenCall->getCalledScalarFunction()
4014 : cast<Function>(R->getOperand(R->getNumOperands() - 1)
4015 ->getLiveInIRValue());
4016 Name = CalledFn->getName();
4017 }
4018 OS << " call to " << Name;
4019 } else
4020 OS << " " << Instruction::getOpcodeName(Opcode);
4021 reportVectorizationInfo(OutString, "InvalidCost", ORE, OrigLoop, nullptr,
4022 R->getDebugLoc());
4023 Tail = Tail.drop_front(Subset.size());
4024 Subset = {};
4025 } else
4026 // Grow the subset by one element
4027 Subset = Tail.take_front(Subset.size() + 1);
4028 } while (!Tail.empty());
4029}
4030
4031/// Check if any recipe of \p Plan will generate a vector value, which will be
4032/// assigned a vector register.
4034 const TargetTransformInfo &TTI) {
4035 assert(VF.isVector() && "Checking a scalar VF?");
4036 VPTypeAnalysis TypeInfo(Plan);
4037 DenseSet<VPRecipeBase *> EphemeralRecipes;
4038 collectEphemeralRecipesForVPlan(Plan, EphemeralRecipes);
4039 // Set of already visited types.
4040 DenseSet<Type *> Visited;
4043 for (VPRecipeBase &R : *VPBB) {
4044 if (EphemeralRecipes.contains(&R))
4045 continue;
4046 // Continue early if the recipe is considered to not produce a vector
4047 // result. Note that this includes VPInstruction where some opcodes may
4048 // produce a vector, to preserve existing behavior as VPInstructions model
4049 // aspects not directly mapped to existing IR instructions.
4050 switch (R.getVPDefID()) {
4051 case VPDef::VPDerivedIVSC:
4052 case VPDef::VPScalarIVStepsSC:
4053 case VPDef::VPReplicateSC:
4054 case VPDef::VPInstructionSC:
4055 case VPDef::VPCanonicalIVPHISC:
4056 case VPDef::VPVectorPointerSC:
4057 case VPDef::VPVectorEndPointerSC:
4058 case VPDef::VPExpandSCEVSC:
4059 case VPDef::VPEVLBasedIVPHISC:
4060 case VPDef::VPPredInstPHISC:
4061 case VPDef::VPBranchOnMaskSC:
4062 continue;
4063 case VPDef::VPReductionSC:
4064 case VPDef::VPActiveLaneMaskPHISC:
4065 case VPDef::VPWidenCallSC:
4066 case VPDef::VPWidenCanonicalIVSC:
4067 case VPDef::VPWidenCastSC:
4068 case VPDef::VPWidenGEPSC:
4069 case VPDef::VPWidenIntrinsicSC:
4070 case VPDef::VPWidenSC:
4071 case VPDef::VPWidenSelectSC:
4072 case VPDef::VPBlendSC:
4073 case VPDef::VPFirstOrderRecurrencePHISC:
4074 case VPDef::VPHistogramSC:
4075 case VPDef::VPWidenPHISC:
4076 case VPDef::VPWidenIntOrFpInductionSC:
4077 case VPDef::VPWidenPointerInductionSC:
4078 case VPDef::VPReductionPHISC:
4079 case VPDef::VPInterleaveEVLSC:
4080 case VPDef::VPInterleaveSC:
4081 case VPDef::VPWidenLoadEVLSC:
4082 case VPDef::VPWidenLoadSC:
4083 case VPDef::VPWidenStoreEVLSC:
4084 case VPDef::VPWidenStoreSC:
4085 break;
4086 default:
4087 llvm_unreachable("unhandled recipe");
4088 }
4089
4090 auto WillGenerateTargetVectors = [&TTI, VF](Type *VectorTy) {
4091 unsigned NumLegalParts = TTI.getNumberOfParts(VectorTy);
4092 if (!NumLegalParts)
4093 return false;
4094 if (VF.isScalable()) {
4095 // <vscale x 1 x iN> is assumed to be profitable over iN because
4096 // scalable registers are a distinct register class from scalar
4097 // ones. If we ever find a target which wants to lower scalable
4098 // vectors back to scalars, we'll need to update this code to
4099 // explicitly ask TTI about the register class uses for each part.
4100 return NumLegalParts <= VF.getKnownMinValue();
4101 }
4102 // Two or more elements that share a register - are vectorized.
4103 return NumLegalParts < VF.getFixedValue();
4104 };
4105
4106 // If no def nor is a store, e.g., branches, continue - no value to check.
4107 if (R.getNumDefinedValues() == 0 &&
4109 continue;
4110 // For multi-def recipes, currently only interleaved loads, suffice to
4111 // check first def only.
4112 // For stores check their stored value; for interleaved stores suffice
4113 // the check first stored value only. In all cases this is the second
4114 // operand.
4115 VPValue *ToCheck =
4116 R.getNumDefinedValues() >= 1 ? R.getVPValue(0) : R.getOperand(1);
4117 Type *ScalarTy = TypeInfo.inferScalarType(ToCheck);
4118 if (!Visited.insert({ScalarTy}).second)
4119 continue;
4120 Type *WideTy = toVectorizedTy(ScalarTy, VF);
4121 if (any_of(getContainedTypes(WideTy), WillGenerateTargetVectors))
4122 return true;
4123 }
4124 }
4125
4126 return false;
4127}
4128
4129static bool hasReplicatorRegion(VPlan &Plan) {
4131 Plan.getVectorLoopRegion()->getEntry())),
4132 [](auto *VPRB) { return VPRB->isReplicator(); });
4133}
4134
4135#ifndef NDEBUG
4136VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() {
4137 InstructionCost ExpectedCost = CM.expectedCost(ElementCount::getFixed(1));
4138 LLVM_DEBUG(dbgs() << "LV: Scalar loop costs: " << ExpectedCost << ".\n");
4139 assert(ExpectedCost.isValid() && "Unexpected invalid cost for scalar loop");
4140 assert(
4141 any_of(VPlans,
4142 [](std::unique_ptr<VPlan> &P) { return P->hasScalarVFOnly(); }) &&
4143 "Expected Scalar VF to be a candidate");
4144
4145 const VectorizationFactor ScalarCost(ElementCount::getFixed(1), ExpectedCost,
4146 ExpectedCost);
4147 VectorizationFactor ChosenFactor = ScalarCost;
4148
4149 bool ForceVectorization = Hints.getForce() == LoopVectorizeHints::FK_Enabled;
4150 if (ForceVectorization &&
4151 (VPlans.size() > 1 || !VPlans[0]->hasScalarVFOnly())) {
4152 // Ignore scalar width, because the user explicitly wants vectorization.
4153 // Initialize cost to max so that VF = 2 is, at least, chosen during cost
4154 // evaluation.
4155 ChosenFactor.Cost = InstructionCost::getMax();
4156 }
4157
4158 for (auto &P : VPlans) {
4159 ArrayRef<ElementCount> VFs(P->vectorFactors().begin(),
4160 P->vectorFactors().end());
4161
4163 if (any_of(VFs, [this](ElementCount VF) {
4164 return CM.shouldConsiderRegPressureForVF(VF);
4165 }))
4166 RUs = calculateRegisterUsageForPlan(*P, VFs, TTI, CM.ValuesToIgnore);
4167
4168 for (unsigned I = 0; I < VFs.size(); I++) {
4169 ElementCount VF = VFs[I];
4170 // The cost for scalar VF=1 is already calculated, so ignore it.
4171 if (VF.isScalar())
4172 continue;
4173
4174 /// If the register pressure needs to be considered for VF,
4175 /// don't consider the VF as valid if it exceeds the number
4176 /// of registers for the target.
4177 if (CM.shouldConsiderRegPressureForVF(VF) &&
4178 RUs[I].exceedsMaxNumRegs(TTI, ForceTargetNumVectorRegs))
4179 continue;
4180
4181 InstructionCost C = CM.expectedCost(VF);
4182
4183 // Add on other costs that are modelled in VPlan, but not in the legacy
4184 // cost model.
4185 VPCostContext CostCtx(CM.TTI, *CM.TLI, *P, CM, CM.CostKind);
4186 VPRegionBlock *VectorRegion = P->getVectorLoopRegion();
4187 assert(VectorRegion && "Expected to have a vector region!");
4188 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
4189 vp_depth_first_shallow(VectorRegion->getEntry()))) {
4190 for (VPRecipeBase &R : *VPBB) {
4191 auto *VPI = dyn_cast<VPInstruction>(&R);
4192 if (!VPI)
4193 continue;
4194 switch (VPI->getOpcode()) {
4195 // Selects are only modelled in the legacy cost model for safe
4196 // divisors.
4197 case Instruction::Select: {
4198 VPValue *VPV = VPI->getVPSingleValue();
4199 if (VPV->getNumUsers() == 1) {
4200 if (auto *WR = dyn_cast<VPWidenRecipe>(*VPV->user_begin())) {
4201 switch (WR->getOpcode()) {
4202 case Instruction::UDiv:
4203 case Instruction::SDiv:
4204 case Instruction::URem:
4205 case Instruction::SRem:
4206 continue;
4207 default:
4208 break;
4209 }
4210 }
4211 }
4212 C += VPI->cost(VF, CostCtx);
4213 break;
4214 }
4216 unsigned Multiplier =
4217 cast<ConstantInt>(VPI->getOperand(2)->getLiveInIRValue())
4218 ->getZExtValue();
4219 C += VPI->cost(VF * Multiplier, CostCtx);
4220 break;
4221 }
4223 C += VPI->cost(VF, CostCtx);
4224 break;
4225 default:
4226 break;
4227 }
4228 }
4229 }
4230
4231 VectorizationFactor Candidate(VF, C, ScalarCost.ScalarCost);
4232 unsigned Width =
4233 estimateElementCount(Candidate.Width, CM.getVScaleForTuning());
4234 LLVM_DEBUG(dbgs() << "LV: Vector loop of width " << VF
4235 << " costs: " << (Candidate.Cost / Width));
4236 if (VF.isScalable())
4237 LLVM_DEBUG(dbgs() << " (assuming a minimum vscale of "
4238 << CM.getVScaleForTuning().value_or(1) << ")");
4239 LLVM_DEBUG(dbgs() << ".\n");
4240
4241 if (!ForceVectorization && !willGenerateVectors(*P, VF, TTI)) {
4242 LLVM_DEBUG(
4243 dbgs()
4244 << "LV: Not considering vector loop of width " << VF
4245 << " because it will not generate any vector instructions.\n");
4246 continue;
4247 }
4248
4249 if (CM.OptForSize && !ForceVectorization && hasReplicatorRegion(*P)) {
4250 LLVM_DEBUG(
4251 dbgs()
4252 << "LV: Not considering vector loop of width " << VF
4253 << " because it would cause replicated blocks to be generated,"
4254 << " which isn't allowed when optimizing for size.\n");
4255 continue;
4256 }
4257
4258 if (isMoreProfitable(Candidate, ChosenFactor, P->hasScalarTail()))
4259 ChosenFactor = Candidate;
4260 }
4261 }
4262
4263 if (!EnableCondStoresVectorization && CM.hasPredStores()) {
4265 "There are conditional stores.",
4266 "store that is conditionally executed prevents vectorization",
4267 "ConditionalStore", ORE, OrigLoop);
4268 ChosenFactor = ScalarCost;
4269 }
4270
4271 LLVM_DEBUG(if (ForceVectorization && !ChosenFactor.Width.isScalar() &&
4272 !isMoreProfitable(ChosenFactor, ScalarCost,
4273 !CM.foldTailByMasking())) dbgs()
4274 << "LV: Vectorization seems to be not beneficial, "
4275 << "but was forced by a user.\n");
4276 return ChosenFactor;
4277}
4278#endif
4279
4280bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
4281 ElementCount VF) const {
4282 // Cross iteration phis such as fixed-order recurrences and FMaxNum/FMinNum
4283 // reductions need special handling and are currently unsupported.
4284 if (any_of(OrigLoop->getHeader()->phis(), [&](PHINode &Phi) {
4285 if (!Legal->isReductionVariable(&Phi))
4286 return Legal->isFixedOrderRecurrence(&Phi);
4287 RecurKind RK = Legal->getRecurrenceDescriptor(&Phi).getRecurrenceKind();
4288 return RK == RecurKind::FMinNum || RK == RecurKind::FMaxNum;
4289 }))
4290 return false;
4291
4292 // Phis with uses outside of the loop require special handling and are
4293 // currently unsupported.
4294 for (const auto &Entry : Legal->getInductionVars()) {
4295 // Look for uses of the value of the induction at the last iteration.
4296 Value *PostInc =
4297 Entry.first->getIncomingValueForBlock(OrigLoop->getLoopLatch());
4298 for (User *U : PostInc->users())
4299 if (!OrigLoop->contains(cast<Instruction>(U)))
4300 return false;
4301 // Look for uses of penultimate value of the induction.
4302 for (User *U : Entry.first->users())
4303 if (!OrigLoop->contains(cast<Instruction>(U)))
4304 return false;
4305 }
4306
4307 // Epilogue vectorization code has not been auditted to ensure it handles
4308 // non-latch exits properly. It may be fine, but it needs auditted and
4309 // tested.
4310 // TODO: Add support for loops with an early exit.
4311 if (OrigLoop->getExitingBlock() != OrigLoop->getLoopLatch())
4312 return false;
4313
4314 return true;
4315}
4316
4318 const ElementCount VF, const unsigned IC) const {
4319 // FIXME: We need a much better cost-model to take different parameters such
4320 // as register pressure, code size increase and cost of extra branches into
4321 // account. For now we apply a very crude heuristic and only consider loops
4322 // with vectorization factors larger than a certain value.
4323
4324 // Allow the target to opt out entirely.
4325 if (!TTI.preferEpilogueVectorization())
4326 return false;
4327
4328 // We also consider epilogue vectorization unprofitable for targets that don't
4329 // consider interleaving beneficial (eg. MVE).
4330 if (TTI.getMaxInterleaveFactor(VF) <= 1)
4331 return false;
4332
4333 unsigned MinVFThreshold = EpilogueVectorizationMinVF.getNumOccurrences() > 0
4335 : TTI.getEpilogueVectorizationMinVF();
4336 return estimateElementCount(VF * IC, VScaleForTuning) >= MinVFThreshold;
4337}
4338
4340 const ElementCount MainLoopVF, unsigned IC) {
4343 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is disabled.\n");
4344 return Result;
4345 }
4346
4347 if (!CM.isScalarEpilogueAllowed()) {
4348 LLVM_DEBUG(dbgs() << "LEV: Unable to vectorize epilogue because no "
4349 "epilogue is allowed.\n");
4350 return Result;
4351 }
4352
4353 // Not really a cost consideration, but check for unsupported cases here to
4354 // simplify the logic.
4355 if (!isCandidateForEpilogueVectorization(MainLoopVF)) {
4356 LLVM_DEBUG(dbgs() << "LEV: Unable to vectorize epilogue because the loop "
4357 "is not a supported candidate.\n");
4358 return Result;
4359 }
4360
4362 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization factor is forced.\n");
4364 if (hasPlanWithVF(ForcedEC))
4365 return {ForcedEC, 0, 0};
4366
4367 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization forced factor is not "
4368 "viable.\n");
4369 return Result;
4370 }
4371
4372 if (OrigLoop->getHeader()->getParent()->hasOptSize()) {
4373 LLVM_DEBUG(
4374 dbgs() << "LEV: Epilogue vectorization skipped due to opt for size.\n");
4375 return Result;
4376 }
4377
4378 if (!CM.isEpilogueVectorizationProfitable(MainLoopVF, IC)) {
4379 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is not profitable for "
4380 "this loop\n");
4381 return Result;
4382 }
4383
4384 // If MainLoopVF = vscale x 2, and vscale is expected to be 4, then we know
4385 // the main loop handles 8 lanes per iteration. We could still benefit from
4386 // vectorizing the epilogue loop with VF=4.
4387 ElementCount EstimatedRuntimeVF = ElementCount::getFixed(
4388 estimateElementCount(MainLoopVF, CM.getVScaleForTuning()));
4389
4390 ScalarEvolution &SE = *PSE.getSE();
4391 Type *TCType = Legal->getWidestInductionType();
4392 const SCEV *RemainingIterations = nullptr;
4393 unsigned MaxTripCount = 0;
4394 const SCEV *TC =
4395 vputils::getSCEVExprForVPValue(getPlanFor(MainLoopVF).getTripCount(), SE);
4396 assert(!isa<SCEVCouldNotCompute>(TC) && "Trip count SCEV must be computable");
4397 RemainingIterations =
4398 SE.getURemExpr(TC, SE.getElementCount(TCType, MainLoopVF * IC));
4399
4400 // No iterations left to process in the epilogue.
4401 if (RemainingIterations->isZero())
4402 return Result;
4403
4404 if (MainLoopVF.isFixed()) {
4405 MaxTripCount = MainLoopVF.getFixedValue() * IC - 1;
4406 if (SE.isKnownPredicate(CmpInst::ICMP_ULT, RemainingIterations,
4407 SE.getConstant(TCType, MaxTripCount))) {
4408 MaxTripCount = SE.getUnsignedRangeMax(RemainingIterations).getZExtValue();
4409 }
4410 LLVM_DEBUG(dbgs() << "LEV: Maximum Trip Count for Epilogue: "
4411 << MaxTripCount << "\n");
4412 }
4413
4414 for (auto &NextVF : ProfitableVFs) {
4415 // Skip candidate VFs without a corresponding VPlan.
4416 if (!hasPlanWithVF(NextVF.Width))
4417 continue;
4418
4419 // Skip candidate VFs with widths >= the (estimated) runtime VF (scalable
4420 // vectors) or > the VF of the main loop (fixed vectors).
4421 if ((!NextVF.Width.isScalable() && MainLoopVF.isScalable() &&
4422 ElementCount::isKnownGE(NextVF.Width, EstimatedRuntimeVF)) ||
4423 (NextVF.Width.isScalable() &&
4424 ElementCount::isKnownGE(NextVF.Width, MainLoopVF)) ||
4425 (!NextVF.Width.isScalable() && !MainLoopVF.isScalable() &&
4426 ElementCount::isKnownGT(NextVF.Width, MainLoopVF)))
4427 continue;
4428
4429 // If NextVF is greater than the number of remaining iterations, the
4430 // epilogue loop would be dead. Skip such factors.
4431 if (RemainingIterations && !NextVF.Width.isScalable()) {
4432 if (SE.isKnownPredicate(
4434 SE.getConstant(TCType, NextVF.Width.getFixedValue()),
4435 RemainingIterations))
4436 continue;
4437 }
4438
4439 if (Result.Width.isScalar() ||
4440 isMoreProfitable(NextVF, Result, MaxTripCount, !CM.foldTailByMasking(),
4441 /*IsEpilogue*/ true))
4442 Result = NextVF;
4443 }
4444
4445 if (Result != VectorizationFactor::Disabled())
4446 LLVM_DEBUG(dbgs() << "LEV: Vectorizing epilogue loop with VF = "
4447 << Result.Width << "\n");
4448 return Result;
4449}
4450
4451std::pair<unsigned, unsigned>
4453 unsigned MinWidth = -1U;
4454 unsigned MaxWidth = 8;
4455 const DataLayout &DL = TheFunction->getDataLayout();
4456 // For in-loop reductions, no element types are added to ElementTypesInLoop
4457 // if there are no loads/stores in the loop. In this case, check through the
4458 // reduction variables to determine the maximum width.
4459 if (ElementTypesInLoop.empty() && !Legal->getReductionVars().empty()) {
4460 for (const auto &PhiDescriptorPair : Legal->getReductionVars()) {
4461 const RecurrenceDescriptor &RdxDesc = PhiDescriptorPair.second;
4462 // When finding the min width used by the recurrence we need to account
4463 // for casts on the input operands of the recurrence.
4464 MinWidth = std::min(
4465 MinWidth,
4466 std::min(RdxDesc.getMinWidthCastToRecurrenceTypeInBits(),
4468 MaxWidth = std::max(MaxWidth,
4470 }
4471 } else {
4472 for (Type *T : ElementTypesInLoop) {
4473 MinWidth = std::min<unsigned>(
4474 MinWidth, DL.getTypeSizeInBits(T->getScalarType()).getFixedValue());
4475 MaxWidth = std::max<unsigned>(
4476 MaxWidth, DL.getTypeSizeInBits(T->getScalarType()).getFixedValue());
4477 }
4478 }
4479 return {MinWidth, MaxWidth};
4480}
4481
4483 ElementTypesInLoop.clear();
4484 // For each block.
4485 for (BasicBlock *BB : TheLoop->blocks()) {
4486 // For each instruction in the loop.
4487 for (Instruction &I : BB->instructionsWithoutDebug()) {
4488 Type *T = I.getType();
4489
4490 // Skip ignored values.
4491 if (ValuesToIgnore.count(&I))
4492 continue;
4493
4494 // Only examine Loads, Stores and PHINodes.
4495 if (!isa<LoadInst>(I) && !isa<StoreInst>(I) && !isa<PHINode>(I))
4496 continue;
4497
4498 // Examine PHI nodes that are reduction variables. Update the type to
4499 // account for the recurrence type.
4500 if (auto *PN = dyn_cast<PHINode>(&I)) {
4501 if (!Legal->isReductionVariable(PN))
4502 continue;
4503 const RecurrenceDescriptor &RdxDesc =
4504 Legal->getRecurrenceDescriptor(PN);
4506 TTI.preferInLoopReduction(RdxDesc.getRecurrenceKind(),
4507 RdxDesc.getRecurrenceType()))
4508 continue;
4509 T = RdxDesc.getRecurrenceType();
4510 }
4511
4512 // Examine the stored values.
4513 if (auto *ST = dyn_cast<StoreInst>(&I))
4514 T = ST->getValueOperand()->getType();
4515
4516 assert(T->isSized() &&
4517 "Expected the load/store/recurrence type to be sized");
4518
4519 ElementTypesInLoop.insert(T);
4520 }
4521 }
4522}
4523
4524unsigned
4526 InstructionCost LoopCost) {
4527 // -- The interleave heuristics --
4528 // We interleave the loop in order to expose ILP and reduce the loop overhead.
4529 // There are many micro-architectural considerations that we can't predict
4530 // at this level. For example, frontend pressure (on decode or fetch) due to
4531 // code size, or the number and capabilities of the execution ports.
4532 //
4533 // We use the following heuristics to select the interleave count:
4534 // 1. If the code has reductions, then we interleave to break the cross
4535 // iteration dependency.
4536 // 2. If the loop is really small, then we interleave to reduce the loop
4537 // overhead.
4538 // 3. We don't interleave if we think that we will spill registers to memory
4539 // due to the increased register pressure.
4540
4541 if (!CM.isScalarEpilogueAllowed())
4542 return 1;
4543
4546 LLVM_DEBUG(dbgs() << "LV: Preference for VP intrinsics indicated. "
4547 "Unroll factor forced to be 1.\n");
4548 return 1;
4549 }
4550
4551 // We used the distance for the interleave count.
4552 if (!Legal->isSafeForAnyVectorWidth())
4553 return 1;
4554
4555 // We don't attempt to perform interleaving for loops with uncountable early
4556 // exits because the VPInstruction::AnyOf code cannot currently handle
4557 // multiple parts.
4558 if (Plan.hasEarlyExit())
4559 return 1;
4560
4561 const bool HasReductions =
4564
4565 // If we did not calculate the cost for VF (because the user selected the VF)
4566 // then we calculate the cost of VF here.
4567 if (LoopCost == 0) {
4568 if (VF.isScalar())
4569 LoopCost = CM.expectedCost(VF);
4570 else
4571 LoopCost = cost(Plan, VF);
4572 assert(LoopCost.isValid() && "Expected to have chosen a VF with valid cost");
4573
4574 // Loop body is free and there is no need for interleaving.
4575 if (LoopCost == 0)
4576 return 1;
4577 }
4578
4579 VPRegisterUsage R =
4580 calculateRegisterUsageForPlan(Plan, {VF}, TTI, CM.ValuesToIgnore)[0];
4581 // We divide by these constants so assume that we have at least one
4582 // instruction that uses at least one register.
4583 for (auto &Pair : R.MaxLocalUsers) {
4584 Pair.second = std::max(Pair.second, 1U);
4585 }
4586
4587 // We calculate the interleave count using the following formula.
4588 // Subtract the number of loop invariants from the number of available
4589 // registers. These registers are used by all of the interleaved instances.
4590 // Next, divide the remaining registers by the number of registers that is
4591 // required by the loop, in order to estimate how many parallel instances
4592 // fit without causing spills. All of this is rounded down if necessary to be
4593 // a power of two. We want power of two interleave count to simplify any
4594 // addressing operations or alignment considerations.
4595 // We also want power of two interleave counts to ensure that the induction
4596 // variable of the vector loop wraps to zero, when tail is folded by masking;
4597 // this currently happens when OptForSize, in which case IC is set to 1 above.
4598 unsigned IC = UINT_MAX;
4599
4600 for (const auto &Pair : R.MaxLocalUsers) {
4601 unsigned TargetNumRegisters = TTI.getNumberOfRegisters(Pair.first);
4602 LLVM_DEBUG(dbgs() << "LV: The target has " << TargetNumRegisters
4603 << " registers of "
4604 << TTI.getRegisterClassName(Pair.first)
4605 << " register class\n");
4606 if (VF.isScalar()) {
4607 if (ForceTargetNumScalarRegs.getNumOccurrences() > 0)
4608 TargetNumRegisters = ForceTargetNumScalarRegs;
4609 } else {
4610 if (ForceTargetNumVectorRegs.getNumOccurrences() > 0)
4611 TargetNumRegisters = ForceTargetNumVectorRegs;
4612 }
4613 unsigned MaxLocalUsers = Pair.second;
4614 unsigned LoopInvariantRegs = 0;
4615 if (R.LoopInvariantRegs.contains(Pair.first))
4616 LoopInvariantRegs = R.LoopInvariantRegs[Pair.first];
4617
4618 unsigned TmpIC = llvm::bit_floor((TargetNumRegisters - LoopInvariantRegs) /
4619 MaxLocalUsers);
4620 // Don't count the induction variable as interleaved.
4622 TmpIC = llvm::bit_floor((TargetNumRegisters - LoopInvariantRegs - 1) /
4623 std::max(1U, (MaxLocalUsers - 1)));
4624 }
4625
4626 IC = std::min(IC, TmpIC);
4627 }
4628
4629 // Clamp the interleave ranges to reasonable counts.
4630 unsigned MaxInterleaveCount = TTI.getMaxInterleaveFactor(VF);
4631
4632 // Check if the user has overridden the max.
4633 if (VF.isScalar()) {
4634 if (ForceTargetMaxScalarInterleaveFactor.getNumOccurrences() > 0)
4635 MaxInterleaveCount = ForceTargetMaxScalarInterleaveFactor;
4636 } else {
4637 if (ForceTargetMaxVectorInterleaveFactor.getNumOccurrences() > 0)
4638 MaxInterleaveCount = ForceTargetMaxVectorInterleaveFactor;
4639 }
4640
4641 // Try to get the exact trip count, or an estimate based on profiling data or
4642 // ConstantMax from PSE, failing that.
4643 auto BestKnownTC = getSmallBestKnownTC(PSE, OrigLoop);
4644
4645 // For fixed length VFs treat a scalable trip count as unknown.
4646 if (BestKnownTC && (BestKnownTC->isFixed() || VF.isScalable())) {
4647 // Re-evaluate trip counts and VFs to be in the same numerical space.
4648 unsigned AvailableTC =
4649 estimateElementCount(*BestKnownTC, CM.getVScaleForTuning());
4650 unsigned EstimatedVF = estimateElementCount(VF, CM.getVScaleForTuning());
4651
4652 // At least one iteration must be scalar when this constraint holds. So the
4653 // maximum available iterations for interleaving is one less.
4654 if (CM.requiresScalarEpilogue(VF.isVector()))
4655 --AvailableTC;
4656
4657 unsigned InterleaveCountLB = bit_floor(std::max(
4658 1u, std::min(AvailableTC / (EstimatedVF * 2), MaxInterleaveCount)));
4659
4660 if (getSmallConstantTripCount(PSE.getSE(), OrigLoop).isNonZero()) {
4661 // If the best known trip count is exact, we select between two
4662 // prospective ICs, where
4663 //
4664 // 1) the aggressive IC is capped by the trip count divided by VF
4665 // 2) the conservative IC is capped by the trip count divided by (VF * 2)
4666 //
4667 // The final IC is selected in a way that the epilogue loop trip count is
4668 // minimized while maximizing the IC itself, so that we either run the
4669 // vector loop at least once if it generates a small epilogue loop, or
4670 // else we run the vector loop at least twice.
4671
4672 unsigned InterleaveCountUB = bit_floor(std::max(
4673 1u, std::min(AvailableTC / EstimatedVF, MaxInterleaveCount)));
4674 MaxInterleaveCount = InterleaveCountLB;
4675
4676 if (InterleaveCountUB != InterleaveCountLB) {
4677 unsigned TailTripCountUB =
4678 (AvailableTC % (EstimatedVF * InterleaveCountUB));
4679 unsigned TailTripCountLB =
4680 (AvailableTC % (EstimatedVF * InterleaveCountLB));
4681 // If both produce same scalar tail, maximize the IC to do the same work
4682 // in fewer vector loop iterations
4683 if (TailTripCountUB == TailTripCountLB)
4684 MaxInterleaveCount = InterleaveCountUB;
4685 }
4686 } else {
4687 // If trip count is an estimated compile time constant, limit the
4688 // IC to be capped by the trip count divided by VF * 2, such that the
4689 // vector loop runs at least twice to make interleaving seem profitable
4690 // when there is an epilogue loop present. Since exact Trip count is not
4691 // known we choose to be conservative in our IC estimate.
4692 MaxInterleaveCount = InterleaveCountLB;
4693 }
4694 }
4695
4696 assert(MaxInterleaveCount > 0 &&
4697 "Maximum interleave count must be greater than 0");
4698
4699 // Clamp the calculated IC to be between the 1 and the max interleave count
4700 // that the target and trip count allows.
4701 if (IC > MaxInterleaveCount)
4702 IC = MaxInterleaveCount;
4703 else
4704 // Make sure IC is greater than 0.
4705 IC = std::max(1u, IC);
4706
4707 assert(IC > 0 && "Interleave count must be greater than 0.");
4708
4709 // Interleave if we vectorized this loop and there is a reduction that could
4710 // benefit from interleaving.
4711 if (VF.isVector() && HasReductions) {
4712 LLVM_DEBUG(dbgs() << "LV: Interleaving because of reductions.\n");
4713 return IC;
4714 }
4715
4716 // For any scalar loop that either requires runtime checks or predication we
4717 // are better off leaving this to the unroller. Note that if we've already
4718 // vectorized the loop we will have done the runtime check and so interleaving
4719 // won't require further checks.
4720 bool ScalarInterleavingRequiresPredication =
4721 (VF.isScalar() && any_of(OrigLoop->blocks(), [this](BasicBlock *BB) {
4722 return Legal->blockNeedsPredication(BB);
4723 }));
4724 bool ScalarInterleavingRequiresRuntimePointerCheck =
4725 (VF.isScalar() && Legal->getRuntimePointerChecking()->Need);
4726
4727 // We want to interleave small loops in order to reduce the loop overhead and
4728 // potentially expose ILP opportunities.
4729 LLVM_DEBUG(dbgs() << "LV: Loop cost is " << LoopCost << '\n'
4730 << "LV: IC is " << IC << '\n'
4731 << "LV: VF is " << VF << '\n');
4732 const bool AggressivelyInterleaveReductions =
4733 TTI.enableAggressiveInterleaving(HasReductions);
4734 if (!ScalarInterleavingRequiresRuntimePointerCheck &&
4735 !ScalarInterleavingRequiresPredication && LoopCost < SmallLoopCost) {
4736 // We assume that the cost overhead is 1 and we use the cost model
4737 // to estimate the cost of the loop and interleave until the cost of the
4738 // loop overhead is about 5% of the cost of the loop.
4739 unsigned SmallIC = std::min(IC, (unsigned)llvm::bit_floor<uint64_t>(
4740 SmallLoopCost / LoopCost.getValue()));
4741
4742 // Interleave until store/load ports (estimated by max interleave count) are
4743 // saturated.
4744 unsigned NumStores = 0;
4745 unsigned NumLoads = 0;
4748 for (VPRecipeBase &R : *VPBB) {
4750 NumLoads++;
4751 continue;
4752 }
4754 NumStores++;
4755 continue;
4756 }
4757
4758 if (auto *InterleaveR = dyn_cast<VPInterleaveRecipe>(&R)) {
4759 if (unsigned StoreOps = InterleaveR->getNumStoreOperands())
4760 NumStores += StoreOps;
4761 else
4762 NumLoads += InterleaveR->getNumDefinedValues();
4763 continue;
4764 }
4765 if (auto *RepR = dyn_cast<VPReplicateRecipe>(&R)) {
4766 NumLoads += isa<LoadInst>(RepR->getUnderlyingInstr());
4767 NumStores += isa<StoreInst>(RepR->getUnderlyingInstr());
4768 continue;
4769 }
4770 if (isa<VPHistogramRecipe>(&R)) {
4771 NumLoads++;
4772 NumStores++;
4773 continue;
4774 }
4775 }
4776 }
4777 unsigned StoresIC = IC / (NumStores ? NumStores : 1);
4778 unsigned LoadsIC = IC / (NumLoads ? NumLoads : 1);
4779
4780 // There is little point in interleaving for reductions containing selects
4781 // and compares when VF=1 since it may just create more overhead than it's
4782 // worth for loops with small trip counts. This is because we still have to
4783 // do the final reduction after the loop.
4784 bool HasSelectCmpReductions =
4785 HasReductions &&
4787 [](VPRecipeBase &R) {
4788 auto *RedR = dyn_cast<VPReductionPHIRecipe>(&R);
4789 return RedR && (RecurrenceDescriptor::isAnyOfRecurrenceKind(
4790 RedR->getRecurrenceKind()) ||
4791 RecurrenceDescriptor::isFindIVRecurrenceKind(
4792 RedR->getRecurrenceKind()));
4793 });
4794 if (HasSelectCmpReductions) {
4795 LLVM_DEBUG(dbgs() << "LV: Not interleaving select-cmp reductions.\n");
4796 return 1;
4797 }
4798
4799 // If we have a scalar reduction (vector reductions are already dealt with
4800 // by this point), we can increase the critical path length if the loop
4801 // we're interleaving is inside another loop. For tree-wise reductions
4802 // set the limit to 2, and for ordered reductions it's best to disable
4803 // interleaving entirely.
4804 if (HasReductions && OrigLoop->getLoopDepth() > 1) {
4805 bool HasOrderedReductions =
4807 [](VPRecipeBase &R) {
4808 auto *RedR = dyn_cast<VPReductionPHIRecipe>(&R);
4809
4810 return RedR && RedR->isOrdered();
4811 });
4812 if (HasOrderedReductions) {
4813 LLVM_DEBUG(
4814 dbgs() << "LV: Not interleaving scalar ordered reductions.\n");
4815 return 1;
4816 }
4817
4818 unsigned F = MaxNestedScalarReductionIC;
4819 SmallIC = std::min(SmallIC, F);
4820 StoresIC = std::min(StoresIC, F);
4821 LoadsIC = std::min(LoadsIC, F);
4822 }
4823
4825 std::max(StoresIC, LoadsIC) > SmallIC) {
4826 LLVM_DEBUG(
4827 dbgs() << "LV: Interleaving to saturate store or load ports.\n");
4828 return std::max(StoresIC, LoadsIC);
4829 }
4830
4831 // If there are scalar reductions and TTI has enabled aggressive
4832 // interleaving for reductions, we will interleave to expose ILP.
4833 if (VF.isScalar() && AggressivelyInterleaveReductions) {
4834 LLVM_DEBUG(dbgs() << "LV: Interleaving to expose ILP.\n");
4835 // Interleave no less than SmallIC but not as aggressive as the normal IC
4836 // to satisfy the rare situation when resources are too limited.
4837 return std::max(IC / 2, SmallIC);
4838 }
4839
4840 LLVM_DEBUG(dbgs() << "LV: Interleaving to reduce branch cost.\n");
4841 return SmallIC;
4842 }
4843
4844 // Interleave if this is a large loop (small loops are already dealt with by
4845 // this point) that could benefit from interleaving.
4846 if (AggressivelyInterleaveReductions) {
4847 LLVM_DEBUG(dbgs() << "LV: Interleaving to expose ILP.\n");
4848 return IC;
4849 }
4850
4851 LLVM_DEBUG(dbgs() << "LV: Not Interleaving.\n");
4852 return 1;
4853}
4854
4855bool LoopVectorizationCostModel::useEmulatedMaskMemRefHack(Instruction *I,
4856 ElementCount VF) {
4857 // TODO: Cost model for emulated masked load/store is completely
4858 // broken. This hack guides the cost model to use an artificially
4859 // high enough value to practically disable vectorization with such
4860 // operations, except where previously deployed legality hack allowed
4861 // using very low cost values. This is to avoid regressions coming simply
4862 // from moving "masked load/store" check from legality to cost model.
4863 // Masked Load/Gather emulation was previously never allowed.
4864 // Limited number of Masked Store/Scatter emulation was allowed.
4866 "Expecting a scalar emulated instruction");
4867 return isa<LoadInst>(I) ||
4868 (isa<StoreInst>(I) &&
4869 NumPredStores > NumberOfStoresToPredicate);
4870}
4871
4873 assert(VF.isVector() && "Expected VF >= 2");
4874
4875 // If we've already collected the instructions to scalarize or the predicated
4876 // BBs after vectorization, there's nothing to do. Collection may already have
4877 // occurred if we have a user-selected VF and are now computing the expected
4878 // cost for interleaving.
4879 if (InstsToScalarize.contains(VF) ||
4880 PredicatedBBsAfterVectorization.contains(VF))
4881 return;
4882
4883 // Initialize a mapping for VF in InstsToScalalarize. If we find that it's
4884 // not profitable to scalarize any instructions, the presence of VF in the
4885 // map will indicate that we've analyzed it already.
4886 ScalarCostsTy &ScalarCostsVF = InstsToScalarize[VF];
4887
4888 // Find all the instructions that are scalar with predication in the loop and
4889 // determine if it would be better to not if-convert the blocks they are in.
4890 // If so, we also record the instructions to scalarize.
4891 for (BasicBlock *BB : TheLoop->blocks()) {
4893 continue;
4894 for (Instruction &I : *BB)
4895 if (isScalarWithPredication(&I, VF)) {
4896 ScalarCostsTy ScalarCosts;
4897 // Do not apply discount logic for:
4898 // 1. Scalars after vectorization, as there will only be a single copy
4899 // of the instruction.
4900 // 2. Scalable VF, as that would lead to invalid scalarization costs.
4901 // 3. Emulated masked memrefs, if a hacked cost is needed.
4902 if (!isScalarAfterVectorization(&I, VF) && !VF.isScalable() &&
4903 !useEmulatedMaskMemRefHack(&I, VF) &&
4904 computePredInstDiscount(&I, ScalarCosts, VF) >= 0) {
4905 for (const auto &[I, IC] : ScalarCosts)
4906 ScalarCostsVF.insert({I, IC});
4907 // Check if we decided to scalarize a call. If so, update the widening
4908 // decision of the call to CM_Scalarize with the computed scalar cost.
4909 for (const auto &[I, Cost] : ScalarCosts) {
4910 auto *CI = dyn_cast<CallInst>(I);
4911 if (!CI || !CallWideningDecisions.contains({CI, VF}))
4912 continue;
4913 CallWideningDecisions[{CI, VF}].Kind = CM_Scalarize;
4914 CallWideningDecisions[{CI, VF}].Cost = Cost;
4915 }
4916 }
4917 // Remember that BB will remain after vectorization.
4918 PredicatedBBsAfterVectorization[VF].insert(BB);
4919 for (auto *Pred : predecessors(BB)) {
4920 if (Pred->getSingleSuccessor() == BB)
4921 PredicatedBBsAfterVectorization[VF].insert(Pred);
4922 }
4923 }
4924 }
4925}
4926
4927InstructionCost LoopVectorizationCostModel::computePredInstDiscount(
4928 Instruction *PredInst, ScalarCostsTy &ScalarCosts, ElementCount VF) {
4929 assert(!isUniformAfterVectorization(PredInst, VF) &&
4930 "Instruction marked uniform-after-vectorization will be predicated");
4931
4932 // Initialize the discount to zero, meaning that the scalar version and the
4933 // vector version cost the same.
4934 InstructionCost Discount = 0;
4935
4936 // Holds instructions to analyze. The instructions we visit are mapped in
4937 // ScalarCosts. Those instructions are the ones that would be scalarized if
4938 // we find that the scalar version costs less.
4940
4941 // Returns true if the given instruction can be scalarized.
4942 auto CanBeScalarized = [&](Instruction *I) -> bool {
4943 // We only attempt to scalarize instructions forming a single-use chain
4944 // from the original predicated block that would otherwise be vectorized.
4945 // Although not strictly necessary, we give up on instructions we know will
4946 // already be scalar to avoid traversing chains that are unlikely to be
4947 // beneficial.
4948 if (!I->hasOneUse() || PredInst->getParent() != I->getParent() ||
4950 return false;
4951
4952 // If the instruction is scalar with predication, it will be analyzed
4953 // separately. We ignore it within the context of PredInst.
4954 if (isScalarWithPredication(I, VF))
4955 return false;
4956
4957 // If any of the instruction's operands are uniform after vectorization,
4958 // the instruction cannot be scalarized. This prevents, for example, a
4959 // masked load from being scalarized.
4960 //
4961 // We assume we will only emit a value for lane zero of an instruction
4962 // marked uniform after vectorization, rather than VF identical values.
4963 // Thus, if we scalarize an instruction that uses a uniform, we would
4964 // create uses of values corresponding to the lanes we aren't emitting code
4965 // for. This behavior can be changed by allowing getScalarValue to clone
4966 // the lane zero values for uniforms rather than asserting.
4967 for (Use &U : I->operands())
4968 if (auto *J = dyn_cast<Instruction>(U.get()))
4969 if (isUniformAfterVectorization(J, VF))
4970 return false;
4971
4972 // Otherwise, we can scalarize the instruction.
4973 return true;
4974 };
4975
4976 // Compute the expected cost discount from scalarizing the entire expression
4977 // feeding the predicated instruction. We currently only consider expressions
4978 // that are single-use instruction chains.
4979 Worklist.push_back(PredInst);
4980 while (!Worklist.empty()) {
4981 Instruction *I = Worklist.pop_back_val();
4982
4983 // If we've already analyzed the instruction, there's nothing to do.
4984 if (ScalarCosts.contains(I))
4985 continue;
4986
4987 // Cannot scalarize fixed-order recurrence phis at the moment.
4988 if (isa<PHINode>(I) && Legal->isFixedOrderRecurrence(cast<PHINode>(I)))
4989 continue;
4990
4991 // Compute the cost of the vector instruction. Note that this cost already
4992 // includes the scalarization overhead of the predicated instruction.
4993 InstructionCost VectorCost = getInstructionCost(I, VF);
4994
4995 // Compute the cost of the scalarized instruction. This cost is the cost of
4996 // the instruction as if it wasn't if-converted and instead remained in the
4997 // predicated block. We will scale this cost by block probability after
4998 // computing the scalarization overhead.
4999 InstructionCost ScalarCost =
5001
5002 // Compute the scalarization overhead of needed insertelement instructions
5003 // and phi nodes.
5004 if (isScalarWithPredication(I, VF) && !I->getType()->isVoidTy()) {
5005 Type *WideTy = toVectorizedTy(I->getType(), VF);
5006 for (Type *VectorTy : getContainedTypes(WideTy)) {
5007 ScalarCost += TTI.getScalarizationOverhead(
5009 /*Insert=*/true,
5010 /*Extract=*/false, CostKind);
5011 }
5012 ScalarCost +=
5013 VF.getFixedValue() * TTI.getCFInstrCost(Instruction::PHI, CostKind);
5014 }
5015
5016 // Compute the scalarization overhead of needed extractelement
5017 // instructions. For each of the instruction's operands, if the operand can
5018 // be scalarized, add it to the worklist; otherwise, account for the
5019 // overhead.
5020 for (Use &U : I->operands())
5021 if (auto *J = dyn_cast<Instruction>(U.get())) {
5022 assert(canVectorizeTy(J->getType()) &&
5023 "Instruction has non-scalar type");
5024 if (CanBeScalarized(J))
5025 Worklist.push_back(J);
5026 else if (needsExtract(J, VF)) {
5027 Type *WideTy = toVectorizedTy(J->getType(), VF);
5028 for (Type *VectorTy : getContainedTypes(WideTy)) {
5029 ScalarCost += TTI.getScalarizationOverhead(
5030 cast<VectorType>(VectorTy),
5031 APInt::getAllOnes(VF.getFixedValue()), /*Insert*/ false,
5032 /*Extract*/ true, CostKind);
5033 }
5034 }
5035 }
5036
5037 // Scale the total scalar cost by block probability.
5038 ScalarCost /= getPredBlockCostDivisor(CostKind);
5039
5040 // Compute the discount. A non-negative discount means the vector version
5041 // of the instruction costs more, and scalarizing would be beneficial.
5042 Discount += VectorCost - ScalarCost;
5043 ScalarCosts[I] = ScalarCost;
5044 }
5045
5046 return Discount;
5047}
5048
5051
5052 // If the vector loop gets executed exactly once with the given VF, ignore the
5053 // costs of comparison and induction instructions, as they'll get simplified
5054 // away.
5055 SmallPtrSet<Instruction *, 2> ValuesToIgnoreForVF;
5056 auto TC = getSmallConstantTripCount(PSE.getSE(), TheLoop);
5057 if (TC == VF && !foldTailByMasking())
5059 ValuesToIgnoreForVF);
5060
5061 // For each block.
5062 for (BasicBlock *BB : TheLoop->blocks()) {
5063 InstructionCost BlockCost;
5064
5065 // For each instruction in the old loop.
5066 for (Instruction &I : BB->instructionsWithoutDebug()) {
5067 // Skip ignored values.
5068 if (ValuesToIgnore.count(&I) || ValuesToIgnoreForVF.count(&I) ||
5069 (VF.isVector() && VecValuesToIgnore.count(&I)))
5070 continue;
5071
5073
5074 // Check if we should override the cost.
5075 if (C.isValid() && ForceTargetInstructionCost.getNumOccurrences() > 0)
5077
5078 BlockCost += C;
5079 LLVM_DEBUG(dbgs() << "LV: Found an estimated cost of " << C << " for VF "
5080 << VF << " For instruction: " << I << '\n');
5081 }
5082
5083 // If we are vectorizing a predicated block, it will have been
5084 // if-converted. This means that the block's instructions (aside from
5085 // stores and instructions that may divide by zero) will now be
5086 // unconditionally executed. For the scalar case, we may not always execute
5087 // the predicated block, if it is an if-else block. Thus, scale the block's
5088 // cost by the probability of executing it. blockNeedsPredication from
5089 // Legal is used so as to not include all blocks in tail folded loops.
5090 if (VF.isScalar() && Legal->blockNeedsPredication(BB))
5091 BlockCost /= getPredBlockCostDivisor(CostKind);
5092
5093 Cost += BlockCost;
5094 }
5095
5096 return Cost;
5097}
5098
5099/// Gets Address Access SCEV after verifying that the access pattern
5100/// is loop invariant except the induction variable dependence.
5101///
5102/// This SCEV can be sent to the Target in order to estimate the address
5103/// calculation cost.
5105 Value *Ptr,
5108 const Loop *TheLoop) {
5109
5110 auto *Gep = dyn_cast<GetElementPtrInst>(Ptr);
5111 if (!Gep)
5112 return nullptr;
5113
5114 // We are looking for a gep with all loop invariant indices except for one
5115 // which should be an induction variable.
5116 auto *SE = PSE.getSE();
5117 unsigned NumOperands = Gep->getNumOperands();
5118 for (unsigned Idx = 1; Idx < NumOperands; ++Idx) {
5119 Value *Opd = Gep->getOperand(Idx);
5120 if (!SE->isLoopInvariant(SE->getSCEV(Opd), TheLoop) &&
5121 !Legal->isInductionVariable(Opd))
5122 return nullptr;
5123 }
5124
5125 // Now we know we have a GEP ptr, %inv, %ind, %inv. return the Ptr SCEV.
5126 return PSE.getSCEV(Ptr);
5127}
5128
5130LoopVectorizationCostModel::getMemInstScalarizationCost(Instruction *I,
5131 ElementCount VF) {
5132 assert(VF.isVector() &&
5133 "Scalarization cost of instruction implies vectorization.");
5134 if (VF.isScalable())
5136
5137 Type *ValTy = getLoadStoreType(I);
5138 auto *SE = PSE.getSE();
5139
5140 unsigned AS = getLoadStoreAddressSpace(I);
5142 Type *PtrTy = toVectorTy(Ptr->getType(), VF);
5143 // NOTE: PtrTy is a vector to signal `TTI::getAddressComputationCost`
5144 // that it is being called from this specific place.
5145
5146 // Figure out whether the access is strided and get the stride value
5147 // if it's known in compile time
5148 const SCEV *PtrSCEV = getAddressAccessSCEV(Ptr, Legal, PSE, TheLoop);
5149
5150 // Get the cost of the scalar memory instruction and address computation.
5151 InstructionCost Cost = VF.getFixedValue() * TTI.getAddressComputationCost(
5152 PtrTy, SE, PtrSCEV, CostKind);
5153
5154 // Don't pass *I here, since it is scalar but will actually be part of a
5155 // vectorized loop where the user of it is a vectorized instruction.
5156 const Align Alignment = getLoadStoreAlignment(I);
5157 TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(I->getOperand(0));
5158 Cost += VF.getFixedValue() *
5159 TTI.getMemoryOpCost(I->getOpcode(), ValTy->getScalarType(), Alignment,
5160 AS, CostKind, OpInfo);
5161
5162 // Get the overhead of the extractelement and insertelement instructions
5163 // we might create due to scalarization.
5164 Cost += getScalarizationOverhead(I, VF);
5165
5166 // If we have a predicated load/store, it will need extra i1 extracts and
5167 // conditional branches, but may not be executed for each vector lane. Scale
5168 // the cost by the probability of executing the predicated block.
5169 if (isPredicatedInst(I)) {
5171
5172 // Add the cost of an i1 extract and a branch
5173 auto *VecI1Ty =
5175 Cost += TTI.getScalarizationOverhead(
5176 VecI1Ty, APInt::getAllOnes(VF.getFixedValue()),
5177 /*Insert=*/false, /*Extract=*/true, CostKind);
5178 Cost += TTI.getCFInstrCost(Instruction::Br, CostKind);
5179
5180 if (useEmulatedMaskMemRefHack(I, VF))
5181 // Artificially setting to a high enough value to practically disable
5182 // vectorization with such operations.
5183 Cost = 3000000;
5184 }
5185
5186 return Cost;
5187}
5188
5190LoopVectorizationCostModel::getConsecutiveMemOpCost(Instruction *I,
5191 ElementCount VF) {
5192 Type *ValTy = getLoadStoreType(I);
5193 auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
5195 unsigned AS = getLoadStoreAddressSpace(I);
5196 int ConsecutiveStride = Legal->isConsecutivePtr(ValTy, Ptr);
5197
5198 assert((ConsecutiveStride == 1 || ConsecutiveStride == -1) &&
5199 "Stride should be 1 or -1 for consecutive memory access");
5200 const Align Alignment = getLoadStoreAlignment(I);
5202 if (Legal->isMaskRequired(I)) {
5203 Cost += TTI.getMaskedMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS,
5204 CostKind);
5205 } else {
5206 TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(I->getOperand(0));
5207 Cost += TTI.getMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS,
5208 CostKind, OpInfo, I);
5209 }
5210
5211 bool Reverse = ConsecutiveStride < 0;
5212 if (Reverse)
5213 Cost += TTI.getShuffleCost(TargetTransformInfo::SK_Reverse, VectorTy,
5214 VectorTy, {}, CostKind, 0);
5215 return Cost;
5216}
5217
5219LoopVectorizationCostModel::getUniformMemOpCost(Instruction *I,
5220 ElementCount VF) {
5221 assert(Legal->isUniformMemOp(*I, VF));
5222
5223 Type *ValTy = getLoadStoreType(I);
5225 auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
5226 const Align Alignment = getLoadStoreAlignment(I);
5227 unsigned AS = getLoadStoreAddressSpace(I);
5228 if (isa<LoadInst>(I)) {
5229 return TTI.getAddressComputationCost(PtrTy, nullptr, nullptr, CostKind) +
5230 TTI.getMemoryOpCost(Instruction::Load, ValTy, Alignment, AS,
5231 CostKind) +
5232 TTI.getShuffleCost(TargetTransformInfo::SK_Broadcast, VectorTy,
5233 VectorTy, {}, CostKind);
5234 }
5235 StoreInst *SI = cast<StoreInst>(I);
5236
5237 bool IsLoopInvariantStoreValue = Legal->isInvariant(SI->getValueOperand());
5238 // TODO: We have existing tests that request the cost of extracting element
5239 // VF.getKnownMinValue() - 1 from a scalable vector. This does not represent
5240 // the actual generated code, which involves extracting the last element of
5241 // a scalable vector where the lane to extract is unknown at compile time.
5243 TTI.getAddressComputationCost(PtrTy, nullptr, nullptr, CostKind) +
5244 TTI.getMemoryOpCost(Instruction::Store, ValTy, Alignment, AS, CostKind);
5245 if (!IsLoopInvariantStoreValue)
5246 Cost += TTI.getIndexedVectorInstrCostFromEnd(Instruction::ExtractElement,
5247 VectorTy, CostKind, 0);
5248 return Cost;
5249}
5250
5252LoopVectorizationCostModel::getGatherScatterCost(Instruction *I,
5253 ElementCount VF) {
5254 Type *ValTy = getLoadStoreType(I);
5255 auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
5256 const Align Alignment = getLoadStoreAlignment(I);
5258 Type *PtrTy = Ptr->getType();
5259
5260 if (!Legal->isUniform(Ptr, VF))
5261 PtrTy = toVectorTy(PtrTy, VF);
5262
5263 return TTI.getAddressComputationCost(PtrTy, nullptr, nullptr, CostKind) +
5264 TTI.getGatherScatterOpCost(I->getOpcode(), VectorTy, Ptr,
5265 Legal->isMaskRequired(I), Alignment,
5266 CostKind, I);
5267}
5268
5270LoopVectorizationCostModel::getInterleaveGroupCost(Instruction *I,
5271 ElementCount VF) {
5272 const auto *Group = getInterleavedAccessGroup(I);
5273 assert(Group && "Fail to get an interleaved access group.");
5274
5275 Instruction *InsertPos = Group->getInsertPos();
5276 Type *ValTy = getLoadStoreType(InsertPos);
5277 auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
5278 unsigned AS = getLoadStoreAddressSpace(InsertPos);
5279
5280 unsigned InterleaveFactor = Group->getFactor();
5281 auto *WideVecTy = VectorType::get(ValTy, VF * InterleaveFactor);
5282
5283 // Holds the indices of existing members in the interleaved group.
5284 SmallVector<unsigned, 4> Indices;
5285 for (unsigned IF = 0; IF < InterleaveFactor; IF++)
5286 if (Group->getMember(IF))
5287 Indices.push_back(IF);
5288
5289 // Calculate the cost of the whole interleaved group.
5290 bool UseMaskForGaps =
5291 (Group->requiresScalarEpilogue() && !isScalarEpilogueAllowed()) ||
5292 (isa<StoreInst>(I) && !Group->isFull());
5293 InstructionCost Cost = TTI.getInterleavedMemoryOpCost(
5294 InsertPos->getOpcode(), WideVecTy, Group->getFactor(), Indices,
5295 Group->getAlign(), AS, CostKind, Legal->isMaskRequired(I),
5296 UseMaskForGaps);
5297
5298 if (Group->isReverse()) {
5299 // TODO: Add support for reversed masked interleaved access.
5300 assert(!Legal->isMaskRequired(I) &&
5301 "Reverse masked interleaved access not supported.");
5302 Cost += Group->getNumMembers() *
5303 TTI.getShuffleCost(TargetTransformInfo::SK_Reverse, VectorTy,
5304 VectorTy, {}, CostKind, 0);
5305 }
5306 return Cost;
5307}
5308
5309std::optional<InstructionCost>
5311 ElementCount VF,
5312 Type *Ty) const {
5313 using namespace llvm::PatternMatch;
5314 // Early exit for no inloop reductions
5315 if (InLoopReductions.empty() || VF.isScalar() || !isa<VectorType>(Ty))
5316 return std::nullopt;
5317 auto *VectorTy = cast<VectorType>(Ty);
5318
5319 // We are looking for a pattern of, and finding the minimal acceptable cost:
5320 // reduce(mul(ext(A), ext(B))) or
5321 // reduce(mul(A, B)) or
5322 // reduce(ext(A)) or
5323 // reduce(A).
5324 // The basic idea is that we walk down the tree to do that, finding the root
5325 // reduction instruction in InLoopReductionImmediateChains. From there we find
5326 // the pattern of mul/ext and test the cost of the entire pattern vs the cost
5327 // of the components. If the reduction cost is lower then we return it for the
5328 // reduction instruction and 0 for the other instructions in the pattern. If
5329 // it is not we return an invalid cost specifying the orignal cost method
5330 // should be used.
5331 Instruction *RetI = I;
5332 if (match(RetI, m_ZExtOrSExt(m_Value()))) {
5333 if (!RetI->hasOneUser())
5334 return std::nullopt;
5335 RetI = RetI->user_back();
5336 }
5337
5338 if (match(RetI, m_OneUse(m_Mul(m_Value(), m_Value()))) &&
5339 RetI->user_back()->getOpcode() == Instruction::Add) {
5340 RetI = RetI->user_back();
5341 }
5342
5343 // Test if the found instruction is a reduction, and if not return an invalid
5344 // cost specifying the parent to use the original cost modelling.
5345 Instruction *LastChain = InLoopReductionImmediateChains.lookup(RetI);
5346 if (!LastChain)
5347 return std::nullopt;
5348
5349 // Find the reduction this chain is a part of and calculate the basic cost of
5350 // the reduction on its own.
5351 Instruction *ReductionPhi = LastChain;
5352 while (!isa<PHINode>(ReductionPhi))
5353 ReductionPhi = InLoopReductionImmediateChains.at(ReductionPhi);
5354
5355 const RecurrenceDescriptor &RdxDesc =
5356 Legal->getRecurrenceDescriptor(cast<PHINode>(ReductionPhi));
5357
5358 InstructionCost BaseCost;
5359 RecurKind RK = RdxDesc.getRecurrenceKind();
5362 BaseCost = TTI.getMinMaxReductionCost(MinMaxID, VectorTy,
5363 RdxDesc.getFastMathFlags(), CostKind);
5364 } else {
5365 BaseCost = TTI.getArithmeticReductionCost(
5366 RdxDesc.getOpcode(), VectorTy, RdxDesc.getFastMathFlags(), CostKind);
5367 }
5368
5369 // For a call to the llvm.fmuladd intrinsic we need to add the cost of a
5370 // normal fmul instruction to the cost of the fadd reduction.
5371 if (RK == RecurKind::FMulAdd)
5372 BaseCost +=
5373 TTI.getArithmeticInstrCost(Instruction::FMul, VectorTy, CostKind);
5374
5375 // If we're using ordered reductions then we can just return the base cost
5376 // here, since getArithmeticReductionCost calculates the full ordered
5377 // reduction cost when FP reassociation is not allowed.
5378 if (useOrderedReductions(RdxDesc))
5379 return BaseCost;
5380
5381 // Get the operand that was not the reduction chain and match it to one of the
5382 // patterns, returning the better cost if it is found.
5383 Instruction *RedOp = RetI->getOperand(1) == LastChain
5386
5387 VectorTy = VectorType::get(I->getOperand(0)->getType(), VectorTy);
5388
5389 Instruction *Op0, *Op1;
5390 if (RedOp && RdxDesc.getOpcode() == Instruction::Add &&
5391 match(RedOp,
5393 match(Op0, m_ZExtOrSExt(m_Value())) &&
5394 Op0->getOpcode() == Op1->getOpcode() &&
5395 Op0->getOperand(0)->getType() == Op1->getOperand(0)->getType() &&
5396 !TheLoop->isLoopInvariant(Op0) && !TheLoop->isLoopInvariant(Op1) &&
5397 (Op0->getOpcode() == RedOp->getOpcode() || Op0 == Op1)) {
5398
5399 // Matched reduce.add(ext(mul(ext(A), ext(B)))
5400 // Note that the extend opcodes need to all match, or if A==B they will have
5401 // been converted to zext(mul(sext(A), sext(A))) as it is known positive,
5402 // which is equally fine.
5403 bool IsUnsigned = isa<ZExtInst>(Op0);
5404 auto *ExtType = VectorType::get(Op0->getOperand(0)->getType(), VectorTy);
5405 auto *MulType = VectorType::get(Op0->getType(), VectorTy);
5406
5407 InstructionCost ExtCost =
5408 TTI.getCastInstrCost(Op0->getOpcode(), MulType, ExtType,
5410 InstructionCost MulCost =
5411 TTI.getArithmeticInstrCost(Instruction::Mul, MulType, CostKind);
5412 InstructionCost Ext2Cost =
5413 TTI.getCastInstrCost(RedOp->getOpcode(), VectorTy, MulType,
5415
5416 InstructionCost RedCost = TTI.getMulAccReductionCost(
5417 IsUnsigned, RdxDesc.getOpcode(), RdxDesc.getRecurrenceType(), ExtType,
5418 CostKind);
5419
5420 if (RedCost.isValid() &&
5421 RedCost < ExtCost * 2 + MulCost + Ext2Cost + BaseCost)
5422 return I == RetI ? RedCost : 0;
5423 } else if (RedOp && match(RedOp, m_ZExtOrSExt(m_Value())) &&
5424 !TheLoop->isLoopInvariant(RedOp)) {
5425 // Matched reduce(ext(A))
5426 bool IsUnsigned = isa<ZExtInst>(RedOp);
5427 auto *ExtType = VectorType::get(RedOp->getOperand(0)->getType(), VectorTy);
5428 InstructionCost RedCost = TTI.getExtendedReductionCost(
5429 RdxDesc.getOpcode(), IsUnsigned, RdxDesc.getRecurrenceType(), ExtType,
5430 RdxDesc.getFastMathFlags(), CostKind);
5431
5432 InstructionCost ExtCost =
5433 TTI.getCastInstrCost(RedOp->getOpcode(), VectorTy, ExtType,
5435 if (RedCost.isValid() && RedCost < BaseCost + ExtCost)
5436 return I == RetI ? RedCost : 0;
5437 } else if (RedOp && RdxDesc.getOpcode() == Instruction::Add &&
5438 match(RedOp, m_Mul(m_Instruction(Op0), m_Instruction(Op1)))) {
5439 if (match(Op0, m_ZExtOrSExt(m_Value())) &&
5440 Op0->getOpcode() == Op1->getOpcode() &&
5441 !TheLoop->isLoopInvariant(Op0) && !TheLoop->isLoopInvariant(Op1)) {
5442 bool IsUnsigned = isa<ZExtInst>(Op0);
5443 Type *Op0Ty = Op0->getOperand(0)->getType();
5444 Type *Op1Ty = Op1->getOperand(0)->getType();
5445 Type *LargestOpTy =
5446 Op0Ty->getIntegerBitWidth() < Op1Ty->getIntegerBitWidth() ? Op1Ty
5447 : Op0Ty;
5448 auto *ExtType = VectorType::get(LargestOpTy, VectorTy);
5449
5450 // Matched reduce.add(mul(ext(A), ext(B))), where the two ext may be of
5451 // different sizes. We take the largest type as the ext to reduce, and add
5452 // the remaining cost as, for example reduce(mul(ext(ext(A)), ext(B))).
5453 InstructionCost ExtCost0 = TTI.getCastInstrCost(
5454 Op0->getOpcode(), VectorTy, VectorType::get(Op0Ty, VectorTy),
5456 InstructionCost ExtCost1 = TTI.getCastInstrCost(
5457 Op1->getOpcode(), VectorTy, VectorType::get(Op1Ty, VectorTy),
5459 InstructionCost MulCost =
5460 TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy, CostKind);
5461
5462 InstructionCost RedCost = TTI.getMulAccReductionCost(
5463 IsUnsigned, RdxDesc.getOpcode(), RdxDesc.getRecurrenceType(), ExtType,
5464 CostKind);
5465 InstructionCost ExtraExtCost = 0;
5466 if (Op0Ty != LargestOpTy || Op1Ty != LargestOpTy) {
5467 Instruction *ExtraExtOp = (Op0Ty != LargestOpTy) ? Op0 : Op1;
5468 ExtraExtCost = TTI.getCastInstrCost(
5469 ExtraExtOp->getOpcode(), ExtType,
5470 VectorType::get(ExtraExtOp->getOperand(0)->getType(), VectorTy),
5472 }
5473
5474 if (RedCost.isValid() &&
5475 (RedCost + ExtraExtCost) < (ExtCost0 + ExtCost1 + MulCost + BaseCost))
5476 return I == RetI ? RedCost : 0;
5477 } else if (!match(I, m_ZExtOrSExt(m_Value()))) {
5478 // Matched reduce.add(mul())
5479 InstructionCost MulCost =
5480 TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy, CostKind);
5481
5482 InstructionCost RedCost = TTI.getMulAccReductionCost(
5483 true, RdxDesc.getOpcode(), RdxDesc.getRecurrenceType(), VectorTy,
5484 CostKind);
5485
5486 if (RedCost.isValid() && RedCost < MulCost + BaseCost)
5487 return I == RetI ? RedCost : 0;
5488 }
5489 }
5490
5491 return I == RetI ? std::optional<InstructionCost>(BaseCost) : std::nullopt;
5492}
5493
5495LoopVectorizationCostModel::getMemoryInstructionCost(Instruction *I,
5496 ElementCount VF) {
5497 // Calculate scalar cost only. Vectorization cost should be ready at this
5498 // moment.
5499 if (VF.isScalar()) {
5500 Type *ValTy = getLoadStoreType(I);
5502 const Align Alignment = getLoadStoreAlignment(I);
5503 unsigned AS = getLoadStoreAddressSpace(I);
5504
5505 TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(I->getOperand(0));
5506 return TTI.getAddressComputationCost(PtrTy, nullptr, nullptr, CostKind) +
5507 TTI.getMemoryOpCost(I->getOpcode(), ValTy, Alignment, AS, CostKind,
5508 OpInfo, I);
5509 }
5510 return getWideningCost(I, VF);
5511}
5512
5514LoopVectorizationCostModel::getScalarizationOverhead(Instruction *I,
5515 ElementCount VF) const {
5516
5517 // There is no mechanism yet to create a scalable scalarization loop,
5518 // so this is currently Invalid.
5519 if (VF.isScalable())
5521
5522 if (VF.isScalar())
5523 return 0;
5524
5526 Type *RetTy = toVectorizedTy(I->getType(), VF);
5527 if (!RetTy->isVoidTy() &&
5528 (!isa<LoadInst>(I) || !TTI.supportsEfficientVectorElementLoadStore())) {
5529
5530 for (Type *VectorTy : getContainedTypes(RetTy)) {
5531 Cost += TTI.getScalarizationOverhead(
5533 /*Insert=*/true,
5534 /*Extract=*/false, CostKind);
5535 }
5536 }
5537
5538 // Some targets keep addresses scalar.
5539 if (isa<LoadInst>(I) && !TTI.prefersVectorizedAddressing())
5540 return Cost;
5541
5542 // Some targets support efficient element stores.
5543 if (isa<StoreInst>(I) && TTI.supportsEfficientVectorElementLoadStore())
5544 return Cost;
5545
5546 // Collect operands to consider.
5547 CallInst *CI = dyn_cast<CallInst>(I);
5548 Instruction::op_range Ops = CI ? CI->args() : I->operands();
5549
5550 // Skip operands that do not require extraction/scalarization and do not incur
5551 // any overhead.
5553 for (auto *V : filterExtractingOperands(Ops, VF))
5554 Tys.push_back(maybeVectorizeType(V->getType(), VF));
5555 return Cost + TTI.getOperandsScalarizationOverhead(Tys, CostKind);
5556}
5557
5559 if (VF.isScalar())
5560 return;
5561 NumPredStores = 0;
5562 for (BasicBlock *BB : TheLoop->blocks()) {
5563 // For each instruction in the old loop.
5564 for (Instruction &I : *BB) {
5566 if (!Ptr)
5567 continue;
5568
5569 // TODO: We should generate better code and update the cost model for
5570 // predicated uniform stores. Today they are treated as any other
5571 // predicated store (see added test cases in
5572 // invariant-store-vectorization.ll).
5574 NumPredStores++;
5575
5576 if (Legal->isUniformMemOp(I, VF)) {
5577 auto IsLegalToScalarize = [&]() {
5578 if (!VF.isScalable())
5579 // Scalarization of fixed length vectors "just works".
5580 return true;
5581
5582 // We have dedicated lowering for unpredicated uniform loads and
5583 // stores. Note that even with tail folding we know that at least
5584 // one lane is active (i.e. generalized predication is not possible
5585 // here), and the logic below depends on this fact.
5586 if (!foldTailByMasking())
5587 return true;
5588
5589 // For scalable vectors, a uniform memop load is always
5590 // uniform-by-parts and we know how to scalarize that.
5591 if (isa<LoadInst>(I))
5592 return true;
5593
5594 // A uniform store isn't neccessarily uniform-by-part
5595 // and we can't assume scalarization.
5596 auto &SI = cast<StoreInst>(I);
5597 return TheLoop->isLoopInvariant(SI.getValueOperand());
5598 };
5599
5600 const InstructionCost GatherScatterCost =
5602 getGatherScatterCost(&I, VF) : InstructionCost::getInvalid();
5603
5604 // Load: Scalar load + broadcast
5605 // Store: Scalar store + isLoopInvariantStoreValue ? 0 : extract
5606 // FIXME: This cost is a significant under-estimate for tail folded
5607 // memory ops.
5608 const InstructionCost ScalarizationCost =
5609 IsLegalToScalarize() ? getUniformMemOpCost(&I, VF)
5611
5612 // Choose better solution for the current VF, Note that Invalid
5613 // costs compare as maximumal large. If both are invalid, we get
5614 // scalable invalid which signals a failure and a vectorization abort.
5615 if (GatherScatterCost < ScalarizationCost)
5616 setWideningDecision(&I, VF, CM_GatherScatter, GatherScatterCost);
5617 else
5618 setWideningDecision(&I, VF, CM_Scalarize, ScalarizationCost);
5619 continue;
5620 }
5621
5622 // We assume that widening is the best solution when possible.
5623 if (memoryInstructionCanBeWidened(&I, VF)) {
5624 InstructionCost Cost = getConsecutiveMemOpCost(&I, VF);
5625 int ConsecutiveStride = Legal->isConsecutivePtr(
5627 assert((ConsecutiveStride == 1 || ConsecutiveStride == -1) &&
5628 "Expected consecutive stride.");
5629 InstWidening Decision =
5630 ConsecutiveStride == 1 ? CM_Widen : CM_Widen_Reverse;
5631 setWideningDecision(&I, VF, Decision, Cost);
5632 continue;
5633 }
5634
5635 // Choose between Interleaving, Gather/Scatter or Scalarization.
5637 unsigned NumAccesses = 1;
5638 if (isAccessInterleaved(&I)) {
5639 const auto *Group = getInterleavedAccessGroup(&I);
5640 assert(Group && "Fail to get an interleaved access group.");
5641
5642 // Make one decision for the whole group.
5643 if (getWideningDecision(&I, VF) != CM_Unknown)
5644 continue;
5645
5646 NumAccesses = Group->getNumMembers();
5648 InterleaveCost = getInterleaveGroupCost(&I, VF);
5649 }
5650
5651 InstructionCost GatherScatterCost =
5653 ? getGatherScatterCost(&I, VF) * NumAccesses
5655
5656 InstructionCost ScalarizationCost =
5657 getMemInstScalarizationCost(&I, VF) * NumAccesses;
5658
5659 // Choose better solution for the current VF,
5660 // write down this decision and use it during vectorization.
5662 InstWidening Decision;
5663 if (InterleaveCost <= GatherScatterCost &&
5664 InterleaveCost < ScalarizationCost) {
5665 Decision = CM_Interleave;
5666 Cost = InterleaveCost;
5667 } else if (GatherScatterCost < ScalarizationCost) {
5668 Decision = CM_GatherScatter;
5669 Cost = GatherScatterCost;
5670 } else {
5671 Decision = CM_Scalarize;
5672 Cost = ScalarizationCost;
5673 }
5674 // If the instructions belongs to an interleave group, the whole group
5675 // receives the same decision. The whole group receives the cost, but
5676 // the cost will actually be assigned to one instruction.
5677 if (const auto *Group = getInterleavedAccessGroup(&I)) {
5678 if (Decision == CM_Scalarize) {
5679 for (unsigned Idx = 0; Idx < Group->getFactor(); ++Idx) {
5680 if (auto *I = Group->getMember(Idx)) {
5681 setWideningDecision(I, VF, Decision,
5682 getMemInstScalarizationCost(I, VF));
5683 }
5684 }
5685 } else {
5686 setWideningDecision(Group, VF, Decision, Cost);
5687 }
5688 } else
5689 setWideningDecision(&I, VF, Decision, Cost);
5690 }
5691 }
5692
5693 // Make sure that any load of address and any other address computation
5694 // remains scalar unless there is gather/scatter support. This avoids
5695 // inevitable extracts into address registers, and also has the benefit of
5696 // activating LSR more, since that pass can't optimize vectorized
5697 // addresses.
5698 if (TTI.prefersVectorizedAddressing())
5699 return;
5700
5701 // Start with all scalar pointer uses.
5703 for (BasicBlock *BB : TheLoop->blocks())
5704 for (Instruction &I : *BB) {
5705 Instruction *PtrDef =
5707 if (PtrDef && TheLoop->contains(PtrDef) &&
5709 AddrDefs.insert(PtrDef);
5710 }
5711
5712 // Add all instructions used to generate the addresses.
5714 append_range(Worklist, AddrDefs);
5715 while (!Worklist.empty()) {
5716 Instruction *I = Worklist.pop_back_val();
5717 for (auto &Op : I->operands())
5718 if (auto *InstOp = dyn_cast<Instruction>(Op))
5719 if ((InstOp->getParent() == I->getParent()) && !isa<PHINode>(InstOp) &&
5720 AddrDefs.insert(InstOp).second)
5721 Worklist.push_back(InstOp);
5722 }
5723
5724 for (auto *I : AddrDefs) {
5725 if (isa<LoadInst>(I)) {
5726 // Setting the desired widening decision should ideally be handled in
5727 // by cost functions, but since this involves the task of finding out
5728 // if the loaded register is involved in an address computation, it is
5729 // instead changed here when we know this is the case.
5730 InstWidening Decision = getWideningDecision(I, VF);
5731 if (Decision == CM_Widen || Decision == CM_Widen_Reverse ||
5732 (!isPredicatedInst(I) && !Legal->isUniformMemOp(*I, VF) &&
5733 Decision == CM_Scalarize))
5734 // Scalarize a widened load of address or update the cost of a scalar
5735 // load of an address.
5737 I, VF, CM_Scalarize,
5738 (VF.getKnownMinValue() *
5739 getMemoryInstructionCost(I, ElementCount::getFixed(1))));
5740 else if (const auto *Group = getInterleavedAccessGroup(I)) {
5741 // Scalarize an interleave group of address loads.
5742 for (unsigned I = 0; I < Group->getFactor(); ++I) {
5743 if (Instruction *Member = Group->getMember(I))
5745 Member, VF, CM_Scalarize,
5746 (VF.getKnownMinValue() *
5747 getMemoryInstructionCost(Member, ElementCount::getFixed(1))));
5748 }
5749 }
5750 } else {
5751 // Cannot scalarize fixed-order recurrence phis at the moment.
5752 if (isa<PHINode>(I) && Legal->isFixedOrderRecurrence(cast<PHINode>(I)))
5753 continue;
5754
5755 // Make sure I gets scalarized and a cost estimate without
5756 // scalarization overhead.
5757 ForcedScalars[VF].insert(I);
5758 }
5759 }
5760}
5761
5763 assert(!VF.isScalar() &&
5764 "Trying to set a vectorization decision for a scalar VF");
5765
5766 auto ForcedScalar = ForcedScalars.find(VF);
5767 for (BasicBlock *BB : TheLoop->blocks()) {
5768 // For each instruction in the old loop.
5769 for (Instruction &I : *BB) {
5771
5772 if (!CI)
5773 continue;
5774
5778 Function *ScalarFunc = CI->getCalledFunction();
5779 Type *ScalarRetTy = CI->getType();
5780 SmallVector<Type *, 4> Tys, ScalarTys;
5781 for (auto &ArgOp : CI->args())
5782 ScalarTys.push_back(ArgOp->getType());
5783
5784 // Estimate cost of scalarized vector call. The source operands are
5785 // assumed to be vectors, so we need to extract individual elements from
5786 // there, execute VF scalar calls, and then gather the result into the
5787 // vector return value.
5788 if (VF.isFixed()) {
5789 InstructionCost ScalarCallCost =
5790 TTI.getCallInstrCost(ScalarFunc, ScalarRetTy, ScalarTys, CostKind);
5791
5792 // Compute costs of unpacking argument values for the scalar calls and
5793 // packing the return values to a vector.
5794 InstructionCost ScalarizationCost = getScalarizationOverhead(CI, VF);
5795 ScalarCost = ScalarCallCost * VF.getKnownMinValue() + ScalarizationCost;
5796 } else {
5797 // There is no point attempting to calculate the scalar cost for a
5798 // scalable VF as we know it will be Invalid.
5799 assert(!getScalarizationOverhead(CI, VF).isValid() &&
5800 "Unexpected valid cost for scalarizing scalable vectors");
5801 ScalarCost = InstructionCost::getInvalid();
5802 }
5803
5804 // Honor ForcedScalars and UniformAfterVectorization decisions.
5805 // TODO: For calls, it might still be more profitable to widen. Use
5806 // VPlan-based cost model to compare different options.
5807 if (VF.isVector() && ((ForcedScalar != ForcedScalars.end() &&
5808 ForcedScalar->second.contains(CI)) ||
5809 isUniformAfterVectorization(CI, VF))) {
5810 setCallWideningDecision(CI, VF, CM_Scalarize, nullptr,
5811 Intrinsic::not_intrinsic, std::nullopt,
5812 ScalarCost);
5813 continue;
5814 }
5815
5816 bool MaskRequired = Legal->isMaskRequired(CI);
5817 // Compute corresponding vector type for return value and arguments.
5818 Type *RetTy = toVectorizedTy(ScalarRetTy, VF);
5819 for (Type *ScalarTy : ScalarTys)
5820 Tys.push_back(toVectorizedTy(ScalarTy, VF));
5821
5822 // An in-loop reduction using an fmuladd intrinsic is a special case;
5823 // we don't want the normal cost for that intrinsic.
5825 if (auto RedCost = getReductionPatternCost(CI, VF, RetTy)) {
5828 std::nullopt, *RedCost);
5829 continue;
5830 }
5831
5832 // Find the cost of vectorizing the call, if we can find a suitable
5833 // vector variant of the function.
5834 VFInfo FuncInfo;
5835 Function *VecFunc = nullptr;
5836 // Search through any available variants for one we can use at this VF.
5837 for (VFInfo &Info : VFDatabase::getMappings(*CI)) {
5838 // Must match requested VF.
5839 if (Info.Shape.VF != VF)
5840 continue;
5841
5842 // Must take a mask argument if one is required
5843 if (MaskRequired && !Info.isMasked())
5844 continue;
5845
5846 // Check that all parameter kinds are supported
5847 bool ParamsOk = true;
5848 for (VFParameter Param : Info.Shape.Parameters) {
5849 switch (Param.ParamKind) {
5851 break;
5853 Value *ScalarParam = CI->getArgOperand(Param.ParamPos);
5854 // Make sure the scalar parameter in the loop is invariant.
5855 if (!PSE.getSE()->isLoopInvariant(PSE.getSCEV(ScalarParam),
5856 TheLoop))
5857 ParamsOk = false;
5858 break;
5859 }
5861 Value *ScalarParam = CI->getArgOperand(Param.ParamPos);
5862 // Find the stride for the scalar parameter in this loop and see if
5863 // it matches the stride for the variant.
5864 // TODO: do we need to figure out the cost of an extract to get the
5865 // first lane? Or do we hope that it will be folded away?
5866 ScalarEvolution *SE = PSE.getSE();
5867 if (!match(SE->getSCEV(ScalarParam),
5869 m_SCEV(), m_scev_SpecificSInt(Param.LinearStepOrPos),
5871 ParamsOk = false;
5872 break;
5873 }
5875 break;
5876 default:
5877 ParamsOk = false;
5878 break;
5879 }
5880 }
5881
5882 if (!ParamsOk)
5883 continue;
5884
5885 // Found a suitable candidate, stop here.
5886 VecFunc = CI->getModule()->getFunction(Info.VectorName);
5887 FuncInfo = Info;
5888 break;
5889 }
5890
5891 if (TLI && VecFunc && !CI->isNoBuiltin())
5892 VectorCost = TTI.getCallInstrCost(nullptr, RetTy, Tys, CostKind);
5893
5894 // Find the cost of an intrinsic; some targets may have instructions that
5895 // perform the operation without needing an actual call.
5897 if (IID != Intrinsic::not_intrinsic)
5899
5900 InstructionCost Cost = ScalarCost;
5901 InstWidening Decision = CM_Scalarize;
5902
5903 if (VectorCost <= Cost) {
5904 Cost = VectorCost;
5905 Decision = CM_VectorCall;
5906 }
5907
5908 if (IntrinsicCost <= Cost) {
5910 Decision = CM_IntrinsicCall;
5911 }
5912
5913 setCallWideningDecision(CI, VF, Decision, VecFunc, IID,
5915 }
5916 }
5917}
5918
5920 if (!Legal->isInvariant(Op))
5921 return false;
5922 // Consider Op invariant, if it or its operands aren't predicated
5923 // instruction in the loop. In that case, it is not trivially hoistable.
5924 auto *OpI = dyn_cast<Instruction>(Op);
5925 return !OpI || !TheLoop->contains(OpI) ||
5926 (!isPredicatedInst(OpI) &&
5927 (!isa<PHINode>(OpI) || OpI->getParent() != TheLoop->getHeader()) &&
5928 all_of(OpI->operands(),
5929 [this](Value *Op) { return shouldConsiderInvariant(Op); }));
5930}
5931
5934 ElementCount VF) {
5935 // If we know that this instruction will remain uniform, check the cost of
5936 // the scalar version.
5938 VF = ElementCount::getFixed(1);
5939
5940 if (VF.isVector() && isProfitableToScalarize(I, VF))
5941 return InstsToScalarize[VF][I];
5942
5943 // Forced scalars do not have any scalarization overhead.
5944 auto ForcedScalar = ForcedScalars.find(VF);
5945 if (VF.isVector() && ForcedScalar != ForcedScalars.end()) {
5946 auto InstSet = ForcedScalar->second;
5947 if (InstSet.count(I))
5949 VF.getKnownMinValue();
5950 }
5951
5952 Type *RetTy = I->getType();
5954 RetTy = IntegerType::get(RetTy->getContext(), MinBWs[I]);
5955 auto *SE = PSE.getSE();
5956
5957 Type *VectorTy;
5958 if (isScalarAfterVectorization(I, VF)) {
5959 [[maybe_unused]] auto HasSingleCopyAfterVectorization =
5960 [this](Instruction *I, ElementCount VF) -> bool {
5961 if (VF.isScalar())
5962 return true;
5963
5964 auto Scalarized = InstsToScalarize.find(VF);
5965 assert(Scalarized != InstsToScalarize.end() &&
5966 "VF not yet analyzed for scalarization profitability");
5967 return !Scalarized->second.count(I) &&
5968 llvm::all_of(I->users(), [&](User *U) {
5969 auto *UI = cast<Instruction>(U);
5970 return !Scalarized->second.count(UI);
5971 });
5972 };
5973
5974 // With the exception of GEPs and PHIs, after scalarization there should
5975 // only be one copy of the instruction generated in the loop. This is
5976 // because the VF is either 1, or any instructions that need scalarizing
5977 // have already been dealt with by the time we get here. As a result,
5978 // it means we don't have to multiply the instruction cost by VF.
5979 assert(I->getOpcode() == Instruction::GetElementPtr ||
5980 I->getOpcode() == Instruction::PHI ||
5981 (I->getOpcode() == Instruction::BitCast &&
5982 I->getType()->isPointerTy()) ||
5983 HasSingleCopyAfterVectorization(I, VF));
5984 VectorTy = RetTy;
5985 } else
5986 VectorTy = toVectorizedTy(RetTy, VF);
5987
5988 if (VF.isVector() && VectorTy->isVectorTy() &&
5989 !TTI.getNumberOfParts(VectorTy))
5991
5992 // TODO: We need to estimate the cost of intrinsic calls.
5993 switch (I->getOpcode()) {
5994 case Instruction::GetElementPtr:
5995 // We mark this instruction as zero-cost because the cost of GEPs in
5996 // vectorized code depends on whether the corresponding memory instruction
5997 // is scalarized or not. Therefore, we handle GEPs with the memory
5998 // instruction cost.
5999 return 0;
6000 case Instruction::Br: {
6001 // In cases of scalarized and predicated instructions, there will be VF
6002 // predicated blocks in the vectorized loop. Each branch around these
6003 // blocks requires also an extract of its vector compare i1 element.
6004 // Note that the conditional branch from the loop latch will be replaced by
6005 // a single branch controlling the loop, so there is no extra overhead from
6006 // scalarization.
6007 bool ScalarPredicatedBB = false;
6009 if (VF.isVector() && BI->isConditional() &&
6010 (PredicatedBBsAfterVectorization[VF].count(BI->getSuccessor(0)) ||
6011 PredicatedBBsAfterVectorization[VF].count(BI->getSuccessor(1))) &&
6012 BI->getParent() != TheLoop->getLoopLatch())
6013 ScalarPredicatedBB = true;
6014
6015 if (ScalarPredicatedBB) {
6016 // Not possible to scalarize scalable vector with predicated instructions.
6017 if (VF.isScalable())
6019 // Return cost for branches around scalarized and predicated blocks.
6020 auto *VecI1Ty =
6022 return (
6023 TTI.getScalarizationOverhead(
6024 VecI1Ty, APInt::getAllOnes(VF.getFixedValue()),
6025 /*Insert*/ false, /*Extract*/ true, CostKind) +
6026 (TTI.getCFInstrCost(Instruction::Br, CostKind) * VF.getFixedValue()));
6027 }
6028
6029 if (I->getParent() == TheLoop->getLoopLatch() || VF.isScalar())
6030 // The back-edge branch will remain, as will all scalar branches.
6031 return TTI.getCFInstrCost(Instruction::Br, CostKind);
6032
6033 // This branch will be eliminated by if-conversion.
6034 return 0;
6035 // Note: We currently assume zero cost for an unconditional branch inside
6036 // a predicated block since it will become a fall-through, although we
6037 // may decide in the future to call TTI for all branches.
6038 }
6039 case Instruction::Switch: {
6040 if (VF.isScalar())
6041 return TTI.getCFInstrCost(Instruction::Switch, CostKind);
6042 auto *Switch = cast<SwitchInst>(I);
6043 return Switch->getNumCases() *
6044 TTI.getCmpSelInstrCost(
6045 Instruction::ICmp,
6046 toVectorTy(Switch->getCondition()->getType(), VF),
6047 toVectorTy(Type::getInt1Ty(I->getContext()), VF),
6049 }
6050 case Instruction::PHI: {
6051 auto *Phi = cast<PHINode>(I);
6052
6053 // First-order recurrences are replaced by vector shuffles inside the loop.
6054 if (VF.isVector() && Legal->isFixedOrderRecurrence(Phi)) {
6056 std::iota(Mask.begin(), Mask.end(), VF.getKnownMinValue() - 1);
6057 return TTI.getShuffleCost(TargetTransformInfo::SK_Splice,
6058 cast<VectorType>(VectorTy),
6059 cast<VectorType>(VectorTy), Mask, CostKind,
6060 VF.getKnownMinValue() - 1);
6061 }
6062
6063 // Phi nodes in non-header blocks (not inductions, reductions, etc.) are
6064 // converted into select instructions. We require N - 1 selects per phi
6065 // node, where N is the number of incoming values.
6066 if (VF.isVector() && Phi->getParent() != TheLoop->getHeader()) {
6067 Type *ResultTy = Phi->getType();
6068
6069 // All instructions in an Any-of reduction chain are narrowed to bool.
6070 // Check if that is the case for this phi node.
6071 auto *HeaderUser = cast_if_present<PHINode>(
6072 find_singleton<User>(Phi->users(), [this](User *U, bool) -> User * {
6073 auto *Phi = dyn_cast<PHINode>(U);
6074 if (Phi && Phi->getParent() == TheLoop->getHeader())
6075 return Phi;
6076 return nullptr;
6077 }));
6078 if (HeaderUser) {
6079 auto &ReductionVars = Legal->getReductionVars();
6080 auto Iter = ReductionVars.find(HeaderUser);
6081 if (Iter != ReductionVars.end() &&
6083 Iter->second.getRecurrenceKind()))
6084 ResultTy = Type::getInt1Ty(Phi->getContext());
6085 }
6086 return (Phi->getNumIncomingValues() - 1) *
6087 TTI.getCmpSelInstrCost(
6088 Instruction::Select, toVectorTy(ResultTy, VF),
6089 toVectorTy(Type::getInt1Ty(Phi->getContext()), VF),
6091 }
6092
6093 // When tail folding with EVL, if the phi is part of an out of loop
6094 // reduction then it will be transformed into a wide vp_merge.
6095 if (VF.isVector() && foldTailWithEVL() &&
6096 Legal->getReductionVars().contains(Phi) && !isInLoopReduction(Phi)) {
6098 Intrinsic::vp_merge, toVectorTy(Phi->getType(), VF),
6099 {toVectorTy(Type::getInt1Ty(Phi->getContext()), VF)});
6100 return TTI.getIntrinsicInstrCost(ICA, CostKind);
6101 }
6102
6103 return TTI.getCFInstrCost(Instruction::PHI, CostKind);
6104 }
6105 case Instruction::UDiv:
6106 case Instruction::SDiv:
6107 case Instruction::URem:
6108 case Instruction::SRem:
6109 if (VF.isVector() && isPredicatedInst(I)) {
6110 const auto [ScalarCost, SafeDivisorCost] = getDivRemSpeculationCost(I, VF);
6111 return isDivRemScalarWithPredication(ScalarCost, SafeDivisorCost) ?
6112 ScalarCost : SafeDivisorCost;
6113 }
6114 // We've proven all lanes safe to speculate, fall through.
6115 [[fallthrough]];
6116 case Instruction::Add:
6117 case Instruction::Sub: {
6118 auto Info = Legal->getHistogramInfo(I);
6119 if (Info && VF.isVector()) {
6120 const HistogramInfo *HGram = Info.value();
6121 // Assume that a non-constant update value (or a constant != 1) requires
6122 // a multiply, and add that into the cost.
6124 ConstantInt *RHS = dyn_cast<ConstantInt>(I->getOperand(1));
6125 if (!RHS || RHS->getZExtValue() != 1)
6126 MulCost =
6127 TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy, CostKind);
6128
6129 // Find the cost of the histogram operation itself.
6130 Type *PtrTy = VectorType::get(HGram->Load->getPointerOperandType(), VF);
6131 Type *ScalarTy = I->getType();
6132 Type *MaskTy = VectorType::get(Type::getInt1Ty(I->getContext()), VF);
6133 IntrinsicCostAttributes ICA(Intrinsic::experimental_vector_histogram_add,
6134 Type::getVoidTy(I->getContext()),
6135 {PtrTy, ScalarTy, MaskTy});
6136
6137 // Add the costs together with the add/sub operation.
6138 return TTI.getIntrinsicInstrCost(ICA, CostKind) + MulCost +
6139 TTI.getArithmeticInstrCost(I->getOpcode(), VectorTy, CostKind);
6140 }
6141 [[fallthrough]];
6142 }
6143 case Instruction::FAdd:
6144 case Instruction::FSub:
6145 case Instruction::Mul:
6146 case Instruction::FMul:
6147 case Instruction::FDiv:
6148 case Instruction::FRem:
6149 case Instruction::Shl:
6150 case Instruction::LShr:
6151 case Instruction::AShr:
6152 case Instruction::And:
6153 case Instruction::Or:
6154 case Instruction::Xor: {
6155 // If we're speculating on the stride being 1, the multiplication may
6156 // fold away. We can generalize this for all operations using the notion
6157 // of neutral elements. (TODO)
6158 if (I->getOpcode() == Instruction::Mul &&
6159 ((TheLoop->isLoopInvariant(I->getOperand(0)) &&
6160 PSE.getSCEV(I->getOperand(0))->isOne()) ||
6161 (TheLoop->isLoopInvariant(I->getOperand(1)) &&
6162 PSE.getSCEV(I->getOperand(1))->isOne())))
6163 return 0;
6164
6165 // Detect reduction patterns
6166 if (auto RedCost = getReductionPatternCost(I, VF, VectorTy))
6167 return *RedCost;
6168
6169 // Certain instructions can be cheaper to vectorize if they have a constant
6170 // second vector operand. One example of this are shifts on x86.
6171 Value *Op2 = I->getOperand(1);
6172 if (!isa<Constant>(Op2) && TheLoop->isLoopInvariant(Op2) &&
6173 PSE.getSE()->isSCEVable(Op2->getType()) &&
6174 isa<SCEVConstant>(PSE.getSCEV(Op2))) {
6175 Op2 = cast<SCEVConstant>(PSE.getSCEV(Op2))->getValue();
6176 }
6177 auto Op2Info = TTI.getOperandInfo(Op2);
6178 if (Op2Info.Kind == TargetTransformInfo::OK_AnyValue &&
6181
6182 SmallVector<const Value *, 4> Operands(I->operand_values());
6183 return TTI.getArithmeticInstrCost(
6184 I->getOpcode(), VectorTy, CostKind,
6185 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
6186 Op2Info, Operands, I, TLI);
6187 }
6188 case Instruction::FNeg: {
6189 return TTI.getArithmeticInstrCost(
6190 I->getOpcode(), VectorTy, CostKind,
6191 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
6192 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
6193 I->getOperand(0), I);
6194 }
6195 case Instruction::Select: {
6197 const SCEV *CondSCEV = SE->getSCEV(SI->getCondition());
6198 bool ScalarCond = (SE->isLoopInvariant(CondSCEV, TheLoop));
6199
6200 const Value *Op0, *Op1;
6201 using namespace llvm::PatternMatch;
6202 if (!ScalarCond && (match(I, m_LogicalAnd(m_Value(Op0), m_Value(Op1))) ||
6203 match(I, m_LogicalOr(m_Value(Op0), m_Value(Op1))))) {
6204 // select x, y, false --> x & y
6205 // select x, true, y --> x | y
6206 const auto [Op1VK, Op1VP] = TTI::getOperandInfo(Op0);
6207 const auto [Op2VK, Op2VP] = TTI::getOperandInfo(Op1);
6208 assert(Op0->getType()->getScalarSizeInBits() == 1 &&
6209 Op1->getType()->getScalarSizeInBits() == 1);
6210
6211 return TTI.getArithmeticInstrCost(
6212 match(I, m_LogicalOr()) ? Instruction::Or : Instruction::And,
6213 VectorTy, CostKind, {Op1VK, Op1VP}, {Op2VK, Op2VP}, {Op0, Op1}, I);
6214 }
6215
6216 Type *CondTy = SI->getCondition()->getType();
6217 if (!ScalarCond)
6218 CondTy = VectorType::get(CondTy, VF);
6219
6221 if (auto *Cmp = dyn_cast<CmpInst>(SI->getCondition()))
6222 Pred = Cmp->getPredicate();
6223 return TTI.getCmpSelInstrCost(I->getOpcode(), VectorTy, CondTy, Pred,
6224 CostKind, {TTI::OK_AnyValue, TTI::OP_None},
6225 {TTI::OK_AnyValue, TTI::OP_None}, I);
6226 }
6227 case Instruction::ICmp:
6228 case Instruction::FCmp: {
6229 Type *ValTy = I->getOperand(0)->getType();
6230
6232 [[maybe_unused]] Instruction *Op0AsInstruction =
6233 dyn_cast<Instruction>(I->getOperand(0));
6234 assert((!canTruncateToMinimalBitwidth(Op0AsInstruction, VF) ||
6235 MinBWs[I] == MinBWs[Op0AsInstruction]) &&
6236 "if both the operand and the compare are marked for "
6237 "truncation, they must have the same bitwidth");
6238 ValTy = IntegerType::get(ValTy->getContext(), MinBWs[I]);
6239 }
6240
6241 VectorTy = toVectorTy(ValTy, VF);
6242 return TTI.getCmpSelInstrCost(
6243 I->getOpcode(), VectorTy, CmpInst::makeCmpResultType(VectorTy),
6244 cast<CmpInst>(I)->getPredicate(), CostKind,
6245 {TTI::OK_AnyValue, TTI::OP_None}, {TTI::OK_AnyValue, TTI::OP_None}, I);
6246 }
6247 case Instruction::Store:
6248 case Instruction::Load: {
6249 ElementCount Width = VF;
6250 if (Width.isVector()) {
6251 InstWidening Decision = getWideningDecision(I, Width);
6252 assert(Decision != CM_Unknown &&
6253 "CM decision should be taken at this point");
6256 if (Decision == CM_Scalarize)
6257 Width = ElementCount::getFixed(1);
6258 }
6259 VectorTy = toVectorTy(getLoadStoreType(I), Width);
6260 return getMemoryInstructionCost(I, VF);
6261 }
6262 case Instruction::BitCast:
6263 if (I->getType()->isPointerTy())
6264 return 0;
6265 [[fallthrough]];
6266 case Instruction::ZExt:
6267 case Instruction::SExt:
6268 case Instruction::FPToUI:
6269 case Instruction::FPToSI:
6270 case Instruction::FPExt:
6271 case Instruction::PtrToInt:
6272 case Instruction::IntToPtr:
6273 case Instruction::SIToFP:
6274 case Instruction::UIToFP:
6275 case Instruction::Trunc:
6276 case Instruction::FPTrunc: {
6277 // Computes the CastContextHint from a Load/Store instruction.
6278 auto ComputeCCH = [&](Instruction *I) -> TTI::CastContextHint {
6280 "Expected a load or a store!");
6281
6282 if (VF.isScalar() || !TheLoop->contains(I))
6284
6285 switch (getWideningDecision(I, VF)) {
6297 llvm_unreachable("Instr did not go through cost modelling?");
6300 llvm_unreachable_internal("Instr has invalid widening decision");
6301 }
6302
6303 llvm_unreachable("Unhandled case!");
6304 };
6305
6306 unsigned Opcode = I->getOpcode();
6308 // For Trunc, the context is the only user, which must be a StoreInst.
6309 if (Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) {
6310 if (I->hasOneUse())
6311 if (StoreInst *Store = dyn_cast<StoreInst>(*I->user_begin()))
6312 CCH = ComputeCCH(Store);
6313 }
6314 // For Z/Sext, the context is the operand, which must be a LoadInst.
6315 else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt ||
6316 Opcode == Instruction::FPExt) {
6317 if (LoadInst *Load = dyn_cast<LoadInst>(I->getOperand(0)))
6318 CCH = ComputeCCH(Load);
6319 }
6320
6321 // We optimize the truncation of induction variables having constant
6322 // integer steps. The cost of these truncations is the same as the scalar
6323 // operation.
6324 if (isOptimizableIVTruncate(I, VF)) {
6325 auto *Trunc = cast<TruncInst>(I);
6326 return TTI.getCastInstrCost(Instruction::Trunc, Trunc->getDestTy(),
6327 Trunc->getSrcTy(), CCH, CostKind, Trunc);
6328 }
6329
6330 // Detect reduction patterns
6331 if (auto RedCost = getReductionPatternCost(I, VF, VectorTy))
6332 return *RedCost;
6333
6334 Type *SrcScalarTy = I->getOperand(0)->getType();
6335 Instruction *Op0AsInstruction = dyn_cast<Instruction>(I->getOperand(0));
6336 if (canTruncateToMinimalBitwidth(Op0AsInstruction, VF))
6337 SrcScalarTy =
6338 IntegerType::get(SrcScalarTy->getContext(), MinBWs[Op0AsInstruction]);
6339 Type *SrcVecTy =
6340 VectorTy->isVectorTy() ? toVectorTy(SrcScalarTy, VF) : SrcScalarTy;
6341
6343 // If the result type is <= the source type, there will be no extend
6344 // after truncating the users to the minimal required bitwidth.
6345 if (VectorTy->getScalarSizeInBits() <= SrcVecTy->getScalarSizeInBits() &&
6346 (I->getOpcode() == Instruction::ZExt ||
6347 I->getOpcode() == Instruction::SExt))
6348 return 0;
6349 }
6350
6351 return TTI.getCastInstrCost(Opcode, VectorTy, SrcVecTy, CCH, CostKind, I);
6352 }
6353 case Instruction::Call:
6354 return getVectorCallCost(cast<CallInst>(I), VF);
6355 case Instruction::ExtractValue:
6356 return TTI.getInstructionCost(I, CostKind);
6357 case Instruction::Alloca:
6358 // We cannot easily widen alloca to a scalable alloca, as
6359 // the result would need to be a vector of pointers.
6360 if (VF.isScalable())
6362 [[fallthrough]];
6363 default:
6364 // This opcode is unknown. Assume that it is the same as 'mul'.
6365 return TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy, CostKind);
6366 } // end of switch.
6367}
6368
6370 // Ignore ephemeral values.
6372
6373 SmallVector<Value *, 4> DeadInterleavePointerOps;
6375
6376 // If a scalar epilogue is required, users outside the loop won't use
6377 // live-outs from the vector loop but from the scalar epilogue. Ignore them if
6378 // that is the case.
6379 bool RequiresScalarEpilogue = requiresScalarEpilogue(true);
6380 auto IsLiveOutDead = [this, RequiresScalarEpilogue](User *U) {
6381 return RequiresScalarEpilogue &&
6382 !TheLoop->contains(cast<Instruction>(U)->getParent());
6383 };
6384
6386 DFS.perform(LI);
6387 MapVector<Value *, SmallVector<Value *>> DeadInvariantStoreOps;
6388 for (BasicBlock *BB : reverse(make_range(DFS.beginRPO(), DFS.endRPO())))
6389 for (Instruction &I : reverse(*BB)) {
6390 // Find all stores to invariant variables. Since they are going to sink
6391 // outside the loop we do not need calculate cost for them.
6392 StoreInst *SI;
6393 if ((SI = dyn_cast<StoreInst>(&I)) &&
6394 Legal->isInvariantAddressOfReduction(SI->getPointerOperand())) {
6395 ValuesToIgnore.insert(&I);
6396 DeadInvariantStoreOps[SI->getPointerOperand()].push_back(
6397 SI->getValueOperand());
6398 }
6399
6400 if (VecValuesToIgnore.contains(&I) || ValuesToIgnore.contains(&I))
6401 continue;
6402
6403 // Add instructions that would be trivially dead and are only used by
6404 // values already ignored to DeadOps to seed worklist.
6406 all_of(I.users(), [this, IsLiveOutDead](User *U) {
6407 return VecValuesToIgnore.contains(U) ||
6408 ValuesToIgnore.contains(U) || IsLiveOutDead(U);
6409 }))
6410 DeadOps.push_back(&I);
6411
6412 // For interleave groups, we only create a pointer for the start of the
6413 // interleave group. Queue up addresses of group members except the insert
6414 // position for further processing.
6415 if (isAccessInterleaved(&I)) {
6416 auto *Group = getInterleavedAccessGroup(&I);
6417 if (Group->getInsertPos() == &I)
6418 continue;
6419 Value *PointerOp = getLoadStorePointerOperand(&I);
6420 DeadInterleavePointerOps.push_back(PointerOp);
6421 }
6422
6423 // Queue branches for analysis. They are dead, if their successors only
6424 // contain dead instructions.
6425 if (auto *Br = dyn_cast<BranchInst>(&I)) {
6426 if (Br->isConditional())
6427 DeadOps.push_back(&I);
6428 }
6429 }
6430
6431 // Mark ops feeding interleave group members as free, if they are only used
6432 // by other dead computations.
6433 for (unsigned I = 0; I != DeadInterleavePointerOps.size(); ++I) {
6434 auto *Op = dyn_cast<Instruction>(DeadInterleavePointerOps[I]);
6435 if (!Op || !TheLoop->contains(Op) || any_of(Op->users(), [this](User *U) {
6436 Instruction *UI = cast<Instruction>(U);
6437 return !VecValuesToIgnore.contains(U) &&
6438 (!isAccessInterleaved(UI) ||
6439 getInterleavedAccessGroup(UI)->getInsertPos() == UI);
6440 }))
6441 continue;
6442 VecValuesToIgnore.insert(Op);
6443 append_range(DeadInterleavePointerOps, Op->operands());
6444 }
6445
6446 for (const auto &[_, Ops] : DeadInvariantStoreOps)
6447 llvm::append_range(DeadOps, drop_end(Ops));
6448
6449 // Mark ops that would be trivially dead and are only used by ignored
6450 // instructions as free.
6451 BasicBlock *Header = TheLoop->getHeader();
6452
6453 // Returns true if the block contains only dead instructions. Such blocks will
6454 // be removed by VPlan-to-VPlan transforms and won't be considered by the
6455 // VPlan-based cost model, so skip them in the legacy cost-model as well.
6456 auto IsEmptyBlock = [this](BasicBlock *BB) {
6457 return all_of(*BB, [this](Instruction &I) {
6458 return ValuesToIgnore.contains(&I) || VecValuesToIgnore.contains(&I) ||
6459 (isa<BranchInst>(&I) && !cast<BranchInst>(&I)->isConditional());
6460 });
6461 };
6462 for (unsigned I = 0; I != DeadOps.size(); ++I) {
6463 auto *Op = dyn_cast<Instruction>(DeadOps[I]);
6464
6465 // Check if the branch should be considered dead.
6466 if (auto *Br = dyn_cast_or_null<BranchInst>(Op)) {
6467 BasicBlock *ThenBB = Br->getSuccessor(0);
6468 BasicBlock *ElseBB = Br->getSuccessor(1);
6469 // Don't considers branches leaving the loop for simplification.
6470 if (!TheLoop->contains(ThenBB) || !TheLoop->contains(ElseBB))
6471 continue;
6472 bool ThenEmpty = IsEmptyBlock(ThenBB);
6473 bool ElseEmpty = IsEmptyBlock(ElseBB);
6474 if ((ThenEmpty && ElseEmpty) ||
6475 (ThenEmpty && ThenBB->getSingleSuccessor() == ElseBB &&
6476 ElseBB->phis().empty()) ||
6477 (ElseEmpty && ElseBB->getSingleSuccessor() == ThenBB &&
6478 ThenBB->phis().empty())) {
6479 VecValuesToIgnore.insert(Br);
6480 DeadOps.push_back(Br->getCondition());
6481 }
6482 continue;
6483 }
6484
6485 // Skip any op that shouldn't be considered dead.
6486 if (!Op || !TheLoop->contains(Op) ||
6487 (isa<PHINode>(Op) && Op->getParent() == Header) ||
6489 any_of(Op->users(), [this, IsLiveOutDead](User *U) {
6490 return !VecValuesToIgnore.contains(U) &&
6491 !ValuesToIgnore.contains(U) && !IsLiveOutDead(U);
6492 }))
6493 continue;
6494
6495 // If all of Op's users are in ValuesToIgnore, add it to ValuesToIgnore
6496 // which applies for both scalar and vector versions. Otherwise it is only
6497 // dead in vector versions, so only add it to VecValuesToIgnore.
6498 if (all_of(Op->users(),
6499 [this](User *U) { return ValuesToIgnore.contains(U); }))
6500 ValuesToIgnore.insert(Op);
6501
6502 VecValuesToIgnore.insert(Op);
6503 append_range(DeadOps, Op->operands());
6504 }
6505
6506 // Ignore type-promoting instructions we identified during reduction
6507 // detection.
6508 for (const auto &Reduction : Legal->getReductionVars()) {
6509 const RecurrenceDescriptor &RedDes = Reduction.second;
6510 const SmallPtrSetImpl<Instruction *> &Casts = RedDes.getCastInsts();
6511 VecValuesToIgnore.insert_range(Casts);
6512 }
6513 // Ignore type-casting instructions we identified during induction
6514 // detection.
6515 for (const auto &Induction : Legal->getInductionVars()) {
6516 const InductionDescriptor &IndDes = Induction.second;
6517 const SmallVectorImpl<Instruction *> &Casts = IndDes.getCastInsts();
6518 VecValuesToIgnore.insert_range(Casts);
6519 }
6520}
6521
6523 // Avoid duplicating work finding in-loop reductions.
6524 if (!InLoopReductions.empty())
6525 return;
6526
6527 for (const auto &Reduction : Legal->getReductionVars()) {
6528 PHINode *Phi = Reduction.first;
6529 const RecurrenceDescriptor &RdxDesc = Reduction.second;
6530
6531 // We don't collect reductions that are type promoted (yet).
6532 if (RdxDesc.getRecurrenceType() != Phi->getType())
6533 continue;
6534
6535 // If the target would prefer this reduction to happen "in-loop", then we
6536 // want to record it as such.
6537 RecurKind Kind = RdxDesc.getRecurrenceKind();
6538 if (!PreferInLoopReductions && !useOrderedReductions(RdxDesc) &&
6539 !TTI.preferInLoopReduction(Kind, Phi->getType()))
6540 continue;
6541
6542 // Check that we can correctly put the reductions into the loop, by
6543 // finding the chain of operations that leads from the phi to the loop
6544 // exit value.
6545 SmallVector<Instruction *, 4> ReductionOperations =
6546 RdxDesc.getReductionOpChain(Phi, TheLoop);
6547 bool InLoop = !ReductionOperations.empty();
6548
6549 if (InLoop) {
6550 InLoopReductions.insert(Phi);
6551 // Add the elements to InLoopReductionImmediateChains for cost modelling.
6552 Instruction *LastChain = Phi;
6553 for (auto *I : ReductionOperations) {
6554 InLoopReductionImmediateChains[I] = LastChain;
6555 LastChain = I;
6556 }
6557 }
6558 LLVM_DEBUG(dbgs() << "LV: Using " << (InLoop ? "inloop" : "out of loop")
6559 << " reduction for phi: " << *Phi << "\n");
6560 }
6561}
6562
6563// This function will select a scalable VF if the target supports scalable
6564// vectors and a fixed one otherwise.
6565// TODO: we could return a pair of values that specify the max VF and
6566// min VF, to be used in `buildVPlans(MinVF, MaxVF)` instead of
6567// `buildVPlans(VF, VF)`. We cannot do it because VPLAN at the moment
6568// doesn't have a cost model that can choose which plan to execute if
6569// more than one is generated.
6572 unsigned WidestType;
6573 std::tie(std::ignore, WidestType) = CM.getSmallestAndWidestTypes();
6574
6576 TTI.enableScalableVectorization()
6579
6580 TypeSize RegSize = TTI.getRegisterBitWidth(RegKind);
6581 unsigned N = RegSize.getKnownMinValue() / WidestType;
6582 return ElementCount::get(N, RegSize.isScalable());
6583}
6584
6587 ElementCount VF = UserVF;
6588 // Outer loop handling: They may require CFG and instruction level
6589 // transformations before even evaluating whether vectorization is profitable.
6590 // Since we cannot modify the incoming IR, we need to build VPlan upfront in
6591 // the vectorization pipeline.
6592 if (!OrigLoop->isInnermost()) {
6593 // If the user doesn't provide a vectorization factor, determine a
6594 // reasonable one.
6595 if (UserVF.isZero()) {
6596 VF = determineVPlanVF(TTI, CM);
6597 LLVM_DEBUG(dbgs() << "LV: VPlan computed VF " << VF << ".\n");
6598
6599 // Make sure we have a VF > 1 for stress testing.
6600 if (VPlanBuildStressTest && (VF.isScalar() || VF.isZero())) {
6601 LLVM_DEBUG(dbgs() << "LV: VPlan stress testing: "
6602 << "overriding computed VF.\n");
6603 VF = ElementCount::getFixed(4);
6604 }
6605 } else if (UserVF.isScalable() && !TTI.supportsScalableVectors() &&
6607 LLVM_DEBUG(dbgs() << "LV: Not vectorizing. Scalable VF requested, but "
6608 << "not supported by the target.\n");
6610 "Scalable vectorization requested but not supported by the target",
6611 "the scalable user-specified vectorization width for outer-loop "
6612 "vectorization cannot be used because the target does not support "
6613 "scalable vectors.",
6614 "ScalableVFUnfeasible", ORE, OrigLoop);
6616 }
6617 assert(EnableVPlanNativePath && "VPlan-native path is not enabled.");
6619 "VF needs to be a power of two");
6620 LLVM_DEBUG(dbgs() << "LV: Using " << (!UserVF.isZero() ? "user " : "")
6621 << "VF " << VF << " to build VPlans.\n");
6622 buildVPlans(VF, VF);
6623
6624 if (VPlans.empty())
6626
6627 // For VPlan build stress testing, we bail out after VPlan construction.
6630
6631 return {VF, 0 /*Cost*/, 0 /* ScalarCost */};
6632 }
6633
6634 LLVM_DEBUG(
6635 dbgs() << "LV: Not vectorizing. Inner loops aren't supported in the "
6636 "VPlan-native path.\n");
6638}
6639
6640void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
6641 assert(OrigLoop->isInnermost() && "Inner loop expected.");
6642 CM.collectValuesToIgnore();
6643 CM.collectElementTypesForWidening();
6644
6645 FixedScalableVFPair MaxFactors = CM.computeMaxVF(UserVF, UserIC);
6646 if (!MaxFactors) // Cases that should not to be vectorized nor interleaved.
6647 return;
6648
6649 // Invalidate interleave groups if all blocks of loop will be predicated.
6650 if (CM.blockNeedsPredicationForAnyReason(OrigLoop->getHeader()) &&
6652 LLVM_DEBUG(
6653 dbgs()
6654 << "LV: Invalidate all interleaved groups due to fold-tail by masking "
6655 "which requires masked-interleaved support.\n");
6656 if (CM.InterleaveInfo.invalidateGroups())
6657 // Invalidating interleave groups also requires invalidating all decisions
6658 // based on them, which includes widening decisions and uniform and scalar
6659 // values.
6660 CM.invalidateCostModelingDecisions();
6661 }
6662
6663 if (CM.foldTailByMasking())
6664 Legal->prepareToFoldTailByMasking();
6665
6666 ElementCount MaxUserVF =
6667 UserVF.isScalable() ? MaxFactors.ScalableVF : MaxFactors.FixedVF;
6668 if (UserVF) {
6669 if (!ElementCount::isKnownLE(UserVF, MaxUserVF)) {
6671 "UserVF ignored because it may be larger than the maximal safe VF",
6672 "InvalidUserVF", ORE, OrigLoop);
6673 } else {
6675 "VF needs to be a power of two");
6676 // Collect the instructions (and their associated costs) that will be more
6677 // profitable to scalarize.
6678 CM.collectInLoopReductions();
6679 if (CM.selectUserVectorizationFactor(UserVF)) {
6680 LLVM_DEBUG(dbgs() << "LV: Using user VF " << UserVF << ".\n");
6681 buildVPlansWithVPRecipes(UserVF, UserVF);
6683 return;
6684 }
6685 reportVectorizationInfo("UserVF ignored because of invalid costs.",
6686 "InvalidCost", ORE, OrigLoop);
6687 }
6688 }
6689
6690 // Collect the Vectorization Factor Candidates.
6691 SmallVector<ElementCount> VFCandidates;
6692 for (auto VF = ElementCount::getFixed(1);
6693 ElementCount::isKnownLE(VF, MaxFactors.FixedVF); VF *= 2)
6694 VFCandidates.push_back(VF);
6695 for (auto VF = ElementCount::getScalable(1);
6696 ElementCount::isKnownLE(VF, MaxFactors.ScalableVF); VF *= 2)
6697 VFCandidates.push_back(VF);
6698
6699 CM.collectInLoopReductions();
6700 for (const auto &VF : VFCandidates) {
6701 // Collect Uniform and Scalar instructions after vectorization with VF.
6702 CM.collectNonVectorizedAndSetWideningDecisions(VF);
6703 }
6704
6705 buildVPlansWithVPRecipes(ElementCount::getFixed(1), MaxFactors.FixedVF);
6706 buildVPlansWithVPRecipes(ElementCount::getScalable(1), MaxFactors.ScalableVF);
6707
6709}
6710
6712 ElementCount VF) const {
6713 InstructionCost Cost = CM.getInstructionCost(UI, VF);
6714 if (Cost.isValid() && ForceTargetInstructionCost.getNumOccurrences())
6716 return Cost;
6717}
6718
6720 ElementCount VF) const {
6721 return CM.isUniformAfterVectorization(I, VF);
6722}
6723
6724bool VPCostContext::skipCostComputation(Instruction *UI, bool IsVector) const {
6725 return CM.ValuesToIgnore.contains(UI) ||
6726 (IsVector && CM.VecValuesToIgnore.contains(UI)) ||
6727 SkipCostComputation.contains(UI);
6728}
6729
6731LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
6732 VPCostContext &CostCtx) const {
6734 // Cost modeling for inductions is inaccurate in the legacy cost model
6735 // compared to the recipes that are generated. To match here initially during
6736 // VPlan cost model bring up directly use the induction costs from the legacy
6737 // cost model. Note that we do this as pre-processing; the VPlan may not have
6738 // any recipes associated with the original induction increment instruction
6739 // and may replace truncates with VPWidenIntOrFpInductionRecipe. We precompute
6740 // the cost of induction phis and increments (both that are represented by
6741 // recipes and those that are not), to avoid distinguishing between them here,
6742 // and skip all recipes that represent induction phis and increments (the
6743 // former case) later on, if they exist, to avoid counting them twice.
6744 // Similarly we pre-compute the cost of any optimized truncates.
6745 // TODO: Switch to more accurate costing based on VPlan.
6746 for (const auto &[IV, IndDesc] : Legal->getInductionVars()) {
6748 IV->getIncomingValueForBlock(OrigLoop->getLoopLatch()));
6749 SmallVector<Instruction *> IVInsts = {IVInc};
6750 for (unsigned I = 0; I != IVInsts.size(); I++) {
6751 for (Value *Op : IVInsts[I]->operands()) {
6752 auto *OpI = dyn_cast<Instruction>(Op);
6753 if (Op == IV || !OpI || !OrigLoop->contains(OpI) || !Op->hasOneUse())
6754 continue;
6755 IVInsts.push_back(OpI);
6756 }
6757 }
6758 IVInsts.push_back(IV);
6759 for (User *U : IV->users()) {
6760 auto *CI = cast<Instruction>(U);
6761 if (!CostCtx.CM.isOptimizableIVTruncate(CI, VF))
6762 continue;
6763 IVInsts.push_back(CI);
6764 }
6765
6766 // If the vector loop gets executed exactly once with the given VF, ignore
6767 // the costs of comparison and induction instructions, as they'll get
6768 // simplified away.
6769 // TODO: Remove this code after stepping away from the legacy cost model and
6770 // adding code to simplify VPlans before calculating their costs.
6771 auto TC = getSmallConstantTripCount(PSE.getSE(), OrigLoop);
6772 if (TC == VF && !CM.foldTailByMasking())
6773 addFullyUnrolledInstructionsToIgnore(OrigLoop, Legal->getInductionVars(),
6774 CostCtx.SkipCostComputation);
6775
6776 for (Instruction *IVInst : IVInsts) {
6777 if (CostCtx.skipCostComputation(IVInst, VF.isVector()))
6778 continue;
6779 InstructionCost InductionCost = CostCtx.getLegacyCost(IVInst, VF);
6780 LLVM_DEBUG({
6781 dbgs() << "Cost of " << InductionCost << " for VF " << VF
6782 << ": induction instruction " << *IVInst << "\n";
6783 });
6784 Cost += InductionCost;
6785 CostCtx.SkipCostComputation.insert(IVInst);
6786 }
6787 }
6788
6789 /// Compute the cost of all exiting conditions of the loop using the legacy
6790 /// cost model. This is to match the legacy behavior, which adds the cost of
6791 /// all exit conditions. Note that this over-estimates the cost, as there will
6792 /// be a single condition to control the vector loop.
6794 CM.TheLoop->getExitingBlocks(Exiting);
6795 SetVector<Instruction *> ExitInstrs;
6796 // Collect all exit conditions.
6797 for (BasicBlock *EB : Exiting) {
6798 auto *Term = dyn_cast<BranchInst>(EB->getTerminator());
6799 if (!Term || CostCtx.skipCostComputation(Term, VF.isVector()))
6800 continue;
6801 if (auto *CondI = dyn_cast<Instruction>(Term->getOperand(0))) {
6802 ExitInstrs.insert(CondI);
6803 }
6804 }
6805 // Compute the cost of all instructions only feeding the exit conditions.
6806 for (unsigned I = 0; I != ExitInstrs.size(); ++I) {
6807 Instruction *CondI = ExitInstrs[I];
6808 if (!OrigLoop->contains(CondI) ||
6809 !CostCtx.SkipCostComputation.insert(CondI).second)
6810 continue;
6811 InstructionCost CondICost = CostCtx.getLegacyCost(CondI, VF);
6812 LLVM_DEBUG({
6813 dbgs() << "Cost of " << CondICost << " for VF " << VF
6814 << ": exit condition instruction " << *CondI << "\n";
6815 });
6816 Cost += CondICost;
6817 for (Value *Op : CondI->operands()) {
6818 auto *OpI = dyn_cast<Instruction>(Op);
6819 if (!OpI || CostCtx.skipCostComputation(OpI, VF.isVector()) ||
6820 any_of(OpI->users(), [&ExitInstrs, this](User *U) {
6821 return OrigLoop->contains(cast<Instruction>(U)->getParent()) &&
6822 !ExitInstrs.contains(cast<Instruction>(U));
6823 }))
6824 continue;
6825 ExitInstrs.insert(OpI);
6826 }
6827 }
6828
6829 // Pre-compute the costs for branches except for the backedge, as the number
6830 // of replicate regions in a VPlan may not directly match the number of
6831 // branches, which would lead to different decisions.
6832 // TODO: Compute cost of branches for each replicate region in the VPlan,
6833 // which is more accurate than the legacy cost model.
6834 for (BasicBlock *BB : OrigLoop->blocks()) {
6835 if (CostCtx.skipCostComputation(BB->getTerminator(), VF.isVector()))
6836 continue;
6837 CostCtx.SkipCostComputation.insert(BB->getTerminator());
6838 if (BB == OrigLoop->getLoopLatch())
6839 continue;
6840 auto BranchCost = CostCtx.getLegacyCost(BB->getTerminator(), VF);
6841 Cost += BranchCost;
6842 }
6843
6844 // Pre-compute costs for instructions that are forced-scalar or profitable to
6845 // scalarize. Their costs will be computed separately in the legacy cost
6846 // model.
6847 for (Instruction *ForcedScalar : CM.ForcedScalars[VF]) {
6848 if (CostCtx.skipCostComputation(ForcedScalar, VF.isVector()))
6849 continue;
6850 CostCtx.SkipCostComputation.insert(ForcedScalar);
6851 InstructionCost ForcedCost = CostCtx.getLegacyCost(ForcedScalar, VF);
6852 LLVM_DEBUG({
6853 dbgs() << "Cost of " << ForcedCost << " for VF " << VF
6854 << ": forced scalar " << *ForcedScalar << "\n";
6855 });
6856 Cost += ForcedCost;
6857 }
6858 for (const auto &[Scalarized, ScalarCost] : CM.InstsToScalarize[VF]) {
6859 if (CostCtx.skipCostComputation(Scalarized, VF.isVector()))
6860 continue;
6861 CostCtx.SkipCostComputation.insert(Scalarized);
6862 LLVM_DEBUG({
6863 dbgs() << "Cost of " << ScalarCost << " for VF " << VF
6864 << ": profitable to scalarize " << *Scalarized << "\n";
6865 });
6866 Cost += ScalarCost;
6867 }
6868
6869 return Cost;
6870}
6871
6872InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan,
6873 ElementCount VF) const {
6874 VPCostContext CostCtx(CM.TTI, *CM.TLI, Plan, CM, CM.CostKind);
6875 InstructionCost Cost = precomputeCosts(Plan, VF, CostCtx);
6876
6877 // Now compute and add the VPlan-based cost.
6878 Cost += Plan.cost(VF, CostCtx);
6879#ifndef NDEBUG
6880 unsigned EstimatedWidth = estimateElementCount(VF, CM.getVScaleForTuning());
6881 LLVM_DEBUG(dbgs() << "Cost for VF " << VF << ": " << Cost
6882 << " (Estimated cost per lane: ");
6883 if (Cost.isValid()) {
6884 double CostPerLane = double(Cost.getValue()) / EstimatedWidth;
6885 LLVM_DEBUG(dbgs() << format("%.1f", CostPerLane));
6886 } else /* No point dividing an invalid cost - it will still be invalid */
6887 LLVM_DEBUG(dbgs() << "Invalid");
6888 LLVM_DEBUG(dbgs() << ")\n");
6889#endif
6890 return Cost;
6891}
6892
6893#ifndef NDEBUG
6894/// Return true if the original loop \ TheLoop contains any instructions that do
6895/// not have corresponding recipes in \p Plan and are not marked to be ignored
6896/// in \p CostCtx. This means the VPlan contains simplification that the legacy
6897/// cost-model did not account for.
6899 VPCostContext &CostCtx,
6900 Loop *TheLoop,
6901 ElementCount VF) {
6902 // First collect all instructions for the recipes in Plan.
6903 auto GetInstructionForCost = [](const VPRecipeBase *R) -> Instruction * {
6904 if (auto *S = dyn_cast<VPSingleDefRecipe>(R))
6905 return dyn_cast_or_null<Instruction>(S->getUnderlyingValue());
6906 if (auto *WidenMem = dyn_cast<VPWidenMemoryRecipe>(R))
6907 return &WidenMem->getIngredient();
6908 return nullptr;
6909 };
6910
6911 DenseSet<Instruction *> SeenInstrs;
6912 auto Iter = vp_depth_first_deep(Plan.getVectorLoopRegion()->getEntry());
6914 for (VPRecipeBase &R : *VPBB) {
6915 if (auto *IR = dyn_cast<VPInterleaveRecipe>(&R)) {
6916 auto *IG = IR->getInterleaveGroup();
6917 unsigned NumMembers = IG->getNumMembers();
6918 for (unsigned I = 0; I != NumMembers; ++I) {
6919 if (Instruction *M = IG->getMember(I))
6920 SeenInstrs.insert(M);
6921 }
6922 continue;
6923 }
6924 // Unused FOR splices are removed by VPlan transforms, so the VPlan-based
6925 // cost model won't cost it whilst the legacy will.
6926 if (auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(&R)) {
6927 using namespace VPlanPatternMatch;
6928 if (none_of(FOR->users(),
6929 match_fn(m_VPInstruction<
6931 return true;
6932 }
6933 // The VPlan-based cost model is more accurate for partial reduction and
6934 // comparing against the legacy cost isn't desirable.
6936 return true;
6937
6938 // The VPlan-based cost model can analyze if recipes are scalar
6939 // recursively, but the legacy cost model cannot.
6940 if (auto *WidenMemR = dyn_cast<VPWidenMemoryRecipe>(&R)) {
6941 auto *AddrI = dyn_cast<Instruction>(
6942 getLoadStorePointerOperand(&WidenMemR->getIngredient()));
6943 if (AddrI && vputils::isSingleScalar(WidenMemR->getAddr()) !=
6944 CostCtx.isLegacyUniformAfterVectorization(AddrI, VF))
6945 return true;
6946 }
6947
6948 /// If a VPlan transform folded a recipe to one producing a single-scalar,
6949 /// but the original instruction wasn't uniform-after-vectorization in the
6950 /// legacy cost model, the legacy cost overestimates the actual cost.
6951 if (auto *RepR = dyn_cast<VPReplicateRecipe>(&R)) {
6952 if (RepR->isSingleScalar() &&
6954 RepR->getUnderlyingInstr(), VF))
6955 return true;
6956 }
6957 if (Instruction *UI = GetInstructionForCost(&R)) {
6958 // If we adjusted the predicate of the recipe, the cost in the legacy
6959 // cost model may be different.
6960 using namespace VPlanPatternMatch;
6961 CmpPredicate Pred;
6962 if (match(&R, m_Cmp(Pred, m_VPValue(), m_VPValue())) &&
6963 cast<VPRecipeWithIRFlags>(R).getPredicate() !=
6964 cast<CmpInst>(UI)->getPredicate())
6965 return true;
6966 SeenInstrs.insert(UI);
6967 }
6968 }
6969 }
6970
6971 // Return true if the loop contains any instructions that are not also part of
6972 // the VPlan or are skipped for VPlan-based cost computations. This indicates
6973 // that the VPlan contains extra simplifications.
6974 return any_of(TheLoop->blocks(), [&SeenInstrs, &CostCtx,
6975 TheLoop](BasicBlock *BB) {
6976 return any_of(*BB, [&SeenInstrs, &CostCtx, TheLoop, BB](Instruction &I) {
6977 // Skip induction phis when checking for simplifications, as they may not
6978 // be lowered directly be lowered to a corresponding PHI recipe.
6979 if (isa<PHINode>(&I) && BB == TheLoop->getHeader() &&
6980 CostCtx.CM.Legal->isInductionPhi(cast<PHINode>(&I)))
6981 return false;
6982 return !SeenInstrs.contains(&I) && !CostCtx.skipCostComputation(&I, true);
6983 });
6984 });
6985}
6986#endif
6987
6989 if (VPlans.empty())
6991 // If there is a single VPlan with a single VF, return it directly.
6992 VPlan &FirstPlan = *VPlans[0];
6993 if (VPlans.size() == 1 && size(FirstPlan.vectorFactors()) == 1)
6994 return {*FirstPlan.vectorFactors().begin(), 0, 0};
6995
6996 LLVM_DEBUG(dbgs() << "LV: Computing best VF using cost kind: "
6997 << (CM.CostKind == TTI::TCK_RecipThroughput
6998 ? "Reciprocal Throughput\n"
6999 : CM.CostKind == TTI::TCK_Latency
7000 ? "Instruction Latency\n"
7001 : CM.CostKind == TTI::TCK_CodeSize ? "Code Size\n"
7002 : CM.CostKind == TTI::TCK_SizeAndLatency
7003 ? "Code Size and Latency\n"
7004 : "Unknown\n"));
7005
7007 assert(hasPlanWithVF(ScalarVF) &&
7008 "More than a single plan/VF w/o any plan having scalar VF");
7009
7010 // TODO: Compute scalar cost using VPlan-based cost model.
7011 InstructionCost ScalarCost = CM.expectedCost(ScalarVF);
7012 LLVM_DEBUG(dbgs() << "LV: Scalar loop costs: " << ScalarCost << ".\n");
7013 VectorizationFactor ScalarFactor(ScalarVF, ScalarCost, ScalarCost);
7014 VectorizationFactor BestFactor = ScalarFactor;
7015
7016 bool ForceVectorization = Hints.getForce() == LoopVectorizeHints::FK_Enabled;
7017 if (ForceVectorization) {
7018 // Ignore scalar width, because the user explicitly wants vectorization.
7019 // Initialize cost to max so that VF = 2 is, at least, chosen during cost
7020 // evaluation.
7021 BestFactor.Cost = InstructionCost::getMax();
7022 }
7023
7024 for (auto &P : VPlans) {
7025 ArrayRef<ElementCount> VFs(P->vectorFactors().begin(),
7026 P->vectorFactors().end());
7027
7029 if (any_of(VFs, [this](ElementCount VF) {
7030 return CM.shouldConsiderRegPressureForVF(VF);
7031 }))
7032 RUs = calculateRegisterUsageForPlan(*P, VFs, TTI, CM.ValuesToIgnore);
7033
7034 for (unsigned I = 0; I < VFs.size(); I++) {
7035 ElementCount VF = VFs[I];
7036 if (VF.isScalar())
7037 continue;
7038 if (!ForceVectorization && !willGenerateVectors(*P, VF, TTI)) {
7039 LLVM_DEBUG(
7040 dbgs()
7041 << "LV: Not considering vector loop of width " << VF
7042 << " because it will not generate any vector instructions.\n");
7043 continue;
7044 }
7045 if (CM.OptForSize && !ForceVectorization && hasReplicatorRegion(*P)) {
7046 LLVM_DEBUG(
7047 dbgs()
7048 << "LV: Not considering vector loop of width " << VF
7049 << " because it would cause replicated blocks to be generated,"
7050 << " which isn't allowed when optimizing for size.\n");
7051 continue;
7052 }
7053
7054 InstructionCost Cost = cost(*P, VF);
7055 VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
7056
7057 if (CM.shouldConsiderRegPressureForVF(VF) &&
7058 RUs[I].exceedsMaxNumRegs(TTI, ForceTargetNumVectorRegs)) {
7059 LLVM_DEBUG(dbgs() << "LV(REG): Not considering vector loop of width "
7060 << VF << " because it uses too many registers\n");
7061 continue;
7062 }
7063
7064 if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail()))
7065 BestFactor = CurrentFactor;
7066
7067 // If profitable add it to ProfitableVF list.
7068 if (isMoreProfitable(CurrentFactor, ScalarFactor, P->hasScalarTail()))
7069 ProfitableVFs.push_back(CurrentFactor);
7070 }
7071 }
7072
7073#ifndef NDEBUG
7074 // Select the optimal vectorization factor according to the legacy cost-model.
7075 // This is now only used to verify the decisions by the new VPlan-based
7076 // cost-model and will be retired once the VPlan-based cost-model is
7077 // stabilized.
7078 VectorizationFactor LegacyVF = selectVectorizationFactor();
7079 VPlan &BestPlan = getPlanFor(BestFactor.Width);
7080
7081 // Pre-compute the cost and use it to check if BestPlan contains any
7082 // simplifications not accounted for in the legacy cost model. If that's the
7083 // case, don't trigger the assertion, as the extra simplifications may cause a
7084 // different VF to be picked by the VPlan-based cost model.
7085 VPCostContext CostCtx(CM.TTI, *CM.TLI, BestPlan, CM, CM.CostKind);
7086 precomputeCosts(BestPlan, BestFactor.Width, CostCtx);
7087 // Verify that the VPlan-based and legacy cost models agree, except for VPlans
7088 // with early exits and plans with additional VPlan simplifications. The
7089 // legacy cost model doesn't properly model costs for such loops.
7090 assert((BestFactor.Width == LegacyVF.Width || BestPlan.hasEarlyExit() ||
7092 CostCtx, OrigLoop,
7093 BestFactor.Width) ||
7095 getPlanFor(LegacyVF.Width), CostCtx, OrigLoop, LegacyVF.Width)) &&
7096 " VPlan cost model and legacy cost model disagreed");
7097 assert((BestFactor.Width.isScalar() || BestFactor.ScalarCost > 0) &&
7098 "when vectorizing, the scalar cost must be computed.");
7099#endif
7100
7101 LLVM_DEBUG(dbgs() << "LV: Selecting VF: " << BestFactor.Width << ".\n");
7102 return BestFactor;
7103}
7104
7106 using namespace VPlanPatternMatch;
7108 "RdxResult must be ComputeFindIVResult");
7109 VPValue *StartVPV = RdxResult->getOperand(1);
7110 match(StartVPV, m_Freeze(m_VPValue(StartVPV)));
7111 return StartVPV->getLiveInIRValue();
7112}
7113
7114// If \p EpiResumePhiR is resume VPPhi for a reduction when vectorizing the
7115// epilog loop, fix the reduction's scalar PHI node by adding the incoming value
7116// from the main vector loop.
7118 VPPhi *EpiResumePhiR, PHINode &EpiResumePhi, BasicBlock *BypassBlock) {
7119 // Get the VPInstruction computing the reduction result in the middle block.
7120 // The first operand may not be from the middle block if it is not connected
7121 // to the scalar preheader. In that case, there's nothing to fix.
7122 VPValue *Incoming = EpiResumePhiR->getOperand(0);
7125 auto *EpiRedResult = dyn_cast<VPInstruction>(Incoming);
7126 if (!EpiRedResult ||
7127 (EpiRedResult->getOpcode() != VPInstruction::ComputeAnyOfResult &&
7128 EpiRedResult->getOpcode() != VPInstruction::ComputeReductionResult &&
7129 EpiRedResult->getOpcode() != VPInstruction::ComputeFindIVResult))
7130 return;
7131
7132 auto *EpiRedHeaderPhi =
7133 cast<VPReductionPHIRecipe>(EpiRedResult->getOperand(0));
7134 RecurKind Kind = EpiRedHeaderPhi->getRecurrenceKind();
7135 Value *MainResumeValue;
7136 if (auto *VPI = dyn_cast<VPInstruction>(EpiRedHeaderPhi->getStartValue())) {
7137 assert((VPI->getOpcode() == VPInstruction::Broadcast ||
7138 VPI->getOpcode() == VPInstruction::ReductionStartVector) &&
7139 "unexpected start recipe");
7140 MainResumeValue = VPI->getOperand(0)->getUnderlyingValue();
7141 } else
7142 MainResumeValue = EpiRedHeaderPhi->getStartValue()->getUnderlyingValue();
7144 [[maybe_unused]] Value *StartV =
7145 EpiRedResult->getOperand(1)->getLiveInIRValue();
7146 auto *Cmp = cast<ICmpInst>(MainResumeValue);
7147 assert(Cmp->getPredicate() == CmpInst::ICMP_NE &&
7148 "AnyOf expected to start with ICMP_NE");
7149 assert(Cmp->getOperand(1) == StartV &&
7150 "AnyOf expected to start by comparing main resume value to original "
7151 "start value");
7152 MainResumeValue = Cmp->getOperand(0);
7154 Value *StartV = getStartValueFromReductionResult(EpiRedResult);
7155 Value *SentinelV = EpiRedResult->getOperand(2)->getLiveInIRValue();
7156 using namespace llvm::PatternMatch;
7157 Value *Cmp, *OrigResumeV, *CmpOp;
7158 [[maybe_unused]] bool IsExpectedPattern =
7159 match(MainResumeValue,
7160 m_Select(m_OneUse(m_Value(Cmp)), m_Specific(SentinelV),
7161 m_Value(OrigResumeV))) &&
7163 m_Value(CmpOp))) &&
7164 ((CmpOp == StartV && isGuaranteedNotToBeUndefOrPoison(CmpOp))));
7165 assert(IsExpectedPattern && "Unexpected reduction resume pattern");
7166 MainResumeValue = OrigResumeV;
7167 }
7168 PHINode *MainResumePhi = cast<PHINode>(MainResumeValue);
7169
7170 // When fixing reductions in the epilogue loop we should already have
7171 // created a bc.merge.rdx Phi after the main vector body. Ensure that we carry
7172 // over the incoming values correctly.
7173 EpiResumePhi.setIncomingValueForBlock(
7174 BypassBlock, MainResumePhi->getIncomingValueForBlock(BypassBlock));
7175}
7176
7178 ElementCount BestVF, unsigned BestUF, VPlan &BestVPlan,
7179 InnerLoopVectorizer &ILV, DominatorTree *DT, bool VectorizingEpilogue) {
7180 assert(BestVPlan.hasVF(BestVF) &&
7181 "Trying to execute plan with unsupported VF");
7182 assert(BestVPlan.hasUF(BestUF) &&
7183 "Trying to execute plan with unsupported UF");
7184 if (BestVPlan.hasEarlyExit())
7185 ++LoopsEarlyExitVectorized;
7186 // TODO: Move to VPlan transform stage once the transition to the VPlan-based
7187 // cost model is complete for better cost estimates.
7192 bool HasBranchWeights =
7193 hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator());
7194 if (HasBranchWeights) {
7195 std::optional<unsigned> VScale = CM.getVScaleForTuning();
7197 BestVPlan, BestVF, VScale);
7198 }
7199
7200 // Checks are the same for all VPlans, added to BestVPlan only for
7201 // compactness.
7202 attachRuntimeChecks(BestVPlan, ILV.RTChecks, HasBranchWeights);
7203
7204 // Retrieving VectorPH now when it's easier while VPlan still has Regions.
7205 VPBasicBlock *VectorPH = cast<VPBasicBlock>(BestVPlan.getVectorPreheader());
7206
7207 VPlanTransforms::optimizeForVFAndUF(BestVPlan, BestVF, BestUF, PSE);
7210 if (BestVPlan.getEntry()->getSingleSuccessor() ==
7211 BestVPlan.getScalarPreheader()) {
7212 // TODO: The vector loop would be dead, should not even try to vectorize.
7213 ORE->emit([&]() {
7214 return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationDead",
7215 OrigLoop->getStartLoc(),
7216 OrigLoop->getHeader())
7217 << "Created vector loop never executes due to insufficient trip "
7218 "count.";
7219 });
7221 }
7222
7224 BestVPlan, BestVF,
7225 TTI.getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector));
7226 VPlanTransforms::cse(BestVPlan);
7228
7230 // Regions are dissolved after optimizing for VF and UF, which completely
7231 // removes unneeded loop regions first.
7233 // Canonicalize EVL loops after regions are dissolved.
7237 BestVPlan, VectorPH, CM.foldTailByMasking(),
7238 CM.requiresScalarEpilogue(BestVF.isVector()));
7239 VPlanTransforms::materializeVFAndVFxUF(BestVPlan, VectorPH, BestVF);
7241
7242 // 0. Generate SCEV-dependent code in the entry, including TripCount, before
7243 // making any changes to the CFG.
7244 DenseMap<const SCEV *, Value *> ExpandedSCEVs =
7245 VPlanTransforms::expandSCEVs(BestVPlan, *PSE.getSE());
7246 if (!ILV.getTripCount())
7247 ILV.setTripCount(BestVPlan.getTripCount()->getLiveInIRValue());
7248 else
7249 assert(VectorizingEpilogue && "should only re-use the existing trip "
7250 "count during epilogue vectorization");
7251
7252 // Perform the actual loop transformation.
7253 VPTransformState State(&TTI, BestVF, LI, DT, ILV.AC, ILV.Builder, &BestVPlan,
7254 OrigLoop->getParentLoop(),
7255 Legal->getWidestInductionType());
7256
7257#ifdef EXPENSIVE_CHECKS
7258 assert(DT->verify(DominatorTree::VerificationLevel::Fast));
7259#endif
7260
7261 // 1. Set up the skeleton for vectorization, including vector pre-header and
7262 // middle block. The vector loop is created during VPlan execution.
7263 State.CFG.PrevBB = ILV.createVectorizedLoopSkeleton();
7265 State.CFG.PrevBB->getSingleSuccessor(), &BestVPlan);
7267
7268 assert(verifyVPlanIsValid(BestVPlan, true /*VerifyLate*/) &&
7269 "final VPlan is invalid");
7270
7271 // After vectorization, the exit blocks of the original loop will have
7272 // additional predecessors. Invalidate SCEVs for the exit phis in case SE
7273 // looked through single-entry phis.
7274 ScalarEvolution &SE = *PSE.getSE();
7275 for (VPIRBasicBlock *Exit : BestVPlan.getExitBlocks()) {
7276 if (!Exit->hasPredecessors())
7277 continue;
7278 for (VPRecipeBase &PhiR : Exit->phis())
7280 OrigLoop, cast<PHINode>(&cast<VPIRPhi>(PhiR).getInstruction()));
7281 }
7282 // Forget the original loop and block dispositions.
7283 SE.forgetLoop(OrigLoop);
7285
7287
7288 //===------------------------------------------------===//
7289 //
7290 // Notice: any optimization or new instruction that go
7291 // into the code below should also be implemented in
7292 // the cost-model.
7293 //
7294 //===------------------------------------------------===//
7295
7296 // Retrieve loop information before executing the plan, which may remove the
7297 // original loop, if it becomes unreachable.
7298 MDNode *LID = OrigLoop->getLoopID();
7299 unsigned OrigLoopInvocationWeight = 0;
7300 std::optional<unsigned> OrigAverageTripCount =
7301 getLoopEstimatedTripCount(OrigLoop, &OrigLoopInvocationWeight);
7302
7303 BestVPlan.execute(&State);
7304
7305 // 2.6. Maintain Loop Hints
7306 // Keep all loop hints from the original loop on the vector loop (we'll
7307 // replace the vectorizer-specific hints below).
7308 VPBasicBlock *HeaderVPBB = vputils::getFirstLoopHeader(BestVPlan, State.VPDT);
7309 // Add metadata to disable runtime unrolling a scalar loop when there
7310 // are no runtime checks about strides and memory. A scalar loop that is
7311 // rarely used is not worth unrolling.
7312 bool DisableRuntimeUnroll = !ILV.RTChecks.hasChecks() && !BestVF.isScalar();
7314 HeaderVPBB ? LI->getLoopFor(State.CFG.VPBB2IRBB.lookup(HeaderVPBB))
7315 : nullptr,
7316 HeaderVPBB, BestVPlan, VectorizingEpilogue, LID, OrigAverageTripCount,
7317 OrigLoopInvocationWeight,
7318 estimateElementCount(BestVF * BestUF, CM.getVScaleForTuning()),
7319 DisableRuntimeUnroll);
7320
7321 // 3. Fix the vectorized code: take care of header phi's, live-outs,
7322 // predication, updating analyses.
7323 ILV.fixVectorizedLoop(State);
7324
7326
7327 return ExpandedSCEVs;
7328}
7329
7330//===--------------------------------------------------------------------===//
7331// EpilogueVectorizerMainLoop
7332//===--------------------------------------------------------------------===//
7333
7334/// This function is partially responsible for generating the control flow
7335/// depicted in https://llvm.org/docs/Vectorizers.html#epilogue-vectorization.
7337 BasicBlock *ScalarPH = createScalarPreheader("");
7338 BasicBlock *VectorPH = ScalarPH->getSinglePredecessor();
7339
7340 // Generate the code to check the minimum iteration count of the vector
7341 // epilogue (see below).
7342 EPI.EpilogueIterationCountCheck =
7343 emitIterationCountCheck(VectorPH, ScalarPH, true);
7344 EPI.EpilogueIterationCountCheck->setName("iter.check");
7345
7346 VectorPH = cast<BranchInst>(EPI.EpilogueIterationCountCheck->getTerminator())
7347 ->getSuccessor(1);
7348 // Generate the iteration count check for the main loop, *after* the check
7349 // for the epilogue loop, so that the path-length is shorter for the case
7350 // that goes directly through the vector epilogue. The longer-path length for
7351 // the main loop is compensated for, by the gain from vectorizing the larger
7352 // trip count. Note: the branch will get updated later on when we vectorize
7353 // the epilogue.
7354 EPI.MainLoopIterationCountCheck =
7355 emitIterationCountCheck(VectorPH, ScalarPH, false);
7356
7357 return cast<BranchInst>(EPI.MainLoopIterationCountCheck->getTerminator())
7358 ->getSuccessor(1);
7359}
7360
7362 LLVM_DEBUG({
7363 dbgs() << "Create Skeleton for epilogue vectorized loop (first pass)\n"
7364 << "Main Loop VF:" << EPI.MainLoopVF
7365 << ", Main Loop UF:" << EPI.MainLoopUF
7366 << ", Epilogue Loop VF:" << EPI.EpilogueVF
7367 << ", Epilogue Loop UF:" << EPI.EpilogueUF << "\n";
7368 });
7369}
7370
7373 dbgs() << "intermediate fn:\n"
7374 << *OrigLoop->getHeader()->getParent() << "\n";
7375 });
7376}
7377
7379 BasicBlock *VectorPH, BasicBlock *Bypass, bool ForEpilogue) {
7380 assert(Bypass && "Expected valid bypass basic block.");
7383 Value *CheckMinIters = createIterationCountCheck(
7384 VectorPH, ForEpilogue ? EPI.EpilogueVF : EPI.MainLoopVF,
7385 ForEpilogue ? EPI.EpilogueUF : EPI.MainLoopUF);
7386
7387 BasicBlock *const TCCheckBlock = VectorPH;
7388 if (!ForEpilogue)
7389 TCCheckBlock->setName("vector.main.loop.iter.check");
7390
7391 // Create new preheader for vector loop.
7392 VectorPH = SplitBlock(TCCheckBlock, TCCheckBlock->getTerminator(),
7393 static_cast<DominatorTree *>(nullptr), LI, nullptr,
7394 "vector.ph");
7395 if (ForEpilogue) {
7396 // Save the trip count so we don't have to regenerate it in the
7397 // vec.epilog.iter.check. This is safe to do because the trip count
7398 // generated here dominates the vector epilog iter check.
7399 EPI.TripCount = Count;
7400 } else {
7402 }
7403
7404 BranchInst &BI = *BranchInst::Create(Bypass, VectorPH, CheckMinIters);
7405 if (hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator()))
7406 setBranchWeights(BI, MinItersBypassWeights, /*IsExpected=*/false);
7407 ReplaceInstWithInst(TCCheckBlock->getTerminator(), &BI);
7408
7409 // When vectorizing the main loop, its trip-count check is placed in a new
7410 // block, whereas the overall trip-count check is placed in the VPlan entry
7411 // block. When vectorizing the epilogue loop, its trip-count check is placed
7412 // in the VPlan entry block.
7413 if (!ForEpilogue)
7414 introduceCheckBlockInVPlan(TCCheckBlock);
7415 return TCCheckBlock;
7416}
7417
7418//===--------------------------------------------------------------------===//
7419// EpilogueVectorizerEpilogueLoop
7420//===--------------------------------------------------------------------===//
7421
7422/// This function is partially responsible for generating the control flow
7423/// depicted in https://llvm.org/docs/Vectorizers.html#epilogue-vectorization.
7425 BasicBlock *ScalarPH = createScalarPreheader("vec.epilog.");
7426 BasicBlock *VectorPH = ScalarPH->getSinglePredecessor();
7427 // Now, compare the remaining count and if there aren't enough iterations to
7428 // execute the vectorized epilogue skip to the scalar part.
7429 VectorPH->setName("vec.epilog.ph");
7430 BasicBlock *VecEpilogueIterationCountCheck =
7431 SplitBlock(VectorPH, VectorPH->begin(), DT, LI, nullptr,
7432 "vec.epilog.iter.check", true);
7434
7435 emitMinimumVectorEpilogueIterCountCheck(VectorPH, ScalarPH,
7436 VecEpilogueIterationCountCheck);
7437 AdditionalBypassBlock = VecEpilogueIterationCountCheck;
7438
7439 // Adjust the control flow taking the state info from the main loop
7440 // vectorization into account.
7441 assert(EPI.MainLoopIterationCountCheck && EPI.EpilogueIterationCountCheck &&
7442 "expected this to be saved from the previous pass.");
7443 EPI.MainLoopIterationCountCheck->getTerminator()->replaceUsesOfWith(
7444 VecEpilogueIterationCountCheck, VectorPH);
7445
7446 EPI.EpilogueIterationCountCheck->getTerminator()->replaceUsesOfWith(
7447 VecEpilogueIterationCountCheck, ScalarPH);
7448
7449 // Adjust the terminators of runtime check blocks and phis using them.
7450 BasicBlock *SCEVCheckBlock = RTChecks.getSCEVChecks().second;
7451 BasicBlock *MemCheckBlock = RTChecks.getMemRuntimeChecks().second;
7452 if (SCEVCheckBlock)
7453 SCEVCheckBlock->getTerminator()->replaceUsesOfWith(
7454 VecEpilogueIterationCountCheck, ScalarPH);
7455 if (MemCheckBlock)
7456 MemCheckBlock->getTerminator()->replaceUsesOfWith(
7457 VecEpilogueIterationCountCheck, ScalarPH);
7458
7459 DT->changeImmediateDominator(ScalarPH, EPI.EpilogueIterationCountCheck);
7460
7461 // The vec.epilog.iter.check block may contain Phi nodes from inductions or
7462 // reductions which merge control-flow from the latch block and the middle
7463 // block. Update the incoming values here and move the Phi into the preheader.
7464 SmallVector<PHINode *, 4> PhisInBlock(
7465 llvm::make_pointer_range(VecEpilogueIterationCountCheck->phis()));
7466
7467 for (PHINode *Phi : PhisInBlock) {
7468 Phi->moveBefore(VectorPH->getFirstNonPHIIt());
7469 Phi->replaceIncomingBlockWith(
7470 VecEpilogueIterationCountCheck->getSinglePredecessor(),
7471 VecEpilogueIterationCountCheck);
7472
7473 // If the phi doesn't have an incoming value from the
7474 // EpilogueIterationCountCheck, we are done. Otherwise remove the incoming
7475 // value and also those from other check blocks. This is needed for
7476 // reduction phis only.
7477 if (none_of(Phi->blocks(), [&](BasicBlock *IncB) {
7478 return EPI.EpilogueIterationCountCheck == IncB;
7479 }))
7480 continue;
7481 Phi->removeIncomingValue(EPI.EpilogueIterationCountCheck);
7482 if (SCEVCheckBlock)
7483 Phi->removeIncomingValue(SCEVCheckBlock);
7484 if (MemCheckBlock)
7485 Phi->removeIncomingValue(MemCheckBlock);
7486 }
7487
7488 return VectorPH;
7489}
7490
7491BasicBlock *
7493 BasicBlock *VectorPH, BasicBlock *Bypass, BasicBlock *Insert) {
7494
7495 assert(EPI.TripCount &&
7496 "Expected trip count to have been saved in the first pass.");
7497 Value *TC = EPI.TripCount;
7498 IRBuilder<> Builder(Insert->getTerminator());
7499 Value *Count = Builder.CreateSub(TC, EPI.VectorTripCount, "n.vec.remaining");
7500
7501 // Generate code to check if the loop's trip count is less than VF * UF of the
7502 // vector epilogue loop.
7503 auto P = Cost->requiresScalarEpilogue(EPI.EpilogueVF.isVector())
7506
7507 Value *CheckMinIters =
7508 Builder.CreateICmp(P, Count,
7509 createStepForVF(Builder, Count->getType(),
7510 EPI.EpilogueVF, EPI.EpilogueUF),
7511 "min.epilog.iters.check");
7512
7513 BranchInst &BI = *BranchInst::Create(Bypass, VectorPH, CheckMinIters);
7514 auto VScale = Cost->getVScaleForTuning();
7515 unsigned MainLoopStep =
7516 estimateElementCount(EPI.MainLoopVF * EPI.MainLoopUF, VScale);
7517 unsigned EpilogueLoopStep =
7518 estimateElementCount(EPI.EpilogueVF * EPI.EpilogueUF, VScale);
7519 // We assume the remaining `Count` is equally distributed in
7520 // [0, MainLoopStep)
7521 // So the probability for `Count < EpilogueLoopStep` should be
7522 // min(MainLoopStep, EpilogueLoopStep) / MainLoopStep
7523 // TODO: Improve the estimate by taking the estimated trip count into
7524 // consideration.
7525 unsigned EstimatedSkipCount = std::min(MainLoopStep, EpilogueLoopStep);
7526 const uint32_t Weights[] = {EstimatedSkipCount,
7527 MainLoopStep - EstimatedSkipCount};
7528 setBranchWeights(BI, Weights, /*IsExpected=*/false);
7529 ReplaceInstWithInst(Insert->getTerminator(), &BI);
7530
7531 // A new entry block has been created for the epilogue VPlan. Hook it in, as
7532 // otherwise we would try to modify the entry to the main vector loop.
7533 VPIRBasicBlock *NewEntry = Plan.createVPIRBasicBlock(Insert);
7534 VPBasicBlock *OldEntry = Plan.getEntry();
7535 VPBlockUtils::reassociateBlocks(OldEntry, NewEntry);
7536 Plan.setEntry(NewEntry);
7537 // OldEntry is now dead and will be cleaned up when the plan gets destroyed.
7538
7539 return Insert;
7540}
7541
7543 LLVM_DEBUG({
7544 dbgs() << "Create Skeleton for epilogue vectorized loop (second pass)\n"
7545 << "Epilogue Loop VF:" << EPI.EpilogueVF
7546 << ", Epilogue Loop UF:" << EPI.EpilogueUF << "\n";
7547 });
7548}
7549
7552 dbgs() << "final fn:\n" << *OrigLoop->getHeader()->getParent() << "\n";
7553 });
7554}
7555
7557VPRecipeBuilder::tryToWidenMemory(Instruction *I, ArrayRef<VPValue *> Operands,
7558 VFRange &Range) {
7560 "Must be called with either a load or store");
7561
7562 auto WillWiden = [&](ElementCount VF) -> bool {
7564 CM.getWideningDecision(I, VF);
7566 "CM decision should be taken at this point.");
7568 return true;
7569 if (CM.isScalarAfterVectorization(I, VF) ||
7570 CM.isProfitableToScalarize(I, VF))
7571 return false;
7573 };
7574
7576 return nullptr;
7577
7578 VPValue *Mask = nullptr;
7579 if (Legal->isMaskRequired(I))
7580 Mask = getBlockInMask(Builder.getInsertBlock());
7581
7582 // Determine if the pointer operand of the access is either consecutive or
7583 // reverse consecutive.
7585 CM.getWideningDecision(I, Range.Start);
7587 bool Consecutive =
7589
7591 if (Consecutive) {
7593 Ptr->getUnderlyingValue()->stripPointerCasts());
7594 VPSingleDefRecipe *VectorPtr;
7595 if (Reverse) {
7596 // When folding the tail, we may compute an address that we don't in the
7597 // original scalar loop and it may not be inbounds. Drop Inbounds in that
7598 // case.
7599 GEPNoWrapFlags Flags =
7600 (CM.foldTailByMasking() || !GEP || !GEP->isInBounds())
7602 : GEPNoWrapFlags::inBounds();
7603 VectorPtr =
7605 /*Stride*/ -1, Flags, I->getDebugLoc());
7606 } else {
7607 VectorPtr = new VPVectorPointerRecipe(Ptr, getLoadStoreType(I),
7608 GEP ? GEP->getNoWrapFlags()
7610 I->getDebugLoc());
7611 }
7612 Builder.insert(VectorPtr);
7613 Ptr = VectorPtr;
7614 }
7615 if (LoadInst *Load = dyn_cast<LoadInst>(I))
7616 return new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, Reverse,
7617 VPIRMetadata(*Load, LVer), I->getDebugLoc());
7618
7619 StoreInst *Store = cast<StoreInst>(I);
7620 return new VPWidenStoreRecipe(*Store, Ptr, Operands[0], Mask, Consecutive,
7621 Reverse, VPIRMetadata(*Store, LVer),
7622 I->getDebugLoc());
7623}
7624
7625/// Creates a VPWidenIntOrFpInductionRecpipe for \p Phi. If needed, it will also
7626/// insert a recipe to expand the step for the induction recipe.
7629 VPValue *Start, const InductionDescriptor &IndDesc,
7630 VPlan &Plan, ScalarEvolution &SE, Loop &OrigLoop) {
7631 assert(IndDesc.getStartValue() ==
7632 Phi->getIncomingValueForBlock(OrigLoop.getLoopPreheader()));
7633 assert(SE.isLoopInvariant(IndDesc.getStep(), &OrigLoop) &&
7634 "step must be loop invariant");
7635
7636 VPValue *Step =
7638 if (auto *TruncI = dyn_cast<TruncInst>(PhiOrTrunc)) {
7639 return new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, &Plan.getVF(),
7640 IndDesc, TruncI,
7641 TruncI->getDebugLoc());
7642 }
7643 assert(isa<PHINode>(PhiOrTrunc) && "must be a phi node here");
7644 return new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, &Plan.getVF(),
7645 IndDesc, Phi->getDebugLoc());
7646}
7647
7648VPHeaderPHIRecipe *VPRecipeBuilder::tryToOptimizeInductionPHI(
7650
7651 // Check if this is an integer or fp induction. If so, build the recipe that
7652 // produces its scalar and vector values.
7653 if (auto *II = Legal->getIntOrFpInductionDescriptor(Phi))
7654 return createWidenInductionRecipes(Phi, Phi, Operands[0], *II, Plan,
7655 *PSE.getSE(), *OrigLoop);
7656
7657 // Check if this is pointer induction. If so, build the recipe for it.
7658 if (auto *II = Legal->getPointerInductionDescriptor(Phi)) {
7659 VPValue *Step = vputils::getOrCreateVPValueForSCEVExpr(Plan, II->getStep());
7660 return new VPWidenPointerInductionRecipe(
7661 Phi, Operands[0], Step, &Plan.getVFxUF(), *II,
7663 [&](ElementCount VF) {
7664 return CM.isScalarAfterVectorization(Phi, VF);
7665 },
7666 Range),
7667 Phi->getDebugLoc());
7668 }
7669 return nullptr;
7670}
7671
7672VPWidenIntOrFpInductionRecipe *VPRecipeBuilder::tryToOptimizeInductionTruncate(
7674 // Optimize the special case where the source is a constant integer
7675 // induction variable. Notice that we can only optimize the 'trunc' case
7676 // because (a) FP conversions lose precision, (b) sext/zext may wrap, and
7677 // (c) other casts depend on pointer size.
7678
7679 // Determine whether \p K is a truncation based on an induction variable that
7680 // can be optimized.
7681 auto IsOptimizableIVTruncate =
7682 [&](Instruction *K) -> std::function<bool(ElementCount)> {
7683 return [=](ElementCount VF) -> bool {
7684 return CM.isOptimizableIVTruncate(K, VF);
7685 };
7686 };
7687
7689 IsOptimizableIVTruncate(I), Range)) {
7690
7691 auto *Phi = cast<PHINode>(I->getOperand(0));
7692 const InductionDescriptor &II = *Legal->getIntOrFpInductionDescriptor(Phi);
7693 VPValue *Start = Plan.getOrAddLiveIn(II.getStartValue());
7694 return createWidenInductionRecipes(Phi, I, Start, II, Plan, *PSE.getSE(),
7695 *OrigLoop);
7696 }
7697 return nullptr;
7698}
7699
7700VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
7702 VFRange &Range) {
7704 [this, CI](ElementCount VF) {
7705 return CM.isScalarWithPredication(CI, VF);
7706 },
7707 Range);
7708
7709 if (IsPredicated)
7710 return nullptr;
7711
7713 if (ID && (ID == Intrinsic::assume || ID == Intrinsic::lifetime_end ||
7714 ID == Intrinsic::lifetime_start || ID == Intrinsic::sideeffect ||
7715 ID == Intrinsic::pseudoprobe ||
7716 ID == Intrinsic::experimental_noalias_scope_decl))
7717 return nullptr;
7718
7720
7721 // Is it beneficial to perform intrinsic call compared to lib call?
7722 bool ShouldUseVectorIntrinsic =
7724 [&](ElementCount VF) -> bool {
7725 return CM.getCallWideningDecision(CI, VF).Kind ==
7727 },
7728 Range);
7729 if (ShouldUseVectorIntrinsic)
7730 return new VPWidenIntrinsicRecipe(*CI, ID, Ops, CI->getType(),
7731 CI->getDebugLoc());
7732
7733 Function *Variant = nullptr;
7734 std::optional<unsigned> MaskPos;
7735 // Is better to call a vectorized version of the function than to to scalarize
7736 // the call?
7737 auto ShouldUseVectorCall = LoopVectorizationPlanner::getDecisionAndClampRange(
7738 [&](ElementCount VF) -> bool {
7739 // The following case may be scalarized depending on the VF.
7740 // The flag shows whether we can use a usual Call for vectorized
7741 // version of the instruction.
7742
7743 // If we've found a variant at a previous VF, then stop looking. A
7744 // vectorized variant of a function expects input in a certain shape
7745 // -- basically the number of input registers, the number of lanes
7746 // per register, and whether there's a mask required.
7747 // We store a pointer to the variant in the VPWidenCallRecipe, so
7748 // once we have an appropriate variant it's only valid for that VF.
7749 // This will force a different vplan to be generated for each VF that
7750 // finds a valid variant.
7751 if (Variant)
7752 return false;
7753 LoopVectorizationCostModel::CallWideningDecision Decision =
7754 CM.getCallWideningDecision(CI, VF);
7756 Variant = Decision.Variant;
7757 MaskPos = Decision.MaskPos;
7758 return true;
7759 }
7760
7761 return false;
7762 },
7763 Range);
7764 if (ShouldUseVectorCall) {
7765 if (MaskPos.has_value()) {
7766 // We have 2 cases that would require a mask:
7767 // 1) The block needs to be predicated, either due to a conditional
7768 // in the scalar loop or use of an active lane mask with
7769 // tail-folding, and we use the appropriate mask for the block.
7770 // 2) No mask is required for the block, but the only available
7771 // vector variant at this VF requires a mask, so we synthesize an
7772 // all-true mask.
7773 VPValue *Mask = nullptr;
7774 if (Legal->isMaskRequired(CI))
7775 Mask = getBlockInMask(Builder.getInsertBlock());
7776 else
7777 Mask = Plan.getOrAddLiveIn(
7779
7780 Ops.insert(Ops.begin() + *MaskPos, Mask);
7781 }
7782
7783 Ops.push_back(Operands.back());
7784 return new VPWidenCallRecipe(CI, Variant, Ops, CI->getDebugLoc());
7785 }
7786
7787 return nullptr;
7788}
7789
7790bool VPRecipeBuilder::shouldWiden(Instruction *I, VFRange &Range) const {
7792 !isa<StoreInst>(I) && "Instruction should have been handled earlier");
7793 // Instruction should be widened, unless it is scalar after vectorization,
7794 // scalarization is profitable or it is predicated.
7795 auto WillScalarize = [this, I](ElementCount VF) -> bool {
7796 return CM.isScalarAfterVectorization(I, VF) ||
7797 CM.isProfitableToScalarize(I, VF) ||
7798 CM.isScalarWithPredication(I, VF);
7799 };
7801 Range);
7802}
7803
7804VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
7806 switch (I->getOpcode()) {
7807 default:
7808 return nullptr;
7809 case Instruction::SDiv:
7810 case Instruction::UDiv:
7811 case Instruction::SRem:
7812 case Instruction::URem: {
7813 // If not provably safe, use a select to form a safe divisor before widening the
7814 // div/rem operation itself. Otherwise fall through to general handling below.
7815 if (CM.isPredicatedInst(I)) {
7817 VPValue *Mask = getBlockInMask(Builder.getInsertBlock());
7818 VPValue *One =
7819 Plan.getOrAddLiveIn(ConstantInt::get(I->getType(), 1u, false));
7820 auto *SafeRHS = Builder.createSelect(Mask, Ops[1], One, I->getDebugLoc());
7821 Ops[1] = SafeRHS;
7822 return new VPWidenRecipe(*I, Ops);
7823 }
7824 [[fallthrough]];
7825 }
7826 case Instruction::Add:
7827 case Instruction::And:
7828 case Instruction::AShr:
7829 case Instruction::FAdd:
7830 case Instruction::FCmp:
7831 case Instruction::FDiv:
7832 case Instruction::FMul:
7833 case Instruction::FNeg:
7834 case Instruction::FRem:
7835 case Instruction::FSub:
7836 case Instruction::ICmp:
7837 case Instruction::LShr:
7838 case Instruction::Mul:
7839 case Instruction::Or:
7840 case Instruction::Select:
7841 case Instruction::Shl:
7842 case Instruction::Sub:
7843 case Instruction::Xor:
7844 case Instruction::Freeze: {
7846 if (Instruction::isBinaryOp(I->getOpcode())) {
7847 // The legacy cost model uses SCEV to check if some of the operands are
7848 // constants. To match the legacy cost model's behavior, use SCEV to try
7849 // to replace operands with constants.
7850 ScalarEvolution &SE = *PSE.getSE();
7851 auto GetConstantViaSCEV = [this, &SE](VPValue *Op) {
7852 if (!Op->isLiveIn())
7853 return Op;
7854 Value *V = Op->getUnderlyingValue();
7855 if (isa<Constant>(V) || !SE.isSCEVable(V->getType()))
7856 return Op;
7857 auto *C = dyn_cast<SCEVConstant>(SE.getSCEV(V));
7858 if (!C)
7859 return Op;
7860 return Plan.getOrAddLiveIn(C->getValue());
7861 };
7862 // For Mul, the legacy cost model checks both operands.
7863 if (I->getOpcode() == Instruction::Mul)
7864 NewOps[0] = GetConstantViaSCEV(NewOps[0]);
7865 // For other binops, the legacy cost model only checks the second operand.
7866 NewOps[1] = GetConstantViaSCEV(NewOps[1]);
7867 }
7868 return new VPWidenRecipe(*I, NewOps);
7869 }
7870 case Instruction::ExtractValue: {
7872 Type *I32Ty = IntegerType::getInt32Ty(I->getContext());
7873 auto *EVI = cast<ExtractValueInst>(I);
7874 assert(EVI->getNumIndices() == 1 && "Expected one extractvalue index");
7875 unsigned Idx = EVI->getIndices()[0];
7876 NewOps.push_back(Plan.getOrAddLiveIn(ConstantInt::get(I32Ty, Idx, false)));
7877 return new VPWidenRecipe(*I, NewOps);
7878 }
7879 };
7880}
7881
7883VPRecipeBuilder::tryToWidenHistogram(const HistogramInfo *HI,
7885 // FIXME: Support other operations.
7886 unsigned Opcode = HI->Update->getOpcode();
7887 assert((Opcode == Instruction::Add || Opcode == Instruction::Sub) &&
7888 "Histogram update operation must be an Add or Sub");
7889
7891 // Bucket address.
7892 HGramOps.push_back(Operands[1]);
7893 // Increment value.
7894 HGramOps.push_back(getVPValueOrAddLiveIn(HI->Update->getOperand(1)));
7895
7896 // In case of predicated execution (due to tail-folding, or conditional
7897 // execution, or both), pass the relevant mask.
7898 if (Legal->isMaskRequired(HI->Store))
7899 HGramOps.push_back(getBlockInMask(Builder.getInsertBlock()));
7900
7901 return new VPHistogramRecipe(Opcode, HGramOps, HI->Store->getDebugLoc());
7902}
7903
7906 VFRange &Range) {
7908 [&](ElementCount VF) { return CM.isUniformAfterVectorization(I, VF); },
7909 Range);
7910
7911 bool IsPredicated = CM.isPredicatedInst(I);
7912
7913 // Even if the instruction is not marked as uniform, there are certain
7914 // intrinsic calls that can be effectively treated as such, so we check for
7915 // them here. Conservatively, we only do this for scalable vectors, since
7916 // for fixed-width VFs we can always fall back on full scalarization.
7917 if (!IsUniform && Range.Start.isScalable() && isa<IntrinsicInst>(I)) {
7918 switch (cast<IntrinsicInst>(I)->getIntrinsicID()) {
7919 case Intrinsic::assume:
7920 case Intrinsic::lifetime_start:
7921 case Intrinsic::lifetime_end:
7922 // For scalable vectors if one of the operands is variant then we still
7923 // want to mark as uniform, which will generate one instruction for just
7924 // the first lane of the vector. We can't scalarize the call in the same
7925 // way as for fixed-width vectors because we don't know how many lanes
7926 // there are.
7927 //
7928 // The reasons for doing it this way for scalable vectors are:
7929 // 1. For the assume intrinsic generating the instruction for the first
7930 // lane is still be better than not generating any at all. For
7931 // example, the input may be a splat across all lanes.
7932 // 2. For the lifetime start/end intrinsics the pointer operand only
7933 // does anything useful when the input comes from a stack object,
7934 // which suggests it should always be uniform. For non-stack objects
7935 // the effect is to poison the object, which still allows us to
7936 // remove the call.
7937 IsUniform = true;
7938 break;
7939 default:
7940 break;
7941 }
7942 }
7943 VPValue *BlockInMask = nullptr;
7944 if (!IsPredicated) {
7945 // Finalize the recipe for Instr, first if it is not predicated.
7946 LLVM_DEBUG(dbgs() << "LV: Scalarizing:" << *I << "\n");
7947 } else {
7948 LLVM_DEBUG(dbgs() << "LV: Scalarizing and predicating:" << *I << "\n");
7949 // Instructions marked for predication are replicated and a mask operand is
7950 // added initially. Masked replicate recipes will later be placed under an
7951 // if-then construct to prevent side-effects. Generate recipes to compute
7952 // the block mask for this region.
7953 BlockInMask = getBlockInMask(Builder.getInsertBlock());
7954 }
7955
7956 // Note that there is some custom logic to mark some intrinsics as uniform
7957 // manually above for scalable vectors, which this assert needs to account for
7958 // as well.
7959 assert((Range.Start.isScalar() || !IsUniform || !IsPredicated ||
7960 (Range.Start.isScalable() && isa<IntrinsicInst>(I))) &&
7961 "Should not predicate a uniform recipe");
7962 auto *Recipe = new VPReplicateRecipe(I, Operands, IsUniform, BlockInMask,
7963 VPIRMetadata(*I, LVer));
7964 return Recipe;
7965}
7966
7967/// Find all possible partial reductions in the loop and track all of those that
7968/// are valid so recipes can be formed later.
7970 // Find all possible partial reductions.
7972 PartialReductionChains;
7973 for (const auto &[Phi, RdxDesc] : Legal->getReductionVars()) {
7974 getScaledReductions(Phi, RdxDesc.getLoopExitInstr(), Range,
7975 PartialReductionChains);
7976 }
7977
7978 // A partial reduction is invalid if any of its extends are used by
7979 // something that isn't another partial reduction. This is because the
7980 // extends are intended to be lowered along with the reduction itself.
7981
7982 // Build up a set of partial reduction ops for efficient use checking.
7983 SmallPtrSet<User *, 4> PartialReductionOps;
7984 for (const auto &[PartialRdx, _] : PartialReductionChains)
7985 PartialReductionOps.insert(PartialRdx.ExtendUser);
7986
7987 auto ExtendIsOnlyUsedByPartialReductions =
7988 [&PartialReductionOps](Instruction *Extend) {
7989 return all_of(Extend->users(), [&](const User *U) {
7990 return PartialReductionOps.contains(U);
7991 });
7992 };
7993
7994 // Check if each use of a chain's two extends is a partial reduction
7995 // and only add those that don't have non-partial reduction users.
7996 for (auto Pair : PartialReductionChains) {
7997 PartialReductionChain Chain = Pair.first;
7998 if (ExtendIsOnlyUsedByPartialReductions(Chain.ExtendA) &&
7999 (!Chain.ExtendB || ExtendIsOnlyUsedByPartialReductions(Chain.ExtendB)))
8000 ScaledReductionMap.try_emplace(Chain.Reduction, Pair.second);
8001 }
8002}
8003
8004bool VPRecipeBuilder::getScaledReductions(
8005 Instruction *PHI, Instruction *RdxExitInstr, VFRange &Range,
8006 SmallVectorImpl<std::pair<PartialReductionChain, unsigned>> &Chains) {
8007 if (!CM.TheLoop->contains(RdxExitInstr))
8008 return false;
8009
8010 auto *Update = dyn_cast<BinaryOperator>(RdxExitInstr);
8011 if (!Update)
8012 return false;
8013
8014 Value *Op = Update->getOperand(0);
8015 Value *PhiOp = Update->getOperand(1);
8016 if (Op == PHI)
8017 std::swap(Op, PhiOp);
8018
8019 // Try and get a scaled reduction from the first non-phi operand.
8020 // If one is found, we use the discovered reduction instruction in
8021 // place of the accumulator for costing.
8022 if (auto *OpInst = dyn_cast<Instruction>(Op)) {
8023 if (getScaledReductions(PHI, OpInst, Range, Chains)) {
8024 PHI = Chains.rbegin()->first.Reduction;
8025
8026 Op = Update->getOperand(0);
8027 PhiOp = Update->getOperand(1);
8028 if (Op == PHI)
8029 std::swap(Op, PhiOp);
8030 }
8031 }
8032 if (PhiOp != PHI)
8033 return false;
8034
8035 using namespace llvm::PatternMatch;
8036
8037 // If the update is a binary operator, check both of its operands to see if
8038 // they are extends. Otherwise, see if the update comes directly from an
8039 // extend.
8040 Instruction *Exts[2] = {nullptr};
8041 BinaryOperator *ExtendUser = dyn_cast<BinaryOperator>(Op);
8042 std::optional<unsigned> BinOpc;
8043 Type *ExtOpTypes[2] = {nullptr};
8044
8045 auto CollectExtInfo = [this, &Exts,
8046 &ExtOpTypes](SmallVectorImpl<Value *> &Ops) -> bool {
8047 unsigned I = 0;
8048 for (Value *OpI : Ops) {
8049 Value *ExtOp;
8050 if (!match(OpI, m_ZExtOrSExt(m_Value(ExtOp))))
8051 return false;
8052 Exts[I] = cast<Instruction>(OpI);
8053
8054 // TODO: We should be able to support live-ins.
8055 if (!CM.TheLoop->contains(Exts[I]))
8056 return false;
8057
8058 ExtOpTypes[I] = ExtOp->getType();
8059 I++;
8060 }
8061 return true;
8062 };
8063
8064 if (ExtendUser) {
8065 if (!ExtendUser->hasOneUse())
8066 return false;
8067
8068 // Use the side-effect of match to replace BinOp only if the pattern is
8069 // matched, we don't care at this point whether it actually matched.
8070 match(ExtendUser, m_Neg(m_BinOp(ExtendUser)));
8071
8072 SmallVector<Value *> Ops(ExtendUser->operands());
8073 if (!CollectExtInfo(Ops))
8074 return false;
8075
8076 BinOpc = std::make_optional(ExtendUser->getOpcode());
8077 } else if (match(Update, m_Add(m_Value(), m_Value()))) {
8078 // We already know the operands for Update are Op and PhiOp.
8080 if (!CollectExtInfo(Ops))
8081 return false;
8082
8083 ExtendUser = Update;
8084 BinOpc = std::nullopt;
8085 } else
8086 return false;
8087
8091 Exts[1] ? TTI::getPartialReductionExtendKind(Exts[1]) : TTI::PR_None;
8092 PartialReductionChain Chain(RdxExitInstr, Exts[0], Exts[1], ExtendUser);
8093
8094 TypeSize PHISize = PHI->getType()->getPrimitiveSizeInBits();
8095 TypeSize ASize = ExtOpTypes[0]->getPrimitiveSizeInBits();
8096 if (!PHISize.hasKnownScalarFactor(ASize))
8097 return false;
8098 unsigned TargetScaleFactor = PHISize.getKnownScalarFactor(ASize);
8099
8101 [&](ElementCount VF) {
8102 InstructionCost Cost = TTI->getPartialReductionCost(
8103 Update->getOpcode(), ExtOpTypes[0], ExtOpTypes[1],
8104 PHI->getType(), VF, OpAExtend, OpBExtend, BinOpc, CM.CostKind);
8105 return Cost.isValid();
8106 },
8107 Range)) {
8108 Chains.emplace_back(Chain, TargetScaleFactor);
8109 return true;
8110 }
8111
8112 return false;
8113}
8114
8116 VFRange &Range) {
8117 // First, check for specific widening recipes that deal with inductions, Phi
8118 // nodes, calls and memory operations.
8119 VPRecipeBase *Recipe;
8120 Instruction *Instr = R->getUnderlyingInstr();
8121 SmallVector<VPValue *, 4> Operands(R->operands());
8122 if (auto *PhiR = dyn_cast<VPPhi>(R)) {
8123 VPBasicBlock *Parent = PhiR->getParent();
8124 [[maybe_unused]] VPRegionBlock *LoopRegionOf =
8125 Parent->getEnclosingLoopRegion();
8126 assert(LoopRegionOf && LoopRegionOf->getEntry() == Parent &&
8127 "Non-header phis should have been handled during predication");
8128 auto *Phi = cast<PHINode>(R->getUnderlyingInstr());
8129 assert(Operands.size() == 2 && "Must have 2 operands for header phis");
8130 if ((Recipe = tryToOptimizeInductionPHI(Phi, Operands, Range)))
8131 return Recipe;
8132
8133 VPHeaderPHIRecipe *PhiRecipe = nullptr;
8134 assert((Legal->isReductionVariable(Phi) ||
8135 Legal->isFixedOrderRecurrence(Phi)) &&
8136 "can only widen reductions and fixed-order recurrences here");
8137 VPValue *StartV = Operands[0];
8138 if (Legal->isReductionVariable(Phi)) {
8139 const RecurrenceDescriptor &RdxDesc = Legal->getRecurrenceDescriptor(Phi);
8140 assert(RdxDesc.getRecurrenceStartValue() ==
8141 Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader()));
8142
8143 // If the PHI is used by a partial reduction, set the scale factor.
8144 unsigned ScaleFactor =
8145 getScalingForReduction(RdxDesc.getLoopExitInstr()).value_or(1);
8146 PhiRecipe = new VPReductionPHIRecipe(
8147 Phi, RdxDesc.getRecurrenceKind(), *StartV, CM.isInLoopReduction(Phi),
8148 CM.useOrderedReductions(RdxDesc), ScaleFactor);
8149 } else {
8150 // TODO: Currently fixed-order recurrences are modeled as chains of
8151 // first-order recurrences. If there are no users of the intermediate
8152 // recurrences in the chain, the fixed order recurrence should be modeled
8153 // directly, enabling more efficient codegen.
8154 PhiRecipe = new VPFirstOrderRecurrencePHIRecipe(Phi, *StartV);
8155 }
8156 // Add backedge value.
8157 PhiRecipe->addOperand(Operands[1]);
8158 return PhiRecipe;
8159 }
8160 assert(!R->isPhi() && "only VPPhi nodes expected at this point");
8161
8162 if (isa<TruncInst>(Instr) && (Recipe = tryToOptimizeInductionTruncate(
8163 cast<TruncInst>(Instr), Operands, Range)))
8164 return Recipe;
8165
8166 // All widen recipes below deal only with VF > 1.
8168 [&](ElementCount VF) { return VF.isScalar(); }, Range))
8169 return nullptr;
8170
8171 if (auto *CI = dyn_cast<CallInst>(Instr))
8172 return tryToWidenCall(CI, Operands, Range);
8173
8174 if (StoreInst *SI = dyn_cast<StoreInst>(Instr))
8175 if (auto HistInfo = Legal->getHistogramInfo(SI))
8176 return tryToWidenHistogram(*HistInfo, Operands);
8177
8178 if (isa<LoadInst>(Instr) || isa<StoreInst>(Instr))
8179 return tryToWidenMemory(Instr, Operands, Range);
8180
8181 if (std::optional<unsigned> ScaleFactor = getScalingForReduction(Instr))
8182 return tryToCreatePartialReduction(Instr, Operands, ScaleFactor.value());
8183
8184 if (!shouldWiden(Instr, Range))
8185 return nullptr;
8186
8187 if (auto *GEP = dyn_cast<GetElementPtrInst>(Instr))
8188 return new VPWidenGEPRecipe(GEP, Operands);
8189
8190 if (auto *SI = dyn_cast<SelectInst>(Instr)) {
8191 return new VPWidenSelectRecipe(*SI, Operands);
8192 }
8193
8194 if (auto *CI = dyn_cast<CastInst>(Instr)) {
8195 return new VPWidenCastRecipe(CI->getOpcode(), Operands[0], CI->getType(),
8196 *CI);
8197 }
8198
8199 return tryToWiden(Instr, Operands);
8200}
8201
8205 unsigned ScaleFactor) {
8206 assert(Operands.size() == 2 &&
8207 "Unexpected number of operands for partial reduction");
8208
8209 VPValue *BinOp = Operands[0];
8211 VPRecipeBase *BinOpRecipe = BinOp->getDefiningRecipe();
8212 if (isa<VPReductionPHIRecipe>(BinOpRecipe) ||
8213 isa<VPPartialReductionRecipe>(BinOpRecipe))
8214 std::swap(BinOp, Accumulator);
8215
8216 unsigned ReductionOpcode = Reduction->getOpcode();
8217 if (ReductionOpcode == Instruction::Sub) {
8218 auto *const Zero = ConstantInt::get(Reduction->getType(), 0);
8220 Ops.push_back(Plan.getOrAddLiveIn(Zero));
8221 Ops.push_back(BinOp);
8222 BinOp = new VPWidenRecipe(*Reduction, Ops);
8223 Builder.insert(BinOp->getDefiningRecipe());
8224 ReductionOpcode = Instruction::Add;
8225 }
8226
8227 VPValue *Cond = nullptr;
8228 if (CM.blockNeedsPredicationForAnyReason(Reduction->getParent())) {
8229 assert((ReductionOpcode == Instruction::Add ||
8230 ReductionOpcode == Instruction::Sub) &&
8231 "Expected an ADD or SUB operation for predicated partial "
8232 "reductions (because the neutral element in the mask is zero)!");
8233 Cond = getBlockInMask(Builder.getInsertBlock());
8234 VPValue *Zero =
8235 Plan.getOrAddLiveIn(ConstantInt::get(Reduction->getType(), 0));
8236 BinOp = Builder.createSelect(Cond, BinOp, Zero, Reduction->getDebugLoc());
8237 }
8238 return new VPPartialReductionRecipe(ReductionOpcode, Accumulator, BinOp, Cond,
8239 ScaleFactor, Reduction);
8240}
8241
8242void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
8243 ElementCount MaxVF) {
8244 if (ElementCount::isKnownGT(MinVF, MaxVF))
8245 return;
8246
8247 assert(OrigLoop->isInnermost() && "Inner loop expected.");
8248
8249 const LoopAccessInfo *LAI = Legal->getLAI();
8251 OrigLoop, LI, DT, PSE.getSE());
8252 if (!LAI->getRuntimePointerChecking()->getChecks().empty() &&
8254 // Only use noalias metadata when using memory checks guaranteeing no
8255 // overlap across all iterations.
8256 LVer.prepareNoAliasMetadata();
8257 }
8258
8259 // Create initial base VPlan0, to serve as common starting point for all
8260 // candidates built later for specific VF ranges.
8261 auto VPlan0 = VPlanTransforms::buildVPlan0(
8262 OrigLoop, *LI, Legal->getWidestInductionType(),
8263 getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()), PSE);
8264
8265 auto MaxVFTimes2 = MaxVF * 2;
8266 for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) {
8267 VFRange SubRange = {VF, MaxVFTimes2};
8268 if (auto Plan = tryToBuildVPlanWithVPRecipes(
8269 std::unique_ptr<VPlan>(VPlan0->duplicate()), SubRange, &LVer)) {
8270 bool HasScalarVF = Plan->hasScalarVFOnly();
8271 // Now optimize the initial VPlan.
8272 if (!HasScalarVF)
8274 *Plan, CM.getMinimalBitwidths());
8276 // TODO: try to put it close to addActiveLaneMask().
8277 if (CM.foldTailWithEVL() && !HasScalarVF)
8279 *Plan, CM.getMaxSafeElements());
8280 assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
8281 VPlans.push_back(std::move(Plan));
8282 }
8283 VF = SubRange.End;
8284 }
8285}
8286
8287/// Create and return a ResumePhi for \p WideIV, unless it is truncated. If the
8288/// induction recipe is not canonical, creates a VPDerivedIVRecipe to compute
8289/// the end value of the induction.
8291 VPWidenInductionRecipe *WideIV, VPBuilder &VectorPHBuilder,
8292 VPBuilder &ScalarPHBuilder, VPTypeAnalysis &TypeInfo, VPValue *VectorTC) {
8293 auto *WideIntOrFp = dyn_cast<VPWidenIntOrFpInductionRecipe>(WideIV);
8294 // Truncated wide inductions resume from the last lane of their vector value
8295 // in the last vector iteration which is handled elsewhere.
8296 if (WideIntOrFp && WideIntOrFp->getTruncInst())
8297 return nullptr;
8298
8299 VPValue *Start = WideIV->getStartValue();
8300 VPValue *Step = WideIV->getStepValue();
8302 VPValue *EndValue = VectorTC;
8303 if (!WideIntOrFp || !WideIntOrFp->isCanonical()) {
8304 EndValue = VectorPHBuilder.createDerivedIV(
8305 ID.getKind(), dyn_cast_or_null<FPMathOperator>(ID.getInductionBinOp()),
8306 Start, VectorTC, Step);
8307 }
8308
8309 // EndValue is derived from the vector trip count (which has the same type as
8310 // the widest induction) and thus may be wider than the induction here.
8311 Type *ScalarTypeOfWideIV = TypeInfo.inferScalarType(WideIV);
8312 if (ScalarTypeOfWideIV != TypeInfo.inferScalarType(EndValue)) {
8313 EndValue = VectorPHBuilder.createScalarCast(Instruction::Trunc, EndValue,
8314 ScalarTypeOfWideIV,
8315 WideIV->getDebugLoc());
8316 }
8317
8318 auto *ResumePhiRecipe = ScalarPHBuilder.createScalarPhi(
8319 {EndValue, Start}, WideIV->getDebugLoc(), "bc.resume.val");
8320 return ResumePhiRecipe;
8321}
8322
8323/// Create resume phis in the scalar preheader for first-order recurrences,
8324/// reductions and inductions, and update the VPIRInstructions wrapping the
8325/// original phis in the scalar header. End values for inductions are added to
8326/// \p IVEndValues.
8327static void addScalarResumePhis(VPRecipeBuilder &Builder, VPlan &Plan,
8328 DenseMap<VPValue *, VPValue *> &IVEndValues) {
8329 VPTypeAnalysis TypeInfo(Plan);
8330 auto *ScalarPH = Plan.getScalarPreheader();
8331 auto *MiddleVPBB = cast<VPBasicBlock>(ScalarPH->getPredecessors()[0]);
8332 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
8333 VPBuilder VectorPHBuilder(
8334 cast<VPBasicBlock>(VectorRegion->getSinglePredecessor()));
8335 VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
8336 VPBuilder ScalarPHBuilder(ScalarPH);
8337 for (VPRecipeBase &ScalarPhiR : Plan.getScalarHeader()->phis()) {
8338 auto *ScalarPhiIRI = cast<VPIRPhi>(&ScalarPhiR);
8339
8340 // TODO: Extract final value from induction recipe initially, optimize to
8341 // pre-computed end value together in optimizeInductionExitUsers.
8342 auto *VectorPhiR =
8343 cast<VPHeaderPHIRecipe>(Builder.getRecipe(&ScalarPhiIRI->getIRPhi()));
8344 if (auto *WideIVR = dyn_cast<VPWidenInductionRecipe>(VectorPhiR)) {
8346 WideIVR, VectorPHBuilder, ScalarPHBuilder, TypeInfo,
8347 &Plan.getVectorTripCount())) {
8348 assert(isa<VPPhi>(ResumePhi) && "Expected a phi");
8349 IVEndValues[WideIVR] = ResumePhi->getOperand(0);
8350 ScalarPhiIRI->addOperand(ResumePhi);
8351 continue;
8352 }
8353 // TODO: Also handle truncated inductions here. Computing end-values
8354 // separately should be done as VPlan-to-VPlan optimization, after
8355 // legalizing all resume values to use the last lane from the loop.
8356 assert(cast<VPWidenIntOrFpInductionRecipe>(VectorPhiR)->getTruncInst() &&
8357 "should only skip truncated wide inductions");
8358 continue;
8359 }
8360
8361 // The backedge value provides the value to resume coming out of a loop,
8362 // which for FORs is a vector whose last element needs to be extracted. The
8363 // start value provides the value if the loop is bypassed.
8364 bool IsFOR = isa<VPFirstOrderRecurrencePHIRecipe>(VectorPhiR);
8365 auto *ResumeFromVectorLoop = VectorPhiR->getBackedgeValue();
8366 assert(VectorRegion->getSingleSuccessor() == Plan.getMiddleBlock() &&
8367 "Cannot handle loops with uncountable early exits");
8368 if (IsFOR)
8369 ResumeFromVectorLoop = MiddleBuilder.createNaryOp(
8370 VPInstruction::ExtractLastElement, {ResumeFromVectorLoop}, {},
8371 "vector.recur.extract");
8372 StringRef Name = IsFOR ? "scalar.recur.init" : "bc.merge.rdx";
8373 auto *ResumePhiR = ScalarPHBuilder.createScalarPhi(
8374 {ResumeFromVectorLoop, VectorPhiR->getStartValue()}, {}, Name);
8375 ScalarPhiIRI->addOperand(ResumePhiR);
8376 }
8377}
8378
8379/// Handle users in the exit block for first order reductions in the original
8380/// exit block. The penultimate value of recurrences is fed to their LCSSA phi
8381/// users in the original exit block using the VPIRInstruction wrapping to the
8382/// LCSSA phi.
8384 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
8385 auto *ScalarPHVPBB = Plan.getScalarPreheader();
8386 auto *MiddleVPBB = Plan.getMiddleBlock();
8387 VPBuilder ScalarPHBuilder(ScalarPHVPBB);
8388 VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
8389
8390 auto IsScalableOne = [](ElementCount VF) -> bool {
8391 return VF == ElementCount::getScalable(1);
8392 };
8393
8394 for (auto &HeaderPhi : VectorRegion->getEntryBasicBlock()->phis()) {
8395 auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(&HeaderPhi);
8396 if (!FOR)
8397 continue;
8398
8399 assert(VectorRegion->getSingleSuccessor() == Plan.getMiddleBlock() &&
8400 "Cannot handle loops with uncountable early exits");
8401
8402 // This is the second phase of vectorizing first-order recurrences, creating
8403 // extract for users outside the loop. An overview of the transformation is
8404 // described below. Suppose we have the following loop with some use after
8405 // the loop of the last a[i-1],
8406 //
8407 // for (int i = 0; i < n; ++i) {
8408 // t = a[i - 1];
8409 // b[i] = a[i] - t;
8410 // }
8411 // use t;
8412 //
8413 // There is a first-order recurrence on "a". For this loop, the shorthand
8414 // scalar IR looks like:
8415 //
8416 // scalar.ph:
8417 // s.init = a[-1]
8418 // br scalar.body
8419 //
8420 // scalar.body:
8421 // i = phi [0, scalar.ph], [i+1, scalar.body]
8422 // s1 = phi [s.init, scalar.ph], [s2, scalar.body]
8423 // s2 = a[i]
8424 // b[i] = s2 - s1
8425 // br cond, scalar.body, exit.block
8426 //
8427 // exit.block:
8428 // use = lcssa.phi [s1, scalar.body]
8429 //
8430 // In this example, s1 is a recurrence because it's value depends on the
8431 // previous iteration. In the first phase of vectorization, we created a
8432 // VPFirstOrderRecurrencePHIRecipe v1 for s1. Now we create the extracts
8433 // for users in the scalar preheader and exit block.
8434 //
8435 // vector.ph:
8436 // v_init = vector(..., ..., ..., a[-1])
8437 // br vector.body
8438 //
8439 // vector.body
8440 // i = phi [0, vector.ph], [i+4, vector.body]
8441 // v1 = phi [v_init, vector.ph], [v2, vector.body]
8442 // v2 = a[i, i+1, i+2, i+3]
8443 // b[i] = v2 - v1
8444 // // Next, third phase will introduce v1' = splice(v1(3), v2(0, 1, 2))
8445 // b[i, i+1, i+2, i+3] = v2 - v1
8446 // br cond, vector.body, middle.block
8447 //
8448 // middle.block:
8449 // vector.recur.extract.for.phi = v2(2)
8450 // vector.recur.extract = v2(3)
8451 // br cond, scalar.ph, exit.block
8452 //
8453 // scalar.ph:
8454 // scalar.recur.init = phi [vector.recur.extract, middle.block],
8455 // [s.init, otherwise]
8456 // br scalar.body
8457 //
8458 // scalar.body:
8459 // i = phi [0, scalar.ph], [i+1, scalar.body]
8460 // s1 = phi [scalar.recur.init, scalar.ph], [s2, scalar.body]
8461 // s2 = a[i]
8462 // b[i] = s2 - s1
8463 // br cond, scalar.body, exit.block
8464 //
8465 // exit.block:
8466 // lo = lcssa.phi [s1, scalar.body],
8467 // [vector.recur.extract.for.phi, middle.block]
8468 //
8469 // Now update VPIRInstructions modeling LCSSA phis in the exit block.
8470 // Extract the penultimate value of the recurrence and use it as operand for
8471 // the VPIRInstruction modeling the phi.
8472 for (VPUser *U : FOR->users()) {
8473 using namespace llvm::VPlanPatternMatch;
8474 if (!match(U, m_ExtractLastElement(m_Specific(FOR))))
8475 continue;
8476 // For VF vscale x 1, if vscale = 1, we are unable to extract the
8477 // penultimate value of the recurrence. Instead we rely on the existing
8478 // extract of the last element from the result of
8479 // VPInstruction::FirstOrderRecurrenceSplice.
8480 // TODO: Consider vscale_range info and UF.
8482 Range))
8483 return;
8484 VPValue *PenultimateElement = MiddleBuilder.createNaryOp(
8485 VPInstruction::ExtractPenultimateElement, {FOR->getBackedgeValue()},
8486 {}, "vector.recur.extract.for.phi");
8487 cast<VPInstruction>(U)->replaceAllUsesWith(PenultimateElement);
8488 }
8489 }
8490}
8491
8492VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
8493 VPlanPtr Plan, VFRange &Range, LoopVersioning *LVer) {
8494
8495 using namespace llvm::VPlanPatternMatch;
8496 SmallPtrSet<const InterleaveGroup<Instruction> *, 1> InterleaveGroups;
8497
8498 // ---------------------------------------------------------------------------
8499 // Build initial VPlan: Scan the body of the loop in a topological order to
8500 // visit each basic block after having visited its predecessor basic blocks.
8501 // ---------------------------------------------------------------------------
8502
8503 bool RequiresScalarEpilogueCheck =
8505 [this](ElementCount VF) {
8506 return !CM.requiresScalarEpilogue(VF.isVector());
8507 },
8508 Range);
8509 VPlanTransforms::handleEarlyExits(*Plan, Legal->hasUncountableEarlyExit());
8510 VPlanTransforms::addMiddleCheck(*Plan, RequiresScalarEpilogueCheck,
8511 CM.foldTailByMasking());
8512
8514
8515 // Don't use getDecisionAndClampRange here, because we don't know the UF
8516 // so this function is better to be conservative, rather than to split
8517 // it up into different VPlans.
8518 // TODO: Consider using getDecisionAndClampRange here to split up VPlans.
8519 bool IVUpdateMayOverflow = false;
8520 for (ElementCount VF : Range)
8521 IVUpdateMayOverflow |= !isIndvarOverflowCheckKnownFalse(&CM, VF);
8522
8523 TailFoldingStyle Style = CM.getTailFoldingStyle(IVUpdateMayOverflow);
8524 // Use NUW for the induction increment if we proved that it won't overflow in
8525 // the vector loop or when not folding the tail. In the later case, we know
8526 // that the canonical induction increment will not overflow as the vector trip
8527 // count is >= increment and a multiple of the increment.
8528 bool HasNUW = !IVUpdateMayOverflow || Style == TailFoldingStyle::None;
8529 if (!HasNUW) {
8530 auto *IVInc = Plan->getVectorLoopRegion()
8531 ->getExitingBasicBlock()
8532 ->getTerminator()
8533 ->getOperand(0);
8534 assert(match(IVInc, m_VPInstruction<Instruction::Add>(
8535 m_Specific(Plan->getCanonicalIV()), m_VPValue())) &&
8536 "Did not find the canonical IV increment");
8537 cast<VPRecipeWithIRFlags>(IVInc)->dropPoisonGeneratingFlags();
8538 }
8539
8540 // ---------------------------------------------------------------------------
8541 // Pre-construction: record ingredients whose recipes we'll need to further
8542 // process after constructing the initial VPlan.
8543 // ---------------------------------------------------------------------------
8544
8545 // For each interleave group which is relevant for this (possibly trimmed)
8546 // Range, add it to the set of groups to be later applied to the VPlan and add
8547 // placeholders for its members' Recipes which we'll be replacing with a
8548 // single VPInterleaveRecipe.
8549 for (InterleaveGroup<Instruction> *IG : IAI.getInterleaveGroups()) {
8550 auto ApplyIG = [IG, this](ElementCount VF) -> bool {
8551 bool Result = (VF.isVector() && // Query is illegal for VF == 1
8552 CM.getWideningDecision(IG->getInsertPos(), VF) ==
8554 // For scalable vectors, the interleave factors must be <= 8 since we
8555 // require the (de)interleaveN intrinsics instead of shufflevectors.
8556 assert((!Result || !VF.isScalable() || IG->getFactor() <= 8) &&
8557 "Unsupported interleave factor for scalable vectors");
8558 return Result;
8559 };
8560 if (!getDecisionAndClampRange(ApplyIG, Range))
8561 continue;
8562 InterleaveGroups.insert(IG);
8563 }
8564
8565 // ---------------------------------------------------------------------------
8566 // Predicate and linearize the top-level loop region.
8567 // ---------------------------------------------------------------------------
8568 auto BlockMaskCache = VPlanTransforms::introduceMasksAndLinearize(
8569 *Plan, CM.foldTailByMasking());
8570
8571 // ---------------------------------------------------------------------------
8572 // Construct wide recipes and apply predication for original scalar
8573 // VPInstructions in the loop.
8574 // ---------------------------------------------------------------------------
8575 VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
8576 Builder, BlockMaskCache, LVer);
8577 RecipeBuilder.collectScaledReductions(Range);
8578
8579 // Scan the body of the loop in a topological order to visit each basic block
8580 // after having visited its predecessor basic blocks.
8581 VPRegionBlock *LoopRegion = Plan->getVectorLoopRegion();
8582 VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
8583 ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
8584 HeaderVPBB);
8585
8586 auto *MiddleVPBB = Plan->getMiddleBlock();
8587 VPBasicBlock::iterator MBIP = MiddleVPBB->getFirstNonPhi();
8588 // Mapping from VPValues in the initial plan to their widened VPValues. Needed
8589 // temporarily to update created block masks.
8590 DenseMap<VPValue *, VPValue *> Old2New;
8591 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
8592 // Convert input VPInstructions to widened recipes.
8593 for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
8594 auto *SingleDef = cast<VPSingleDefRecipe>(&R);
8595 auto *UnderlyingValue = SingleDef->getUnderlyingValue();
8596 // Skip recipes that do not need transforming, including canonical IV,
8597 // wide canonical IV and VPInstructions without underlying values. The
8598 // latter are added above for masking.
8599 // FIXME: Migrate code relying on the underlying instruction from VPlan0
8600 // to construct recipes below to not use the underlying instruction.
8602 &R) ||
8603 (isa<VPInstruction>(&R) && !UnderlyingValue))
8604 continue;
8605
8606 // FIXME: VPlan0, which models a copy of the original scalar loop, should
8607 // not use VPWidenPHIRecipe to model the phis.
8609 UnderlyingValue && "unsupported recipe");
8610
8611 // TODO: Gradually replace uses of underlying instruction by analyses on
8612 // VPlan.
8613 Instruction *Instr = cast<Instruction>(UnderlyingValue);
8614 Builder.setInsertPoint(SingleDef);
8615
8616 // The stores with invariant address inside the loop will be deleted, and
8617 // in the exit block, a uniform store recipe will be created for the final
8618 // invariant store of the reduction.
8619 StoreInst *SI;
8620 if ((SI = dyn_cast<StoreInst>(Instr)) &&
8621 Legal->isInvariantAddressOfReduction(SI->getPointerOperand())) {
8622 // Only create recipe for the final invariant store of the reduction.
8623 if (Legal->isInvariantStoreOfReduction(SI)) {
8624 auto *Recipe =
8625 new VPReplicateRecipe(SI, R.operands(), true /* IsUniform */,
8626 nullptr /*Mask*/, VPIRMetadata(*SI, LVer));
8627 Recipe->insertBefore(*MiddleVPBB, MBIP);
8628 }
8629 R.eraseFromParent();
8630 continue;
8631 }
8632
8633 VPRecipeBase *Recipe =
8634 RecipeBuilder.tryToCreateWidenRecipe(SingleDef, Range);
8635 if (!Recipe)
8636 Recipe = RecipeBuilder.handleReplication(Instr, R.operands(), Range);
8637
8638 RecipeBuilder.setRecipe(Instr, Recipe);
8639 if (isa<VPWidenIntOrFpInductionRecipe>(Recipe) && isa<TruncInst>(Instr)) {
8640 // Optimized a truncate to VPWidenIntOrFpInductionRecipe. It needs to be
8641 // moved to the phi section in the header.
8642 Recipe->insertBefore(*HeaderVPBB, HeaderVPBB->getFirstNonPhi());
8643 } else {
8644 Builder.insert(Recipe);
8645 }
8646 if (Recipe->getNumDefinedValues() == 1) {
8647 SingleDef->replaceAllUsesWith(Recipe->getVPSingleValue());
8648 Old2New[SingleDef] = Recipe->getVPSingleValue();
8649 } else {
8650 assert(Recipe->getNumDefinedValues() == 0 &&
8651 "Unexpected multidef recipe");
8652 R.eraseFromParent();
8653 }
8654 }
8655 }
8656
8657 // replaceAllUsesWith above may invalidate the block masks. Update them here.
8658 // TODO: Include the masks as operands in the predicated VPlan directly
8659 // to remove the need to keep a map of masks beyond the predication
8660 // transform.
8661 RecipeBuilder.updateBlockMaskCache(Old2New);
8662 for (VPValue *Old : Old2New.keys())
8663 Old->getDefiningRecipe()->eraseFromParent();
8664
8665 assert(isa<VPRegionBlock>(Plan->getVectorLoopRegion()) &&
8666 !Plan->getVectorLoopRegion()->getEntryBasicBlock()->empty() &&
8667 "entry block must be set to a VPRegionBlock having a non-empty entry "
8668 "VPBasicBlock");
8669
8670 // Update wide induction increments to use the same step as the corresponding
8671 // wide induction. This enables detecting induction increments directly in
8672 // VPlan and removes redundant splats.
8673 for (const auto &[Phi, ID] : Legal->getInductionVars()) {
8674 auto *IVInc = cast<Instruction>(
8675 Phi->getIncomingValueForBlock(OrigLoop->getLoopLatch()));
8676 if (IVInc->getOperand(0) != Phi || IVInc->getOpcode() != Instruction::Add)
8677 continue;
8678 VPWidenInductionRecipe *WideIV =
8679 cast<VPWidenInductionRecipe>(RecipeBuilder.getRecipe(Phi));
8680 VPRecipeBase *R = RecipeBuilder.getRecipe(IVInc);
8681 R->setOperand(1, WideIV->getStepValue());
8682 }
8683
8685 DenseMap<VPValue *, VPValue *> IVEndValues;
8686 addScalarResumePhis(RecipeBuilder, *Plan, IVEndValues);
8687
8688 // ---------------------------------------------------------------------------
8689 // Transform initial VPlan: Apply previously taken decisions, in order, to
8690 // bring the VPlan to its final state.
8691 // ---------------------------------------------------------------------------
8692
8693 // Adjust the recipes for any inloop reductions.
8694 adjustRecipesForReductions(Plan, RecipeBuilder, Range.Start);
8695
8696 // Apply mandatory transformation to handle FP maxnum/minnum reduction with
8697 // NaNs if possible, bail out otherwise.
8699 *Plan))
8700 return nullptr;
8701
8702 // Transform recipes to abstract recipes if it is legal and beneficial and
8703 // clamp the range for better cost estimation.
8704 // TODO: Enable following transform when the EVL-version of extended-reduction
8705 // and mulacc-reduction are implemented.
8706 if (!CM.foldTailWithEVL()) {
8707 VPCostContext CostCtx(CM.TTI, *CM.TLI, *Plan, CM, CM.CostKind);
8709 CostCtx, Range);
8710 }
8711
8712 for (ElementCount VF : Range)
8713 Plan->addVF(VF);
8714 Plan->setName("Initial VPlan");
8715
8716 // Interleave memory: for each Interleave Group we marked earlier as relevant
8717 // for this VPlan, replace the Recipes widening its memory instructions with a
8718 // single VPInterleaveRecipe at its insertion point.
8720 InterleaveGroups, RecipeBuilder,
8721 CM.isScalarEpilogueAllowed());
8722
8723 // Replace VPValues for known constant strides.
8725 Legal->getLAI()->getSymbolicStrides());
8726
8727 auto BlockNeedsPredication = [this](BasicBlock *BB) {
8728 return Legal->blockNeedsPredication(BB);
8729 };
8731 BlockNeedsPredication);
8732
8733 // Sink users of fixed-order recurrence past the recipe defining the previous
8734 // value and introduce FirstOrderRecurrenceSplice VPInstructions.
8736 *Plan, Builder))
8737 return nullptr;
8738
8739 if (useActiveLaneMask(Style)) {
8740 // TODO: Move checks to VPlanTransforms::addActiveLaneMask once
8741 // TailFoldingStyle is visible there.
8742 bool ForControlFlow = useActiveLaneMaskForControlFlow(Style);
8743 bool WithoutRuntimeCheck =
8745 VPlanTransforms::addActiveLaneMask(*Plan, ForControlFlow,
8746 WithoutRuntimeCheck);
8747 }
8748 VPlanTransforms::optimizeInductionExitUsers(*Plan, IVEndValues, *PSE.getSE());
8749
8750 assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
8751 return Plan;
8752}
8753
8754VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
8755 // Outer loop handling: They may require CFG and instruction level
8756 // transformations before even evaluating whether vectorization is profitable.
8757 // Since we cannot modify the incoming IR, we need to build VPlan upfront in
8758 // the vectorization pipeline.
8759 assert(!OrigLoop->isInnermost());
8760 assert(EnableVPlanNativePath && "VPlan-native path is not enabled.");
8761
8762 auto Plan = VPlanTransforms::buildVPlan0(
8763 OrigLoop, *LI, Legal->getWidestInductionType(),
8764 getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()), PSE);
8766 /*HasUncountableExit*/ false);
8767 VPlanTransforms::addMiddleCheck(*Plan, /*RequiresScalarEpilogue*/ true,
8768 /*TailFolded*/ false);
8769
8771
8772 for (ElementCount VF : Range)
8773 Plan->addVF(VF);
8774
8776 Plan,
8777 [this](PHINode *P) {
8778 return Legal->getIntOrFpInductionDescriptor(P);
8779 },
8780 *TLI))
8781 return nullptr;
8782
8783 // Collect mapping of IR header phis to header phi recipes, to be used in
8784 // addScalarResumePhis.
8785 DenseMap<VPBasicBlock *, VPValue *> BlockMaskCache;
8786 VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
8787 Builder, BlockMaskCache, nullptr /*LVer*/);
8788 for (auto &R : Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
8790 continue;
8791 auto *HeaderR = cast<VPHeaderPHIRecipe>(&R);
8792 RecipeBuilder.setRecipe(HeaderR->getUnderlyingInstr(), HeaderR);
8793 }
8794 DenseMap<VPValue *, VPValue *> IVEndValues;
8795 // TODO: IVEndValues are not used yet in the native path, to optimize exit
8796 // values.
8797 addScalarResumePhis(RecipeBuilder, *Plan, IVEndValues);
8798
8799 assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
8800 return Plan;
8801}
8802
8803// Adjust the recipes for reductions. For in-loop reductions the chain of
8804// instructions leading from the loop exit instr to the phi need to be converted
8805// to reductions, with one operand being vector and the other being the scalar
8806// reduction chain. For other reductions, a select is introduced between the phi
8807// and users outside the vector region when folding the tail.
8808//
8809// A ComputeReductionResult recipe is added to the middle block, also for
8810// in-loop reductions which compute their result in-loop, because generating
8811// the subsequent bc.merge.rdx phi is driven by ComputeReductionResult recipes.
8812//
8813// Adjust AnyOf reductions; replace the reduction phi for the selected value
8814// with a boolean reduction phi node to check if the condition is true in any
8815// iteration. The final value is selected by the final ComputeReductionResult.
8816void LoopVectorizationPlanner::adjustRecipesForReductions(
8817 VPlanPtr &Plan, VPRecipeBuilder &RecipeBuilder, ElementCount MinVF) {
8818 using namespace VPlanPatternMatch;
8819 VPRegionBlock *VectorLoopRegion = Plan->getVectorLoopRegion();
8820 VPBasicBlock *Header = VectorLoopRegion->getEntryBasicBlock();
8821 VPBasicBlock *MiddleVPBB = Plan->getMiddleBlock();
8823
8824 for (VPRecipeBase &R : Header->phis()) {
8825 auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
8826 if (!PhiR || !PhiR->isInLoop() || (MinVF.isScalar() && !PhiR->isOrdered()))
8827 continue;
8828
8829 RecurKind Kind = PhiR->getRecurrenceKind();
8830 assert(
8833 "AnyOf and FindIV reductions are not allowed for in-loop reductions");
8834
8835 // Collect the chain of "link" recipes for the reduction starting at PhiR.
8836 SetVector<VPSingleDefRecipe *> Worklist;
8837 Worklist.insert(PhiR);
8838 for (unsigned I = 0; I != Worklist.size(); ++I) {
8839 VPSingleDefRecipe *Cur = Worklist[I];
8840 for (VPUser *U : Cur->users()) {
8841 auto *UserRecipe = cast<VPSingleDefRecipe>(U);
8842 if (!UserRecipe->getParent()->getEnclosingLoopRegion()) {
8843 assert((UserRecipe->getParent() == MiddleVPBB ||
8844 UserRecipe->getParent() == Plan->getScalarPreheader()) &&
8845 "U must be either in the loop region, the middle block or the "
8846 "scalar preheader.");
8847 continue;
8848 }
8849 Worklist.insert(UserRecipe);
8850 }
8851 }
8852
8853 // Visit operation "Links" along the reduction chain top-down starting from
8854 // the phi until LoopExitValue. We keep track of the previous item
8855 // (PreviousLink) to tell which of the two operands of a Link will remain
8856 // scalar and which will be reduced. For minmax by select(cmp), Link will be
8857 // the select instructions. Blend recipes of in-loop reduction phi's will
8858 // get folded to their non-phi operand, as the reduction recipe handles the
8859 // condition directly.
8860 VPSingleDefRecipe *PreviousLink = PhiR; // Aka Worklist[0].
8861 for (VPSingleDefRecipe *CurrentLink : drop_begin(Worklist)) {
8862 if (auto *Blend = dyn_cast<VPBlendRecipe>(CurrentLink)) {
8863 assert(Blend->getNumIncomingValues() == 2 &&
8864 "Blend must have 2 incoming values");
8865 if (Blend->getIncomingValue(0) == PhiR) {
8866 Blend->replaceAllUsesWith(Blend->getIncomingValue(1));
8867 } else {
8868 assert(Blend->getIncomingValue(1) == PhiR &&
8869 "PhiR must be an operand of the blend");
8870 Blend->replaceAllUsesWith(Blend->getIncomingValue(0));
8871 }
8872 continue;
8873 }
8874
8875 Instruction *CurrentLinkI = CurrentLink->getUnderlyingInstr();
8876
8877 // Index of the first operand which holds a non-mask vector operand.
8878 unsigned IndexOfFirstOperand;
8879 // Recognize a call to the llvm.fmuladd intrinsic.
8880 bool IsFMulAdd = (Kind == RecurKind::FMulAdd);
8881 VPValue *VecOp;
8882 VPBasicBlock *LinkVPBB = CurrentLink->getParent();
8883 if (IsFMulAdd) {
8884 assert(
8886 "Expected instruction to be a call to the llvm.fmuladd intrinsic");
8887 assert(((MinVF.isScalar() && isa<VPReplicateRecipe>(CurrentLink)) ||
8888 isa<VPWidenIntrinsicRecipe>(CurrentLink)) &&
8889 CurrentLink->getOperand(2) == PreviousLink &&
8890 "expected a call where the previous link is the added operand");
8891
8892 // If the instruction is a call to the llvm.fmuladd intrinsic then we
8893 // need to create an fmul recipe (multiplying the first two operands of
8894 // the fmuladd together) to use as the vector operand for the fadd
8895 // reduction.
8896 VPInstruction *FMulRecipe = new VPInstruction(
8897 Instruction::FMul,
8898 {CurrentLink->getOperand(0), CurrentLink->getOperand(1)},
8899 CurrentLinkI->getFastMathFlags());
8900 LinkVPBB->insert(FMulRecipe, CurrentLink->getIterator());
8901 VecOp = FMulRecipe;
8902 } else if (PhiR->isInLoop() && Kind == RecurKind::AddChainWithSubs &&
8903 CurrentLinkI->getOpcode() == Instruction::Sub) {
8904 Type *PhiTy = PhiR->getUnderlyingValue()->getType();
8905 auto *Zero = Plan->getOrAddLiveIn(ConstantInt::get(PhiTy, 0));
8906 VPWidenRecipe *Sub = new VPWidenRecipe(
8907 Instruction::Sub, {Zero, CurrentLink->getOperand(1)}, {},
8908 VPIRMetadata(), CurrentLinkI->getDebugLoc());
8909 Sub->setUnderlyingValue(CurrentLinkI);
8910 LinkVPBB->insert(Sub, CurrentLink->getIterator());
8911 VecOp = Sub;
8912 } else {
8914 if (isa<VPWidenRecipe>(CurrentLink)) {
8915 assert(isa<CmpInst>(CurrentLinkI) &&
8916 "need to have the compare of the select");
8917 continue;
8918 }
8919 assert(isa<VPWidenSelectRecipe>(CurrentLink) &&
8920 "must be a select recipe");
8921 IndexOfFirstOperand = 1;
8922 } else {
8923 assert((MinVF.isScalar() || isa<VPWidenRecipe>(CurrentLink)) &&
8924 "Expected to replace a VPWidenSC");
8925 IndexOfFirstOperand = 0;
8926 }
8927 // Note that for non-commutable operands (cmp-selects), the semantics of
8928 // the cmp-select are captured in the recurrence kind.
8929 unsigned VecOpId =
8930 CurrentLink->getOperand(IndexOfFirstOperand) == PreviousLink
8931 ? IndexOfFirstOperand + 1
8932 : IndexOfFirstOperand;
8933 VecOp = CurrentLink->getOperand(VecOpId);
8934 assert(VecOp != PreviousLink &&
8935 CurrentLink->getOperand(CurrentLink->getNumOperands() - 1 -
8936 (VecOpId - IndexOfFirstOperand)) ==
8937 PreviousLink &&
8938 "PreviousLink must be the operand other than VecOp");
8939 }
8940
8941 VPValue *CondOp = nullptr;
8942 if (CM.blockNeedsPredicationForAnyReason(CurrentLinkI->getParent()))
8943 CondOp = RecipeBuilder.getBlockInMask(CurrentLink->getParent());
8944
8945 // TODO: Retrieve FMFs from recipes directly.
8946 RecurrenceDescriptor RdxDesc = Legal->getRecurrenceDescriptor(
8947 cast<PHINode>(PhiR->getUnderlyingInstr()));
8948 // Non-FP RdxDescs will have all fast math flags set, so clear them.
8949 FastMathFlags FMFs = isa<FPMathOperator>(CurrentLinkI)
8950 ? RdxDesc.getFastMathFlags()
8951 : FastMathFlags();
8952 auto *RedRecipe = new VPReductionRecipe(
8953 Kind, FMFs, CurrentLinkI, PreviousLink, VecOp, CondOp,
8954 PhiR->isOrdered(), CurrentLinkI->getDebugLoc());
8955 // Append the recipe to the end of the VPBasicBlock because we need to
8956 // ensure that it comes after all of it's inputs, including CondOp.
8957 // Delete CurrentLink as it will be invalid if its operand is replaced
8958 // with a reduction defined at the bottom of the block in the next link.
8959 if (LinkVPBB->getNumSuccessors() == 0)
8960 RedRecipe->insertBefore(&*std::prev(std::prev(LinkVPBB->end())));
8961 else
8962 LinkVPBB->appendRecipe(RedRecipe);
8963
8964 CurrentLink->replaceAllUsesWith(RedRecipe);
8965 ToDelete.push_back(CurrentLink);
8966 PreviousLink = RedRecipe;
8967 }
8968 }
8969 VPBasicBlock *LatchVPBB = VectorLoopRegion->getExitingBasicBlock();
8970 Builder.setInsertPoint(&*std::prev(std::prev(LatchVPBB->end())));
8971 VPBasicBlock::iterator IP = MiddleVPBB->getFirstNonPhi();
8972 for (VPRecipeBase &R :
8973 Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
8974 VPReductionPHIRecipe *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
8975 if (!PhiR)
8976 continue;
8977
8978 const RecurrenceDescriptor &RdxDesc = Legal->getRecurrenceDescriptor(
8980 Type *PhiTy = PhiR->getUnderlyingValue()->getType();
8981 // If tail is folded by masking, introduce selects between the phi
8982 // and the users outside the vector region of each reduction, at the
8983 // beginning of the dedicated latch block.
8984 auto *OrigExitingVPV = PhiR->getBackedgeValue();
8985 auto *NewExitingVPV = PhiR->getBackedgeValue();
8986 // Don't output selects for partial reductions because they have an output
8987 // with fewer lanes than the VF. So the operands of the select would have
8988 // different numbers of lanes. Partial reductions mask the input instead.
8989 if (!PhiR->isInLoop() && CM.foldTailByMasking() &&
8990 !isa<VPPartialReductionRecipe>(OrigExitingVPV->getDefiningRecipe())) {
8991 VPValue *Cond = RecipeBuilder.getBlockInMask(PhiR->getParent());
8992 std::optional<FastMathFlags> FMFs =
8993 PhiTy->isFloatingPointTy()
8994 ? std::make_optional(RdxDesc.getFastMathFlags())
8995 : std::nullopt;
8996 NewExitingVPV =
8997 Builder.createSelect(Cond, OrigExitingVPV, PhiR, {}, "", FMFs);
8998 OrigExitingVPV->replaceUsesWithIf(NewExitingVPV, [](VPUser &U, unsigned) {
8999 return isa<VPInstruction>(&U) &&
9000 (cast<VPInstruction>(&U)->getOpcode() ==
9002 cast<VPInstruction>(&U)->getOpcode() ==
9004 cast<VPInstruction>(&U)->getOpcode() ==
9006 });
9007 if (CM.usePredicatedReductionSelect())
9008 PhiR->setOperand(1, NewExitingVPV);
9009 }
9010
9011 // We want code in the middle block to appear to execute on the location of
9012 // the scalar loop's latch terminator because: (a) it is all compiler
9013 // generated, (b) these instructions are always executed after evaluating
9014 // the latch conditional branch, and (c) other passes may add new
9015 // predecessors which terminate on this line. This is the easiest way to
9016 // ensure we don't accidentally cause an extra step back into the loop while
9017 // debugging.
9018 DebugLoc ExitDL = OrigLoop->getLoopLatch()->getTerminator()->getDebugLoc();
9019
9020 // TODO: At the moment ComputeReductionResult also drives creation of the
9021 // bc.merge.rdx phi nodes, hence it needs to be created unconditionally here
9022 // even for in-loop reductions, until the reduction resume value handling is
9023 // also modeled in VPlan.
9024 VPInstruction *FinalReductionResult;
9025 VPBuilder::InsertPointGuard Guard(Builder);
9026 Builder.setInsertPoint(MiddleVPBB, IP);
9027 RecurKind RecurrenceKind = PhiR->getRecurrenceKind();
9029 VPValue *Start = PhiR->getStartValue();
9030 VPValue *Sentinel = Plan->getOrAddLiveIn(RdxDesc.getSentinelValue());
9031 FinalReductionResult =
9032 Builder.createNaryOp(VPInstruction::ComputeFindIVResult,
9033 {PhiR, Start, Sentinel, NewExitingVPV}, ExitDL);
9034 } else if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RecurrenceKind)) {
9035 VPValue *Start = PhiR->getStartValue();
9036 FinalReductionResult =
9037 Builder.createNaryOp(VPInstruction::ComputeAnyOfResult,
9038 {PhiR, Start, NewExitingVPV}, ExitDL);
9039 } else {
9040 VPIRFlags Flags =
9042 ? VPIRFlags(RdxDesc.getFastMathFlags())
9043 : VPIRFlags();
9044 FinalReductionResult =
9045 Builder.createNaryOp(VPInstruction::ComputeReductionResult,
9046 {PhiR, NewExitingVPV}, Flags, ExitDL);
9047 }
9048 // If the vector reduction can be performed in a smaller type, we truncate
9049 // then extend the loop exit value to enable InstCombine to evaluate the
9050 // entire expression in the smaller type.
9051 if (MinVF.isVector() && PhiTy != RdxDesc.getRecurrenceType() &&
9053 assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
9055 "Unexpected truncated min-max recurrence!");
9056 Type *RdxTy = RdxDesc.getRecurrenceType();
9057 auto *Trunc =
9058 new VPWidenCastRecipe(Instruction::Trunc, NewExitingVPV, RdxTy);
9059 Instruction::CastOps ExtendOpc =
9060 RdxDesc.isSigned() ? Instruction::SExt : Instruction::ZExt;
9061 auto *Extnd = new VPWidenCastRecipe(ExtendOpc, Trunc, PhiTy);
9062 Trunc->insertAfter(NewExitingVPV->getDefiningRecipe());
9063 Extnd->insertAfter(Trunc);
9064 if (PhiR->getOperand(1) == NewExitingVPV)
9065 PhiR->setOperand(1, Extnd->getVPSingleValue());
9066
9067 // Update ComputeReductionResult with the truncated exiting value and
9068 // extend its result.
9069 FinalReductionResult->setOperand(1, Trunc);
9070 FinalReductionResult =
9071 Builder.createScalarCast(ExtendOpc, FinalReductionResult, PhiTy, {});
9072 }
9073
9074 // Update all users outside the vector region. Also replace redundant
9075 // ExtractLastElement.
9076 for (auto *U : to_vector(OrigExitingVPV->users())) {
9077 auto *Parent = cast<VPRecipeBase>(U)->getParent();
9078 if (FinalReductionResult == U || Parent->getParent())
9079 continue;
9080 U->replaceUsesOfWith(OrigExitingVPV, FinalReductionResult);
9082 cast<VPInstruction>(U)->replaceAllUsesWith(FinalReductionResult);
9083 }
9084
9085 // Adjust AnyOf reductions; replace the reduction phi for the selected value
9086 // with a boolean reduction phi node to check if the condition is true in
9087 // any iteration. The final value is selected by the final
9088 // ComputeReductionResult.
9089 if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RecurrenceKind)) {
9090 auto *Select = cast<VPRecipeBase>(*find_if(PhiR->users(), [](VPUser *U) {
9091 return isa<VPWidenSelectRecipe>(U) ||
9092 (isa<VPReplicateRecipe>(U) &&
9093 cast<VPReplicateRecipe>(U)->getUnderlyingInstr()->getOpcode() ==
9094 Instruction::Select);
9095 }));
9096 VPValue *Cmp = Select->getOperand(0);
9097 // If the compare is checking the reduction PHI node, adjust it to check
9098 // the start value.
9099 if (VPRecipeBase *CmpR = Cmp->getDefiningRecipe())
9100 CmpR->replaceUsesOfWith(PhiR, PhiR->getStartValue());
9101 Builder.setInsertPoint(Select);
9102
9103 // If the true value of the select is the reduction phi, the new value is
9104 // selected if the negated condition is true in any iteration.
9105 if (Select->getOperand(1) == PhiR)
9106 Cmp = Builder.createNot(Cmp);
9107 VPValue *Or = Builder.createOr(PhiR, Cmp);
9108 Select->getVPSingleValue()->replaceAllUsesWith(Or);
9109 // Delete Select now that it has invalid types.
9110 ToDelete.push_back(Select);
9111
9112 // Convert the reduction phi to operate on bools.
9113 PhiR->setOperand(0, Plan->getOrAddLiveIn(ConstantInt::getFalse(
9114 OrigLoop->getHeader()->getContext())));
9115 continue;
9116 }
9117
9119 RdxDesc.getRecurrenceKind())) {
9120 // Adjust the start value for FindFirstIV/FindLastIV recurrences to use
9121 // the sentinel value after generating the ResumePhi recipe, which uses
9122 // the original start value.
9123 PhiR->setOperand(0, Plan->getOrAddLiveIn(RdxDesc.getSentinelValue()));
9124 }
9125 RecurKind RK = RdxDesc.getRecurrenceKind();
9129 VPBuilder PHBuilder(Plan->getVectorPreheader());
9130 VPValue *Iden = Plan->getOrAddLiveIn(
9131 getRecurrenceIdentity(RK, PhiTy, RdxDesc.getFastMathFlags()));
9132 // If the PHI is used by a partial reduction, set the scale factor.
9133 unsigned ScaleFactor =
9134 RecipeBuilder.getScalingForReduction(RdxDesc.getLoopExitInstr())
9135 .value_or(1);
9136 Type *I32Ty = IntegerType::getInt32Ty(PhiTy->getContext());
9137 auto *ScaleFactorVPV =
9138 Plan->getOrAddLiveIn(ConstantInt::get(I32Ty, ScaleFactor));
9139 VPValue *StartV = PHBuilder.createNaryOp(
9141 {PhiR->getStartValue(), Iden, ScaleFactorVPV},
9142 PhiTy->isFloatingPointTy() ? RdxDesc.getFastMathFlags()
9143 : FastMathFlags());
9144 PhiR->setOperand(0, StartV);
9145 }
9146 }
9147 for (VPRecipeBase *R : ToDelete)
9148 R->eraseFromParent();
9149
9151}
9152
9153void LoopVectorizationPlanner::attachRuntimeChecks(
9154 VPlan &Plan, GeneratedRTChecks &RTChecks, bool HasBranchWeights) const {
9155 const auto &[SCEVCheckCond, SCEVCheckBlock] = RTChecks.getSCEVChecks();
9156 if (SCEVCheckBlock && SCEVCheckBlock->hasNPredecessors(0)) {
9157 assert((!CM.OptForSize ||
9158 CM.Hints->getForce() == LoopVectorizeHints::FK_Enabled) &&
9159 "Cannot SCEV check stride or overflow when optimizing for size");
9160 VPlanTransforms::attachCheckBlock(Plan, SCEVCheckCond, SCEVCheckBlock,
9161 HasBranchWeights);
9162 }
9163 const auto &[MemCheckCond, MemCheckBlock] = RTChecks.getMemRuntimeChecks();
9164 if (MemCheckBlock && MemCheckBlock->hasNPredecessors(0)) {
9165 // VPlan-native path does not do any analysis for runtime checks
9166 // currently.
9167 assert((!EnableVPlanNativePath || OrigLoop->isInnermost()) &&
9168 "Runtime checks are not supported for outer loops yet");
9169
9170 if (CM.OptForSize) {
9171 assert(
9172 CM.Hints->getForce() == LoopVectorizeHints::FK_Enabled &&
9173 "Cannot emit memory checks when optimizing for size, unless forced "
9174 "to vectorize.");
9175 ORE->emit([&]() {
9176 return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationCodeSize",
9177 OrigLoop->getStartLoc(),
9178 OrigLoop->getHeader())
9179 << "Code-size may be reduced by not forcing "
9180 "vectorization, or by source-code modifications "
9181 "eliminating the need for runtime checks "
9182 "(e.g., adding 'restrict').";
9183 });
9184 }
9185 VPlanTransforms::attachCheckBlock(Plan, MemCheckCond, MemCheckBlock,
9186 HasBranchWeights);
9187 }
9188}
9189
9191 VPlan &Plan, ElementCount VF, unsigned UF,
9192 ElementCount MinProfitableTripCount) const {
9193 // vscale is not necessarily a power-of-2, which means we cannot guarantee
9194 // an overflow to zero when updating induction variables and so an
9195 // additional overflow check is required before entering the vector loop.
9196 bool IsIndvarOverflowCheckNeededForVF =
9197 VF.isScalable() && !TTI.isVScaleKnownToBeAPowerOfTwo() &&
9198 !isIndvarOverflowCheckKnownFalse(&CM, VF, UF) &&
9199 CM.getTailFoldingStyle() !=
9201 const uint32_t *BranchWeigths =
9202 hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator())
9204 : nullptr;
9206 Plan, VF, UF, MinProfitableTripCount,
9207 CM.requiresScalarEpilogue(VF.isVector()), CM.foldTailByMasking(),
9208 IsIndvarOverflowCheckNeededForVF, OrigLoop, BranchWeigths,
9209 OrigLoop->getLoopPredecessor()->getTerminator()->getDebugLoc(),
9210 *PSE.getSE());
9211}
9212
9214 assert(!State.Lane && "VPDerivedIVRecipe being replicated.");
9215
9216 // Fast-math-flags propagate from the original induction instruction.
9217 IRBuilder<>::FastMathFlagGuard FMFG(State.Builder);
9218 if (FPBinOp)
9219 State.Builder.setFastMathFlags(FPBinOp->getFastMathFlags());
9220
9221 Value *Step = State.get(getStepValue(), VPLane(0));
9222 Value *Index = State.get(getOperand(1), VPLane(0));
9223 Value *DerivedIV = emitTransformedIndex(
9224 State.Builder, Index, getStartValue()->getLiveInIRValue(), Step, Kind,
9226 DerivedIV->setName(Name);
9227 State.set(this, DerivedIV, VPLane(0));
9228}
9229
9230// Determine how to lower the scalar epilogue, which depends on 1) optimising
9231// for minimum code-size, 2) predicate compiler options, 3) loop hints forcing
9232// predication, and 4) a TTI hook that analyses whether the loop is suitable
9233// for predication.
9238 // 1) OptSize takes precedence over all other options, i.e. if this is set,
9239 // don't look at hints or options, and don't request a scalar epilogue.
9240 // (For PGSO, as shouldOptimizeForSize isn't currently accessible from
9241 // LoopAccessInfo (due to code dependency and not being able to reliably get
9242 // PSI/BFI from a loop analysis under NPM), we cannot suppress the collection
9243 // of strides in LoopAccessInfo::analyzeLoop() and vectorize without
9244 // versioning when the vectorization is forced, unlike hasOptSize. So revert
9245 // back to the old way and vectorize with versioning when forced. See D81345.)
9246 if (F->hasOptSize() || (llvm::shouldOptimizeForSize(L->getHeader(), PSI, BFI,
9250
9251 // 2) If set, obey the directives
9252 if (PreferPredicateOverEpilogue.getNumOccurrences()) {
9260 };
9261 }
9262
9263 // 3) If set, obey the hints
9264 switch (Hints.getPredicate()) {
9269 };
9270
9271 // 4) if the TTI hook indicates this is profitable, request predication.
9272 TailFoldingInfo TFI(TLI, &LVL, IAI);
9273 if (TTI->preferPredicateOverEpilogue(&TFI))
9275
9277}
9278
9279// Process the loop in the VPlan-native vectorization path. This path builds
9280// VPlan upfront in the vectorization pipeline, which allows to apply
9281// VPlan-to-VPlan transformations from the very beginning without modifying the
9282// input LLVM IR.
9289 LoopVectorizationRequirements &Requirements) {
9290
9292 LLVM_DEBUG(dbgs() << "LV: cannot compute the outer-loop trip count\n");
9293 return false;
9294 }
9295 assert(EnableVPlanNativePath && "VPlan-native path is disabled.");
9296 Function *F = L->getHeader()->getParent();
9297 InterleavedAccessInfo IAI(PSE, L, DT, LI, LVL->getLAI());
9298
9300 getScalarEpilogueLowering(F, L, Hints, PSI, BFI, TTI, TLI, *LVL, &IAI);
9301
9302 LoopVectorizationCostModel CM(SEL, L, PSE, LI, LVL, *TTI, TLI, DB, AC, ORE, F,
9303 &Hints, IAI, PSI, BFI);
9304 // Use the planner for outer loop vectorization.
9305 // TODO: CM is not used at this point inside the planner. Turn CM into an
9306 // optional argument if we don't need it in the future.
9307 LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, LVL, CM, IAI, PSE, Hints,
9308 ORE);
9309
9310 // Get user vectorization factor.
9311 ElementCount UserVF = Hints.getWidth();
9312
9314
9315 // Plan how to best vectorize, return the best VF and its cost.
9316 const VectorizationFactor VF = LVP.planInVPlanNativePath(UserVF);
9317
9318 // If we are stress testing VPlan builds, do not attempt to generate vector
9319 // code. Masked vector code generation support will follow soon.
9320 // Also, do not attempt to vectorize if no vector code will be produced.
9322 return false;
9323
9324 VPlan &BestPlan = LVP.getPlanFor(VF.Width);
9325
9326 {
9327 GeneratedRTChecks Checks(PSE, DT, LI, TTI, F->getDataLayout(), CM.CostKind);
9328 InnerLoopVectorizer LB(L, PSE, LI, DT, TTI, AC, VF.Width, /*UF=*/1, &CM,
9329 BFI, PSI, Checks, BestPlan);
9330 LLVM_DEBUG(dbgs() << "Vectorizing outer loop in \""
9331 << L->getHeader()->getParent()->getName() << "\"\n");
9332 LVP.addMinimumIterationCheck(BestPlan, VF.Width, /*UF=*/1,
9334
9335 LVP.executePlan(VF.Width, /*UF=*/1, BestPlan, LB, DT, false);
9336 }
9337
9338 reportVectorization(ORE, L, VF, 1);
9339
9340 assert(!verifyFunction(*L->getHeader()->getParent(), &dbgs()));
9341 return true;
9342}
9343
9344// Emit a remark if there are stores to floats that required a floating point
9345// extension. If the vectorized loop was generated with floating point there
9346// will be a performance penalty from the conversion overhead and the change in
9347// the vector width.
9350 for (BasicBlock *BB : L->getBlocks()) {
9351 for (Instruction &Inst : *BB) {
9352 if (auto *S = dyn_cast<StoreInst>(&Inst)) {
9353 if (S->getValueOperand()->getType()->isFloatTy())
9354 Worklist.push_back(S);
9355 }
9356 }
9357 }
9358
9359 // Traverse the floating point stores upwards searching, for floating point
9360 // conversions.
9363 while (!Worklist.empty()) {
9364 auto *I = Worklist.pop_back_val();
9365 if (!L->contains(I))
9366 continue;
9367 if (!Visited.insert(I).second)
9368 continue;
9369
9370 // Emit a remark if the floating point store required a floating
9371 // point conversion.
9372 // TODO: More work could be done to identify the root cause such as a
9373 // constant or a function return type and point the user to it.
9374 if (isa<FPExtInst>(I) && EmittedRemark.insert(I).second)
9375 ORE->emit([&]() {
9376 return OptimizationRemarkAnalysis(LV_NAME, "VectorMixedPrecision",
9377 I->getDebugLoc(), L->getHeader())
9378 << "floating point conversion changes vector width. "
9379 << "Mixed floating point precision requires an up/down "
9380 << "cast that will negatively impact performance.";
9381 });
9382
9383 for (Use &Op : I->operands())
9384 if (auto *OpI = dyn_cast<Instruction>(Op))
9385 Worklist.push_back(OpI);
9386 }
9387}
9388
9389/// For loops with uncountable early exits, find the cost of doing work when
9390/// exiting the loop early, such as calculating the final exit values of
9391/// variables used outside the loop.
9392/// TODO: This is currently overly pessimistic because the loop may not take
9393/// the early exit, but better to keep this conservative for now. In future,
9394/// it might be possible to relax this by using branch probabilities.
9396 VPlan &Plan, ElementCount VF) {
9397 InstructionCost Cost = 0;
9398 for (auto *ExitVPBB : Plan.getExitBlocks()) {
9399 for (auto *PredVPBB : ExitVPBB->getPredecessors()) {
9400 // If the predecessor is not the middle.block, then it must be the
9401 // vector.early.exit block, which may contain work to calculate the exit
9402 // values of variables used outside the loop.
9403 if (PredVPBB != Plan.getMiddleBlock()) {
9404 LLVM_DEBUG(dbgs() << "Calculating cost of work in exit block "
9405 << PredVPBB->getName() << ":\n");
9406 Cost += PredVPBB->cost(VF, CostCtx);
9407 }
9408 }
9409 }
9410 return Cost;
9411}
9412
9413/// This function determines whether or not it's still profitable to vectorize
9414/// the loop given the extra work we have to do outside of the loop:
9415/// 1. Perform the runtime checks before entering the loop to ensure it's safe
9416/// to vectorize.
9417/// 2. In the case of loops with uncountable early exits, we may have to do
9418/// extra work when exiting the loop early, such as calculating the final
9419/// exit values of variables used outside the loop.
9420static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks,
9421 VectorizationFactor &VF, Loop *L,
9423 VPCostContext &CostCtx, VPlan &Plan,
9425 std::optional<unsigned> VScale) {
9426 InstructionCost TotalCost = Checks.getCost();
9427 if (!TotalCost.isValid())
9428 return false;
9429
9430 // Add on the cost of any work required in the vector early exit block, if
9431 // one exists.
9432 TotalCost += calculateEarlyExitCost(CostCtx, Plan, VF.Width);
9433
9434 // When interleaving only scalar and vector cost will be equal, which in turn
9435 // would lead to a divide by 0. Fall back to hard threshold.
9436 if (VF.Width.isScalar()) {
9437 // TODO: Should we rename VectorizeMemoryCheckThreshold?
9438 if (TotalCost > VectorizeMemoryCheckThreshold) {
9439 LLVM_DEBUG(
9440 dbgs()
9441 << "LV: Interleaving only is not profitable due to runtime checks\n");
9442 return false;
9443 }
9444 return true;
9445 }
9446
9447 // The scalar cost should only be 0 when vectorizing with a user specified
9448 // VF/IC. In those cases, runtime checks should always be generated.
9449 uint64_t ScalarC = VF.ScalarCost.getValue();
9450 if (ScalarC == 0)
9451 return true;
9452
9453 // First, compute the minimum iteration count required so that the vector
9454 // loop outperforms the scalar loop.
9455 // The total cost of the scalar loop is
9456 // ScalarC * TC
9457 // where
9458 // * TC is the actual trip count of the loop.
9459 // * ScalarC is the cost of a single scalar iteration.
9460 //
9461 // The total cost of the vector loop is
9462 // RtC + VecC * (TC / VF) + EpiC
9463 // where
9464 // * RtC is the cost of the generated runtime checks plus the cost of
9465 // performing any additional work in the vector.early.exit block for loops
9466 // with uncountable early exits.
9467 // * VecC is the cost of a single vector iteration.
9468 // * TC is the actual trip count of the loop
9469 // * VF is the vectorization factor
9470 // * EpiCost is the cost of the generated epilogue, including the cost
9471 // of the remaining scalar operations.
9472 //
9473 // Vectorization is profitable once the total vector cost is less than the
9474 // total scalar cost:
9475 // RtC + VecC * (TC / VF) + EpiC < ScalarC * TC
9476 //
9477 // Now we can compute the minimum required trip count TC as
9478 // VF * (RtC + EpiC) / (ScalarC * VF - VecC) < TC
9479 //
9480 // For now we assume the epilogue cost EpiC = 0 for simplicity. Note that
9481 // the computations are performed on doubles, not integers and the result
9482 // is rounded up, hence we get an upper estimate of the TC.
9483 unsigned IntVF = estimateElementCount(VF.Width, VScale);
9484 uint64_t RtC = TotalCost.getValue();
9485 uint64_t Div = ScalarC * IntVF - VF.Cost.getValue();
9486 uint64_t MinTC1 = Div == 0 ? 0 : divideCeil(RtC * IntVF, Div);
9487
9488 // Second, compute a minimum iteration count so that the cost of the
9489 // runtime checks is only a fraction of the total scalar loop cost. This
9490 // adds a loop-dependent bound on the overhead incurred if the runtime
9491 // checks fail. In case the runtime checks fail, the cost is RtC + ScalarC
9492 // * TC. To bound the runtime check to be a fraction 1/X of the scalar
9493 // cost, compute
9494 // RtC < ScalarC * TC * (1 / X) ==> RtC * X / ScalarC < TC
9495 uint64_t MinTC2 = divideCeil(RtC * 10, ScalarC);
9496
9497 // Now pick the larger minimum. If it is not a multiple of VF and a scalar
9498 // epilogue is allowed, choose the next closest multiple of VF. This should
9499 // partly compensate for ignoring the epilogue cost.
9500 uint64_t MinTC = std::max(MinTC1, MinTC2);
9501 if (SEL == CM_ScalarEpilogueAllowed)
9502 MinTC = alignTo(MinTC, IntVF);
9504
9505 LLVM_DEBUG(
9506 dbgs() << "LV: Minimum required TC for runtime checks to be profitable:"
9507 << VF.MinProfitableTripCount << "\n");
9508
9509 // Skip vectorization if the expected trip count is less than the minimum
9510 // required trip count.
9511 if (auto ExpectedTC = getSmallBestKnownTC(PSE, L)) {
9512 if (ElementCount::isKnownLT(*ExpectedTC, VF.MinProfitableTripCount)) {
9513 LLVM_DEBUG(dbgs() << "LV: Vectorization is not beneficial: expected "
9514 "trip count < minimum profitable VF ("
9515 << *ExpectedTC << " < " << VF.MinProfitableTripCount
9516 << ")\n");
9517
9518 return false;
9519 }
9520 }
9521 return true;
9522}
9523
9525 : InterleaveOnlyWhenForced(Opts.InterleaveOnlyWhenForced ||
9527 VectorizeOnlyWhenForced(Opts.VectorizeOnlyWhenForced ||
9529
9530/// Prepare \p MainPlan for vectorizing the main vector loop during epilogue
9531/// vectorization. Remove ResumePhis from \p MainPlan for inductions that
9532/// don't have a corresponding wide induction in \p EpiPlan.
9533static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
9534 // Collect PHI nodes of widened phis in the VPlan for the epilogue. Those
9535 // will need their resume-values computed in the main vector loop. Others
9536 // can be removed from the main VPlan.
9537 SmallPtrSet<PHINode *, 2> EpiWidenedPhis;
9538 for (VPRecipeBase &R :
9541 continue;
9542 EpiWidenedPhis.insert(
9543 cast<PHINode>(R.getVPSingleValue()->getUnderlyingValue()));
9544 }
9545 for (VPRecipeBase &R :
9546 make_early_inc_range(MainPlan.getScalarHeader()->phis())) {
9547 auto *VPIRInst = cast<VPIRPhi>(&R);
9548 if (EpiWidenedPhis.contains(&VPIRInst->getIRPhi()))
9549 continue;
9550 // There is no corresponding wide induction in the epilogue plan that would
9551 // need a resume value. Remove the VPIRInst wrapping the scalar header phi
9552 // together with the corresponding ResumePhi. The resume values for the
9553 // scalar loop will be created during execution of EpiPlan.
9554 VPRecipeBase *ResumePhi = VPIRInst->getOperand(0)->getDefiningRecipe();
9555 VPIRInst->eraseFromParent();
9556 ResumePhi->eraseFromParent();
9557 }
9559
9560 using namespace VPlanPatternMatch;
9561 // When vectorizing the epilogue, FindFirstIV & FindLastIV reductions can
9562 // introduce multiple uses of undef/poison. If the reduction start value may
9563 // be undef or poison it needs to be frozen and the frozen start has to be
9564 // used when computing the reduction result. We also need to use the frozen
9565 // value in the resume phi generated by the main vector loop, as this is also
9566 // used to compute the reduction result after the epilogue vector loop.
9567 auto AddFreezeForFindLastIVReductions = [](VPlan &Plan,
9568 bool UpdateResumePhis) {
9569 VPBuilder Builder(Plan.getEntry());
9570 for (VPRecipeBase &R : *Plan.getMiddleBlock()) {
9571 auto *VPI = dyn_cast<VPInstruction>(&R);
9572 if (!VPI || VPI->getOpcode() != VPInstruction::ComputeFindIVResult)
9573 continue;
9574 VPValue *OrigStart = VPI->getOperand(1);
9576 continue;
9577 VPInstruction *Freeze =
9578 Builder.createNaryOp(Instruction::Freeze, {OrigStart}, {}, "fr");
9579 VPI->setOperand(1, Freeze);
9580 if (UpdateResumePhis)
9581 OrigStart->replaceUsesWithIf(Freeze, [Freeze](VPUser &U, unsigned) {
9582 return Freeze != &U && isa<VPPhi>(&U);
9583 });
9584 }
9585 };
9586 AddFreezeForFindLastIVReductions(MainPlan, true);
9587 AddFreezeForFindLastIVReductions(EpiPlan, false);
9588
9589 VPBasicBlock *MainScalarPH = MainPlan.getScalarPreheader();
9590 VPValue *VectorTC = &MainPlan.getVectorTripCount();
9591 // If there is a suitable resume value for the canonical induction in the
9592 // scalar (which will become vector) epilogue loop, use it and move it to the
9593 // beginning of the scalar preheader. Otherwise create it below.
9594 auto ResumePhiIter =
9595 find_if(MainScalarPH->phis(), [VectorTC](VPRecipeBase &R) {
9596 return match(&R, m_VPInstruction<Instruction::PHI>(m_Specific(VectorTC),
9597 m_ZeroInt()));
9598 });
9599 VPPhi *ResumePhi = nullptr;
9600 if (ResumePhiIter == MainScalarPH->phis().end()) {
9601 VPBuilder ScalarPHBuilder(MainScalarPH, MainScalarPH->begin());
9602 ResumePhi = ScalarPHBuilder.createScalarPhi(
9603 {VectorTC, MainPlan.getCanonicalIV()->getStartValue()}, {},
9604 "vec.epilog.resume.val");
9605 } else {
9606 ResumePhi = cast<VPPhi>(&*ResumePhiIter);
9607 if (MainScalarPH->begin() == MainScalarPH->end())
9608 ResumePhi->moveBefore(*MainScalarPH, MainScalarPH->end());
9609 else if (&*MainScalarPH->begin() != ResumePhi)
9610 ResumePhi->moveBefore(*MainScalarPH, MainScalarPH->begin());
9611 }
9612 // Add a user to to make sure the resume phi won't get removed.
9613 VPBuilder(MainScalarPH)
9615}
9616
9617/// Prepare \p Plan for vectorizing the epilogue loop. That is, re-use expanded
9618/// SCEVs from \p ExpandedSCEVs and set resume values for header recipes.
9619static void
9621 const SCEV2ValueTy &ExpandedSCEVs,
9623 VPRegionBlock *VectorLoop = Plan.getVectorLoopRegion();
9624 VPBasicBlock *Header = VectorLoop->getEntryBasicBlock();
9625 Header->setName("vec.epilog.vector.body");
9626
9628 // Ensure that the start values for all header phi recipes are updated before
9629 // vectorizing the epilogue loop.
9630 for (VPRecipeBase &R : Header->phis()) {
9631 if (auto *IV = dyn_cast<VPCanonicalIVPHIRecipe>(&R)) {
9632 // When vectorizing the epilogue loop, the canonical induction start
9633 // value needs to be changed from zero to the value after the main
9634 // vector loop. Find the resume value created during execution of the main
9635 // VPlan. It must be the first phi in the loop preheader.
9636 // FIXME: Improve modeling for canonical IV start values in the epilogue
9637 // loop.
9638 using namespace llvm::PatternMatch;
9639 PHINode *EPResumeVal = &*L->getLoopPreheader()->phis().begin();
9640 for (Value *Inc : EPResumeVal->incoming_values()) {
9641 if (match(Inc, m_SpecificInt(0)))
9642 continue;
9643 assert(!EPI.VectorTripCount &&
9644 "Must only have a single non-zero incoming value");
9645 EPI.VectorTripCount = Inc;
9646 }
9647 // If we didn't find a non-zero vector trip count, all incoming values
9648 // must be zero, which also means the vector trip count is zero. Pick the
9649 // first zero as vector trip count.
9650 // TODO: We should not choose VF * UF so the main vector loop is known to
9651 // be dead.
9652 if (!EPI.VectorTripCount) {
9653 assert(
9654 EPResumeVal->getNumIncomingValues() > 0 &&
9655 all_of(EPResumeVal->incoming_values(),
9656 [](Value *Inc) { return match(Inc, m_SpecificInt(0)); }) &&
9657 "all incoming values must be 0");
9658 EPI.VectorTripCount = EPResumeVal->getOperand(0);
9659 }
9660 VPValue *VPV = Plan.getOrAddLiveIn(EPResumeVal);
9661 assert(all_of(IV->users(),
9662 [](const VPUser *U) {
9663 return isa<VPScalarIVStepsRecipe>(U) ||
9664 isa<VPDerivedIVRecipe>(U) ||
9665 cast<VPRecipeBase>(U)->isScalarCast() ||
9666 cast<VPInstruction>(U)->getOpcode() ==
9667 Instruction::Add;
9668 }) &&
9669 "the canonical IV should only be used by its increment or "
9670 "ScalarIVSteps when resetting the start value");
9671 IV->setOperand(0, VPV);
9672 continue;
9673 }
9674
9675 Value *ResumeV = nullptr;
9676 // TODO: Move setting of resume values to prepareToExecute.
9677 if (auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(&R)) {
9678 auto *RdxResult =
9679 cast<VPInstruction>(*find_if(ReductionPhi->users(), [](VPUser *U) {
9680 auto *VPI = dyn_cast<VPInstruction>(U);
9681 return VPI &&
9682 (VPI->getOpcode() == VPInstruction::ComputeAnyOfResult ||
9683 VPI->getOpcode() == VPInstruction::ComputeReductionResult ||
9684 VPI->getOpcode() == VPInstruction::ComputeFindIVResult);
9685 }));
9686 ResumeV = cast<PHINode>(ReductionPhi->getUnderlyingInstr())
9687 ->getIncomingValueForBlock(L->getLoopPreheader());
9688 RecurKind RK = ReductionPhi->getRecurrenceKind();
9690 Value *StartV = RdxResult->getOperand(1)->getLiveInIRValue();
9691 // VPReductionPHIRecipes for AnyOf reductions expect a boolean as
9692 // start value; compare the final value from the main vector loop
9693 // to the start value.
9694 BasicBlock *PBB = cast<Instruction>(ResumeV)->getParent();
9695 IRBuilder<> Builder(PBB, PBB->getFirstNonPHIIt());
9696 ResumeV = Builder.CreateICmpNE(ResumeV, StartV);
9698 Value *StartV = getStartValueFromReductionResult(RdxResult);
9699 ToFrozen[StartV] = cast<PHINode>(ResumeV)->getIncomingValueForBlock(
9701
9702 // VPReductionPHIRecipe for FindFirstIV/FindLastIV reductions requires
9703 // an adjustment to the resume value. The resume value is adjusted to
9704 // the sentinel value when the final value from the main vector loop
9705 // equals the start value. This ensures correctness when the start value
9706 // might not be less than the minimum value of a monotonically
9707 // increasing induction variable.
9708 BasicBlock *ResumeBB = cast<Instruction>(ResumeV)->getParent();
9709 IRBuilder<> Builder(ResumeBB, ResumeBB->getFirstNonPHIIt());
9710 Value *Cmp = Builder.CreateICmpEQ(ResumeV, ToFrozen[StartV]);
9711 Value *Sentinel = RdxResult->getOperand(2)->getLiveInIRValue();
9712 ResumeV = Builder.CreateSelect(Cmp, Sentinel, ResumeV);
9713 } else {
9714 VPValue *StartVal = Plan.getOrAddLiveIn(ResumeV);
9715 auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
9716 if (auto *VPI = dyn_cast<VPInstruction>(PhiR->getStartValue())) {
9717 assert(VPI->getOpcode() == VPInstruction::ReductionStartVector &&
9718 "unexpected start value");
9719 VPI->setOperand(0, StartVal);
9720 continue;
9721 }
9722 }
9723 } else {
9724 // Retrieve the induction resume values for wide inductions from
9725 // their original phi nodes in the scalar loop.
9726 PHINode *IndPhi = cast<VPWidenInductionRecipe>(&R)->getPHINode();
9727 // Hook up to the PHINode generated by a ResumePhi recipe of main
9728 // loop VPlan, which feeds the scalar loop.
9729 ResumeV = IndPhi->getIncomingValueForBlock(L->getLoopPreheader());
9730 }
9731 assert(ResumeV && "Must have a resume value");
9732 VPValue *StartVal = Plan.getOrAddLiveIn(ResumeV);
9733 cast<VPHeaderPHIRecipe>(&R)->setStartValue(StartVal);
9734 }
9735
9736 // For some VPValues in the epilogue plan we must re-use the generated IR
9737 // values from the main plan. Replace them with live-in VPValues.
9738 // TODO: This is a workaround needed for epilogue vectorization and it
9739 // should be removed once induction resume value creation is done
9740 // directly in VPlan.
9741 for (auto &R : make_early_inc_range(*Plan.getEntry())) {
9742 // Re-use frozen values from the main plan for Freeze VPInstructions in the
9743 // epilogue plan. This ensures all users use the same frozen value.
9744 auto *VPI = dyn_cast<VPInstruction>(&R);
9745 if (VPI && VPI->getOpcode() == Instruction::Freeze) {
9746 VPI->replaceAllUsesWith(Plan.getOrAddLiveIn(
9747 ToFrozen.lookup(VPI->getOperand(0)->getLiveInIRValue())));
9748 continue;
9749 }
9750
9751 // Re-use the trip count and steps expanded for the main loop, as
9752 // skeleton creation needs it as a value that dominates both the scalar
9753 // and vector epilogue loops
9754 auto *ExpandR = dyn_cast<VPExpandSCEVRecipe>(&R);
9755 if (!ExpandR)
9756 continue;
9757 VPValue *ExpandedVal =
9758 Plan.getOrAddLiveIn(ExpandedSCEVs.lookup(ExpandR->getSCEV()));
9759 ExpandR->replaceAllUsesWith(ExpandedVal);
9760 if (Plan.getTripCount() == ExpandR)
9761 Plan.resetTripCount(ExpandedVal);
9762 ExpandR->eraseFromParent();
9763 }
9764}
9765
9766// Generate bypass values from the additional bypass block. Note that when the
9767// vectorized epilogue is skipped due to iteration count check, then the
9768// resume value for the induction variable comes from the trip count of the
9769// main vector loop, passed as the second argument.
9771 PHINode *OrigPhi, const InductionDescriptor &II, IRBuilder<> &BypassBuilder,
9772 const SCEV2ValueTy &ExpandedSCEVs, Value *MainVectorTripCount,
9773 Instruction *OldInduction) {
9774 Value *Step = getExpandedStep(II, ExpandedSCEVs);
9775 // For the primary induction the additional bypass end value is known.
9776 // Otherwise it is computed.
9777 Value *EndValueFromAdditionalBypass = MainVectorTripCount;
9778 if (OrigPhi != OldInduction) {
9779 auto *BinOp = II.getInductionBinOp();
9780 // Fast-math-flags propagate from the original induction instruction.
9782 BypassBuilder.setFastMathFlags(BinOp->getFastMathFlags());
9783
9784 // Compute the end value for the additional bypass.
9785 EndValueFromAdditionalBypass =
9786 emitTransformedIndex(BypassBuilder, MainVectorTripCount,
9787 II.getStartValue(), Step, II.getKind(), BinOp);
9788 EndValueFromAdditionalBypass->setName("ind.end");
9789 }
9790 return EndValueFromAdditionalBypass;
9791}
9792
9794 VPlan &BestEpiPlan,
9796 const SCEV2ValueTy &ExpandedSCEVs,
9797 Value *MainVectorTripCount) {
9798 // Fix reduction resume values from the additional bypass block.
9799 BasicBlock *PH = L->getLoopPreheader();
9800 for (auto *Pred : predecessors(PH)) {
9801 for (PHINode &Phi : PH->phis()) {
9802 if (Phi.getBasicBlockIndex(Pred) != -1)
9803 continue;
9804 Phi.addIncoming(Phi.getIncomingValueForBlock(BypassBlock), Pred);
9805 }
9806 }
9807 auto *ScalarPH = cast<VPIRBasicBlock>(BestEpiPlan.getScalarPreheader());
9808 if (ScalarPH->hasPredecessors()) {
9809 // If ScalarPH has predecessors, we may need to update its reduction
9810 // resume values.
9811 for (const auto &[R, IRPhi] :
9812 zip(ScalarPH->phis(), ScalarPH->getIRBasicBlock()->phis())) {
9814 BypassBlock);
9815 }
9816 }
9817
9818 // Fix induction resume values from the additional bypass block.
9819 IRBuilder<> BypassBuilder(BypassBlock, BypassBlock->getFirstInsertionPt());
9820 for (const auto &[IVPhi, II] : LVL.getInductionVars()) {
9821 auto *Inc = cast<PHINode>(IVPhi->getIncomingValueForBlock(PH));
9823 IVPhi, II, BypassBuilder, ExpandedSCEVs, MainVectorTripCount,
9824 LVL.getPrimaryInduction());
9825 // TODO: Directly add as extra operand to the VPResumePHI recipe.
9826 Inc->setIncomingValueForBlock(BypassBlock, V);
9827 }
9828}
9829
9831 assert((EnableVPlanNativePath || L->isInnermost()) &&
9832 "VPlan-native path is not enabled. Only process inner loops.");
9833
9834 LLVM_DEBUG(dbgs() << "\nLV: Checking a loop in '"
9835 << L->getHeader()->getParent()->getName() << "' from "
9836 << L->getLocStr() << "\n");
9837
9838 LoopVectorizeHints Hints(L, InterleaveOnlyWhenForced, *ORE, TTI);
9839
9840 LLVM_DEBUG(
9841 dbgs() << "LV: Loop hints:"
9842 << " force="
9844 ? "disabled"
9846 ? "enabled"
9847 : "?"))
9848 << " width=" << Hints.getWidth()
9849 << " interleave=" << Hints.getInterleave() << "\n");
9850
9851 // Function containing loop
9852 Function *F = L->getHeader()->getParent();
9853
9854 // Looking at the diagnostic output is the only way to determine if a loop
9855 // was vectorized (other than looking at the IR or machine code), so it
9856 // is important to generate an optimization remark for each loop. Most of
9857 // these messages are generated as OptimizationRemarkAnalysis. Remarks
9858 // generated as OptimizationRemark and OptimizationRemarkMissed are
9859 // less verbose reporting vectorized loops and unvectorized loops that may
9860 // benefit from vectorization, respectively.
9861
9862 if (!Hints.allowVectorization(F, L, VectorizeOnlyWhenForced)) {
9863 LLVM_DEBUG(dbgs() << "LV: Loop hints prevent vectorization.\n");
9864 return false;
9865 }
9866
9867 PredicatedScalarEvolution PSE(*SE, *L);
9868
9869 // Check if it is legal to vectorize the loop.
9870 LoopVectorizationRequirements Requirements;
9871 LoopVectorizationLegality LVL(L, PSE, DT, TTI, TLI, F, *LAIs, LI, ORE,
9872 &Requirements, &Hints, DB, AC, BFI, PSI, AA);
9874 LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Cannot prove legality.\n");
9875 Hints.emitRemarkWithHints();
9876 return false;
9877 }
9878
9880 reportVectorizationFailure("Auto-vectorization of loops with uncountable "
9881 "early exit is not enabled",
9882 "UncountableEarlyExitLoopsDisabled", ORE, L);
9883 return false;
9884 }
9885
9886 if (!LVL.getPotentiallyFaultingLoads().empty()) {
9887 reportVectorizationFailure("Auto-vectorization of loops with potentially "
9888 "faulting load is not supported",
9889 "PotentiallyFaultingLoadsNotSupported", ORE, L);
9890 return false;
9891 }
9892
9893 // Entrance to the VPlan-native vectorization path. Outer loops are processed
9894 // here. They may require CFG and instruction level transformations before
9895 // even evaluating whether vectorization is profitable. Since we cannot modify
9896 // the incoming IR, we need to build VPlan upfront in the vectorization
9897 // pipeline.
9898 if (!L->isInnermost())
9899 return processLoopInVPlanNativePath(L, PSE, LI, DT, &LVL, TTI, TLI, DB, AC,
9900 ORE, BFI, PSI, Hints, Requirements);
9901
9902 assert(L->isInnermost() && "Inner loop expected.");
9903
9904 InterleavedAccessInfo IAI(PSE, L, DT, LI, LVL.getLAI());
9905 bool UseInterleaved = TTI->enableInterleavedAccessVectorization();
9906
9907 // If an override option has been passed in for interleaved accesses, use it.
9908 if (EnableInterleavedMemAccesses.getNumOccurrences() > 0)
9909 UseInterleaved = EnableInterleavedMemAccesses;
9910
9911 // Analyze interleaved memory accesses.
9912 if (UseInterleaved)
9914
9915 if (LVL.hasUncountableEarlyExit()) {
9916 BasicBlock *LoopLatch = L->getLoopLatch();
9917 if (IAI.requiresScalarEpilogue() ||
9919 [LoopLatch](BasicBlock *BB) { return BB != LoopLatch; })) {
9920 reportVectorizationFailure("Auto-vectorization of early exit loops "
9921 "requiring a scalar epilogue is unsupported",
9922 "UncountableEarlyExitUnsupported", ORE, L);
9923 return false;
9924 }
9925 }
9926
9927 // Check the function attributes and profiles to find out if this function
9928 // should be optimized for size.
9930 getScalarEpilogueLowering(F, L, Hints, PSI, BFI, TTI, TLI, LVL, &IAI);
9931
9932 // Check the loop for a trip count threshold: vectorize loops with a tiny trip
9933 // count by optimizing for size, to minimize overheads.
9934 auto ExpectedTC = getSmallBestKnownTC(PSE, L);
9935 if (ExpectedTC && ExpectedTC->isFixed() &&
9936 ExpectedTC->getFixedValue() < TinyTripCountVectorThreshold) {
9937 LLVM_DEBUG(dbgs() << "LV: Found a loop with a very small trip count. "
9938 << "This loop is worth vectorizing only if no scalar "
9939 << "iteration overheads are incurred.");
9941 LLVM_DEBUG(dbgs() << " But vectorizing was explicitly forced.\n");
9942 else {
9943 LLVM_DEBUG(dbgs() << "\n");
9944 // Predicate tail-folded loops are efficient even when the loop
9945 // iteration count is low. However, setting the epilogue policy to
9946 // `CM_ScalarEpilogueNotAllowedLowTripLoop` prevents vectorizing loops
9947 // with runtime checks. It's more effective to let
9948 // `isOutsideLoopWorkProfitable` determine if vectorization is
9949 // beneficial for the loop.
9952 }
9953 }
9954
9955 // Check the function attributes to see if implicit floats or vectors are
9956 // allowed.
9957 if (F->hasFnAttribute(Attribute::NoImplicitFloat)) {
9959 "Can't vectorize when the NoImplicitFloat attribute is used",
9960 "loop not vectorized due to NoImplicitFloat attribute",
9961 "NoImplicitFloat", ORE, L);
9962 Hints.emitRemarkWithHints();
9963 return false;
9964 }
9965
9966 // Check if the target supports potentially unsafe FP vectorization.
9967 // FIXME: Add a check for the type of safety issue (denormal, signaling)
9968 // for the target we're vectorizing for, to make sure none of the
9969 // additional fp-math flags can help.
9970 if (Hints.isPotentiallyUnsafe() &&
9971 TTI->isFPVectorizationPotentiallyUnsafe()) {
9973 "Potentially unsafe FP op prevents vectorization",
9974 "loop not vectorized due to unsafe FP support.",
9975 "UnsafeFP", ORE, L);
9976 Hints.emitRemarkWithHints();
9977 return false;
9978 }
9979
9980 bool AllowOrderedReductions;
9981 // If the flag is set, use that instead and override the TTI behaviour.
9982 if (ForceOrderedReductions.getNumOccurrences() > 0)
9983 AllowOrderedReductions = ForceOrderedReductions;
9984 else
9985 AllowOrderedReductions = TTI->enableOrderedReductions();
9986 if (!LVL.canVectorizeFPMath(AllowOrderedReductions)) {
9987 ORE->emit([&]() {
9988 auto *ExactFPMathInst = Requirements.getExactFPInst();
9989 return OptimizationRemarkAnalysisFPCommute(DEBUG_TYPE, "CantReorderFPOps",
9990 ExactFPMathInst->getDebugLoc(),
9991 ExactFPMathInst->getParent())
9992 << "loop not vectorized: cannot prove it is safe to reorder "
9993 "floating-point operations";
9994 });
9995 LLVM_DEBUG(dbgs() << "LV: loop not vectorized: cannot prove it is safe to "
9996 "reorder floating-point operations\n");
9997 Hints.emitRemarkWithHints();
9998 return false;
9999 }
10000
10001 // Use the cost model.
10002 LoopVectorizationCostModel CM(SEL, L, PSE, LI, &LVL, *TTI, TLI, DB, AC, ORE,
10003 F, &Hints, IAI, PSI, BFI);
10004 // Use the planner for vectorization.
10005 LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, &LVL, CM, IAI, PSE, Hints,
10006 ORE);
10007
10008 // Get user vectorization factor and interleave count.
10009 ElementCount UserVF = Hints.getWidth();
10010 unsigned UserIC = Hints.getInterleave();
10011
10012 // Plan how to best vectorize.
10013 LVP.plan(UserVF, UserIC);
10015 unsigned IC = 1;
10016
10017 if (ORE->allowExtraAnalysis(LV_NAME))
10019
10020 GeneratedRTChecks Checks(PSE, DT, LI, TTI, F->getDataLayout(), CM.CostKind);
10021 if (LVP.hasPlanWithVF(VF.Width)) {
10022 // Select the interleave count.
10023 IC = LVP.selectInterleaveCount(LVP.getPlanFor(VF.Width), VF.Width, VF.Cost);
10024
10025 unsigned SelectedIC = std::max(IC, UserIC);
10026 // Optimistically generate runtime checks if they are needed. Drop them if
10027 // they turn out to not be profitable.
10028 if (VF.Width.isVector() || SelectedIC > 1) {
10029 Checks.create(L, *LVL.getLAI(), PSE.getPredicate(), VF.Width, SelectedIC);
10030
10031 // Bail out early if either the SCEV or memory runtime checks are known to
10032 // fail. In that case, the vector loop would never execute.
10033 using namespace llvm::PatternMatch;
10034 if (Checks.getSCEVChecks().first &&
10035 match(Checks.getSCEVChecks().first, m_One()))
10036 return false;
10037 if (Checks.getMemRuntimeChecks().first &&
10038 match(Checks.getMemRuntimeChecks().first, m_One()))
10039 return false;
10040 }
10041
10042 // Check if it is profitable to vectorize with runtime checks.
10043 bool ForceVectorization =
10045 VPCostContext CostCtx(CM.TTI, *CM.TLI, LVP.getPlanFor(VF.Width), CM,
10046 CM.CostKind);
10047 if (!ForceVectorization &&
10048 !isOutsideLoopWorkProfitable(Checks, VF, L, PSE, CostCtx,
10049 LVP.getPlanFor(VF.Width), SEL,
10050 CM.getVScaleForTuning())) {
10051 ORE->emit([&]() {
10053 DEBUG_TYPE, "CantReorderMemOps", L->getStartLoc(),
10054 L->getHeader())
10055 << "loop not vectorized: cannot prove it is safe to reorder "
10056 "memory operations";
10057 });
10058 LLVM_DEBUG(dbgs() << "LV: Too many memory checks needed.\n");
10059 Hints.emitRemarkWithHints();
10060 return false;
10061 }
10062 }
10063
10064 // Identify the diagnostic messages that should be produced.
10065 std::pair<StringRef, std::string> VecDiagMsg, IntDiagMsg;
10066 bool VectorizeLoop = true, InterleaveLoop = true;
10067 if (VF.Width.isScalar()) {
10068 LLVM_DEBUG(dbgs() << "LV: Vectorization is possible but not beneficial.\n");
10069 VecDiagMsg = {
10070 "VectorizationNotBeneficial",
10071 "the cost-model indicates that vectorization is not beneficial"};
10072 VectorizeLoop = false;
10073 }
10074
10075 if (!LVP.hasPlanWithVF(VF.Width) && UserIC > 1) {
10076 // Tell the user interleaving was avoided up-front, despite being explicitly
10077 // requested.
10078 LLVM_DEBUG(dbgs() << "LV: Ignoring UserIC, because vectorization and "
10079 "interleaving should be avoided up front\n");
10080 IntDiagMsg = {"InterleavingAvoided",
10081 "Ignoring UserIC, because interleaving was avoided up front"};
10082 InterleaveLoop = false;
10083 } else if (IC == 1 && UserIC <= 1) {
10084 // Tell the user interleaving is not beneficial.
10085 LLVM_DEBUG(dbgs() << "LV: Interleaving is not beneficial.\n");
10086 IntDiagMsg = {
10087 "InterleavingNotBeneficial",
10088 "the cost-model indicates that interleaving is not beneficial"};
10089 InterleaveLoop = false;
10090 if (UserIC == 1) {
10091 IntDiagMsg.first = "InterleavingNotBeneficialAndDisabled";
10092 IntDiagMsg.second +=
10093 " and is explicitly disabled or interleave count is set to 1";
10094 }
10095 } else if (IC > 1 && UserIC == 1) {
10096 // Tell the user interleaving is beneficial, but it explicitly disabled.
10097 LLVM_DEBUG(dbgs() << "LV: Interleaving is beneficial but is explicitly "
10098 "disabled.\n");
10099 IntDiagMsg = {"InterleavingBeneficialButDisabled",
10100 "the cost-model indicates that interleaving is beneficial "
10101 "but is explicitly disabled or interleave count is set to 1"};
10102 InterleaveLoop = false;
10103 }
10104
10105 // If there is a histogram in the loop, do not just interleave without
10106 // vectorizing. The order of operations will be incorrect without the
10107 // histogram intrinsics, which are only used for recipes with VF > 1.
10108 if (!VectorizeLoop && InterleaveLoop && LVL.hasHistograms()) {
10109 LLVM_DEBUG(dbgs() << "LV: Not interleaving without vectorization due "
10110 << "to histogram operations.\n");
10111 IntDiagMsg = {
10112 "HistogramPreventsScalarInterleaving",
10113 "Unable to interleave without vectorization due to constraints on "
10114 "the order of histogram operations"};
10115 InterleaveLoop = false;
10116 }
10117
10118 // Override IC if user provided an interleave count.
10119 IC = UserIC > 0 ? UserIC : IC;
10120
10121 // Emit diagnostic messages, if any.
10122 const char *VAPassName = Hints.vectorizeAnalysisPassName();
10123 if (!VectorizeLoop && !InterleaveLoop) {
10124 // Do not vectorize or interleaving the loop.
10125 ORE->emit([&]() {
10126 return OptimizationRemarkMissed(VAPassName, VecDiagMsg.first,
10127 L->getStartLoc(), L->getHeader())
10128 << VecDiagMsg.second;
10129 });
10130 ORE->emit([&]() {
10131 return OptimizationRemarkMissed(LV_NAME, IntDiagMsg.first,
10132 L->getStartLoc(), L->getHeader())
10133 << IntDiagMsg.second;
10134 });
10135 return false;
10136 }
10137
10138 if (!VectorizeLoop && InterleaveLoop) {
10139 LLVM_DEBUG(dbgs() << "LV: Interleave Count is " << IC << '\n');
10140 ORE->emit([&]() {
10141 return OptimizationRemarkAnalysis(VAPassName, VecDiagMsg.first,
10142 L->getStartLoc(), L->getHeader())
10143 << VecDiagMsg.second;
10144 });
10145 } else if (VectorizeLoop && !InterleaveLoop) {
10146 LLVM_DEBUG(dbgs() << "LV: Found a vectorizable loop (" << VF.Width
10147 << ") in " << L->getLocStr() << '\n');
10148 ORE->emit([&]() {
10149 return OptimizationRemarkAnalysis(LV_NAME, IntDiagMsg.first,
10150 L->getStartLoc(), L->getHeader())
10151 << IntDiagMsg.second;
10152 });
10153 } else if (VectorizeLoop && InterleaveLoop) {
10154 LLVM_DEBUG(dbgs() << "LV: Found a vectorizable loop (" << VF.Width
10155 << ") in " << L->getLocStr() << '\n');
10156 LLVM_DEBUG(dbgs() << "LV: Interleave Count is " << IC << '\n');
10157 }
10158
10159 // Report the vectorization decision.
10160 if (VF.Width.isScalar()) {
10161 using namespace ore;
10162 assert(IC > 1);
10163 ORE->emit([&]() {
10164 return OptimizationRemark(LV_NAME, "Interleaved", L->getStartLoc(),
10165 L->getHeader())
10166 << "interleaved loop (interleaved count: "
10167 << NV("InterleaveCount", IC) << ")";
10168 });
10169 } else {
10170 // Report the vectorization decision.
10171 reportVectorization(ORE, L, VF, IC);
10172 }
10173 if (ORE->allowExtraAnalysis(LV_NAME))
10175
10176 // If we decided that it is *legal* to interleave or vectorize the loop, then
10177 // do it.
10178
10179 VPlan &BestPlan = LVP.getPlanFor(VF.Width);
10180 // Consider vectorizing the epilogue too if it's profitable.
10181 VectorizationFactor EpilogueVF =
10183 if (EpilogueVF.Width.isVector()) {
10184 std::unique_ptr<VPlan> BestMainPlan(BestPlan.duplicate());
10185
10186 // The first pass vectorizes the main loop and creates a scalar epilogue
10187 // to be vectorized by executing the plan (potentially with a different
10188 // factor) again shortly afterwards.
10189 VPlan &BestEpiPlan = LVP.getPlanFor(EpilogueVF.Width);
10190 BestEpiPlan.getMiddleBlock()->setName("vec.epilog.middle.block");
10191 preparePlanForMainVectorLoop(*BestMainPlan, BestEpiPlan);
10192 EpilogueLoopVectorizationInfo EPI(VF.Width, IC, EpilogueVF.Width, 1,
10193 BestEpiPlan);
10194 EpilogueVectorizerMainLoop MainILV(L, PSE, LI, DT, TTI, AC, EPI, &CM, BFI,
10195 PSI, Checks, *BestMainPlan);
10196 auto ExpandedSCEVs = LVP.executePlan(EPI.MainLoopVF, EPI.MainLoopUF,
10197 *BestMainPlan, MainILV, DT, false);
10198 ++LoopsVectorized;
10199
10200 // Second pass vectorizes the epilogue and adjusts the control flow
10201 // edges from the first pass.
10202 EpilogueVectorizerEpilogueLoop EpilogILV(L, PSE, LI, DT, TTI, AC, EPI, &CM,
10203 BFI, PSI, Checks, BestEpiPlan);
10204 EpilogILV.setTripCount(MainILV.getTripCount());
10205 preparePlanForEpilogueVectorLoop(BestEpiPlan, L, ExpandedSCEVs, EPI);
10206
10207 LVP.executePlan(EPI.EpilogueVF, EPI.EpilogueUF, BestEpiPlan, EpilogILV, DT,
10208 true);
10209
10211 BestEpiPlan, LVL, ExpandedSCEVs,
10212 EPI.VectorTripCount);
10213 ++LoopsEpilogueVectorized;
10214 } else {
10215 InnerLoopVectorizer LB(L, PSE, LI, DT, TTI, AC, VF.Width, IC, &CM, BFI, PSI,
10216 Checks, BestPlan);
10217 // TODO: Move to general VPlan pipeline once epilogue loops are also
10218 // supported.
10221 IC, PSE);
10222 LVP.addMinimumIterationCheck(BestPlan, VF.Width, IC,
10224
10225 LVP.executePlan(VF.Width, IC, BestPlan, LB, DT, false);
10226 ++LoopsVectorized;
10227 }
10228
10229 assert(DT->verify(DominatorTree::VerificationLevel::Fast) &&
10230 "DT not preserved correctly");
10231 assert(!verifyFunction(*F, &dbgs()));
10232
10233 return true;
10234}
10235
10237
10238 // Don't attempt if
10239 // 1. the target claims to have no vector registers, and
10240 // 2. interleaving won't help ILP.
10241 //
10242 // The second condition is necessary because, even if the target has no
10243 // vector registers, loop vectorization may still enable scalar
10244 // interleaving.
10245 if (!TTI->getNumberOfRegisters(TTI->getRegisterClassForType(true)) &&
10246 TTI->getMaxInterleaveFactor(ElementCount::getFixed(1)) < 2)
10247 return LoopVectorizeResult(false, false);
10248
10249 bool Changed = false, CFGChanged = false;
10250
10251 // The vectorizer requires loops to be in simplified form.
10252 // Since simplification may add new inner loops, it has to run before the
10253 // legality and profitability checks. This means running the loop vectorizer
10254 // will simplify all loops, regardless of whether anything end up being
10255 // vectorized.
10256 for (const auto &L : *LI)
10257 Changed |= CFGChanged |=
10258 simplifyLoop(L, DT, LI, SE, AC, nullptr, false /* PreserveLCSSA */);
10259
10260 // Build up a worklist of inner-loops to vectorize. This is necessary as
10261 // the act of vectorizing or partially unrolling a loop creates new loops
10262 // and can invalidate iterators across the loops.
10263 SmallVector<Loop *, 8> Worklist;
10264
10265 for (Loop *L : *LI)
10266 collectSupportedLoops(*L, LI, ORE, Worklist);
10267
10268 LoopsAnalyzed += Worklist.size();
10269
10270 // Now walk the identified inner loops.
10271 while (!Worklist.empty()) {
10272 Loop *L = Worklist.pop_back_val();
10273
10274 // For the inner loops we actually process, form LCSSA to simplify the
10275 // transform.
10276 Changed |= formLCSSARecursively(*L, *DT, LI, SE);
10277
10278 Changed |= CFGChanged |= processLoop(L);
10279
10280 if (Changed) {
10281 LAIs->clear();
10282
10283#ifndef NDEBUG
10284 if (VerifySCEV)
10285 SE->verify();
10286#endif
10287 }
10288 }
10289
10290 // Process each loop nest in the function.
10291 return LoopVectorizeResult(Changed, CFGChanged);
10292}
10293
10296 LI = &AM.getResult<LoopAnalysis>(F);
10297 // There are no loops in the function. Return before computing other
10298 // expensive analyses.
10299 if (LI->empty())
10300 return PreservedAnalyses::all();
10309 AA = &AM.getResult<AAManager>(F);
10310
10311 auto &MAMProxy = AM.getResult<ModuleAnalysisManagerFunctionProxy>(F);
10312 PSI = MAMProxy.getCachedResult<ProfileSummaryAnalysis>(*F.getParent());
10313 BFI = nullptr;
10314 if (PSI && PSI->hasProfileSummary())
10316 LoopVectorizeResult Result = runImpl(F);
10317 if (!Result.MadeAnyChange)
10318 return PreservedAnalyses::all();
10320
10321 if (isAssignmentTrackingEnabled(*F.getParent())) {
10322 for (auto &BB : F)
10324 }
10325
10326 PA.preserve<LoopAnalysis>();
10330
10331 if (Result.MadeCFGChange) {
10332 // Making CFG changes likely means a loop got vectorized. Indicate that
10333 // extra simplification passes should be run.
10334 // TODO: MadeCFGChanges is not a prefect proxy. Extra passes should only
10335 // be run if runtime checks have been added.
10338 } else {
10340 }
10341 return PA;
10342}
10343
10345 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
10346 static_cast<PassInfoMixin<LoopVectorizePass> *>(this)->printPipeline(
10347 OS, MapClassName2PassName);
10348
10349 OS << '<';
10350 OS << (InterleaveOnlyWhenForced ? "" : "no-") << "interleave-forced-only;";
10351 OS << (VectorizeOnlyWhenForced ? "" : "no-") << "vectorize-forced-only;";
10352 OS << '>';
10353}
static unsigned getIntrinsicID(const SDNode *N)
unsigned RegSize
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
aarch64 promote const
AMDGPU Lower Kernel Arguments
AMDGPU Register Bank Select
Rewrite undef for PHI
This file implements a class to represent arbitrary precision integral constant values and operations...
@ PostInc
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static bool isEqual(const Function &Caller, const Function &Callee)
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
This is the interface for LLVM's primary stateless and local alias analysis.
static bool IsEmptyBlock(MachineBasicBlock *MBB)
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static cl::opt< OutputCostKind > CostKind("cost-kind", cl::desc("Target cost kind"), cl::init(OutputCostKind::RecipThroughput), cl::values(clEnumValN(OutputCostKind::RecipThroughput, "throughput", "Reciprocal throughput"), clEnumValN(OutputCostKind::Latency, "latency", "Instruction latency"), clEnumValN(OutputCostKind::CodeSize, "code-size", "Code size"), clEnumValN(OutputCostKind::SizeAndLatency, "size-latency", "Code size and latency"), clEnumValN(OutputCostKind::All, "all", "Print all cost kinds")))
static cl::opt< IntrinsicCostStrategy > IntrinsicCost("intrinsic-cost-strategy", cl::desc("Costing strategy for intrinsic instructions"), cl::init(IntrinsicCostStrategy::InstructionCost), cl::values(clEnumValN(IntrinsicCostStrategy::InstructionCost, "instruction-cost", "Use TargetTransformInfo::getInstructionCost"), clEnumValN(IntrinsicCostStrategy::IntrinsicCost, "intrinsic-cost", "Use TargetTransformInfo::getIntrinsicInstrCost"), clEnumValN(IntrinsicCostStrategy::TypeBasedIntrinsicCost, "type-based-intrinsic-cost", "Calculate the intrinsic cost based only on argument types")))
static InstructionCost getCost(Instruction &Inst, TTI::TargetCostKind CostKind, TargetTransformInfo &TTI, TargetLibraryInfo &TLI)
Definition CostModel.cpp:74
This file defines DenseMapInfo traits for DenseMap.
This file defines the DenseMap class.
#define DEBUG_TYPE
This is the interface for a simple mod/ref and alias analysis over globals.
Hexagon Common GEP
#define _
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
Module.h This file contains the declarations for the Module class.
This defines the Use class.
static bool hasNoUnsignedWrap(BinaryOperator &I)
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
static std::pair< Value *, APInt > getMask(Value *WideMask, unsigned Factor, ElementCount LeafValueEC)
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Legalize the Machine IR a function s Machine IR
Definition Legalizer.cpp:80
static cl::opt< unsigned, true > VectorizationFactor("force-vector-width", cl::Hidden, cl::desc("Sets the SIMD width. Zero is autoselect."), cl::location(VectorizerParams::VectorizationFactor))
This header provides classes for managing per-loop analyses.
static const char * VerboseDebug
#define LV_NAME
This file defines the LoopVectorizationLegality class.
This file provides a LoopVectorizationPlanner class.
static void collectSupportedLoops(Loop &L, LoopInfo *LI, OptimizationRemarkEmitter *ORE, SmallVectorImpl< Loop * > &V)
static cl::opt< unsigned > EpilogueVectorizationMinVF("epilogue-vectorization-minimum-VF", cl::Hidden, cl::desc("Only loops with vectorization factor equal to or larger than " "the specified value are considered for epilogue vectorization."))
static cl::opt< unsigned > EpilogueVectorizationForceVF("epilogue-vectorization-force-VF", cl::init(1), cl::Hidden, cl::desc("When epilogue vectorization is enabled, and a value greater than " "1 is specified, forces the given VF for all applicable epilogue " "loops."))
static void addScalarResumePhis(VPRecipeBuilder &Builder, VPlan &Plan, DenseMap< VPValue *, VPValue * > &IVEndValues)
Create resume phis in the scalar preheader for first-order recurrences, reductions and inductions,...
static Type * maybeVectorizeType(Type *Ty, ElementCount VF)
static ElementCount determineVPlanVF(const TargetTransformInfo &TTI, LoopVectorizationCostModel &CM)
static ElementCount getSmallConstantTripCount(ScalarEvolution *SE, const Loop *L)
A version of ScalarEvolution::getSmallConstantTripCount that returns an ElementCount to include loops...
static cl::opt< unsigned > VectorizeMemoryCheckThreshold("vectorize-memory-check-threshold", cl::init(128), cl::Hidden, cl::desc("The maximum allowed number of runtime memory checks"))
static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan)
Prepare MainPlan for vectorizing the main vector loop during epilogue vectorization.
static cl::opt< unsigned > TinyTripCountVectorThreshold("vectorizer-min-trip-count", cl::init(16), cl::Hidden, cl::desc("Loops with a constant trip count that is smaller than this " "value are vectorized only if no scalar iteration overheads " "are incurred."))
Loops with a known constant trip count below this number are vectorized only if no scalar iteration o...
static void debugVectorizationMessage(const StringRef Prefix, const StringRef DebugMsg, Instruction *I)
Write a DebugMsg about vectorization to the debug output stream.
static cl::opt< bool > EnableCondStoresVectorization("enable-cond-stores-vec", cl::init(true), cl::Hidden, cl::desc("Enable if predication of stores during vectorization."))
static VPIRBasicBlock * replaceVPBBWithIRVPBB(VPBasicBlock *VPBB, BasicBlock *IRBB, VPlan *Plan=nullptr)
Replace VPBB with a VPIRBasicBlock wrapping IRBB.
static VPInstruction * addResumePhiRecipeForInduction(VPWidenInductionRecipe *WideIV, VPBuilder &VectorPHBuilder, VPBuilder &ScalarPHBuilder, VPTypeAnalysis &TypeInfo, VPValue *VectorTC)
Create and return a ResumePhi for WideIV, unless it is truncated.
static Value * emitTransformedIndex(IRBuilderBase &B, Value *Index, Value *StartValue, Value *Step, InductionDescriptor::InductionKind InductionKind, const BinaryOperator *InductionBinOp)
Compute the transformed value of Index at offset StartValue using step StepValue.
static DebugLoc getDebugLocFromInstOrOperands(Instruction *I)
Look for a meaningful debug location on the instruction or its operands.
static Value * createInductionAdditionalBypassValues(PHINode *OrigPhi, const InductionDescriptor &II, IRBuilder<> &BypassBuilder, const SCEV2ValueTy &ExpandedSCEVs, Value *MainVectorTripCount, Instruction *OldInduction)
static void fixReductionScalarResumeWhenVectorizingEpilog(VPPhi *EpiResumePhiR, PHINode &EpiResumePhi, BasicBlock *BypassBlock)
static Value * getStartValueFromReductionResult(VPInstruction *RdxResult)
static cl::opt< bool > ForceTargetSupportsScalableVectors("force-target-supports-scalable-vectors", cl::init(false), cl::Hidden, cl::desc("Pretend that scalable vectors are supported, even if the target does " "not support them. This flag should only be used for testing."))
static bool useActiveLaneMaskForControlFlow(TailFoldingStyle Style)
static cl::opt< bool > EnableEarlyExitVectorization("enable-early-exit-vectorization", cl::init(true), cl::Hidden, cl::desc("Enable vectorization of early exit loops with uncountable exits."))
static cl::opt< bool > ConsiderRegPressure("vectorizer-consider-reg-pressure", cl::init(false), cl::Hidden, cl::desc("Discard VFs if their register pressure is too high."))
static unsigned estimateElementCount(ElementCount VF, std::optional< unsigned > VScale)
This function attempts to return a value that represents the ElementCount at runtime.
static constexpr uint32_t MinItersBypassWeights[]
static cl::opt< unsigned > ForceTargetNumScalarRegs("force-target-num-scalar-regs", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's number of scalar registers."))
static cl::opt< bool > UseWiderVFIfCallVariantsPresent("vectorizer-maximize-bandwidth-for-vector-calls", cl::init(true), cl::Hidden, cl::desc("Try wider VFs if they enable the use of vector variants"))
static std::optional< unsigned > getMaxVScale(const Function &F, const TargetTransformInfo &TTI)
static cl::opt< unsigned > SmallLoopCost("small-loop-cost", cl::init(20), cl::Hidden, cl::desc("The cost of a loop that is considered 'small' by the interleaver."))
static bool planContainsAdditionalSimplifications(VPlan &Plan, VPCostContext &CostCtx, Loop *TheLoop, ElementCount VF)
Return true if the original loop \ TheLoop contains any instructions that do not have corresponding r...
static cl::opt< unsigned > ForceTargetNumVectorRegs("force-target-num-vector-regs", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's number of vector registers."))
static bool isExplicitVecOuterLoop(Loop *OuterLp, OptimizationRemarkEmitter *ORE)
static cl::opt< bool > EnableIndVarRegisterHeur("enable-ind-var-reg-heur", cl::init(true), cl::Hidden, cl::desc("Count the induction variable only once when interleaving"))
static cl::opt< TailFoldingStyle > ForceTailFoldingStyle("force-tail-folding-style", cl::desc("Force the tail folding style"), cl::init(TailFoldingStyle::None), cl::values(clEnumValN(TailFoldingStyle::None, "none", "Disable tail folding"), clEnumValN(TailFoldingStyle::Data, "data", "Create lane mask for data only, using active.lane.mask intrinsic"), clEnumValN(TailFoldingStyle::DataWithoutLaneMask, "data-without-lane-mask", "Create lane mask with compare/stepvector"), clEnumValN(TailFoldingStyle::DataAndControlFlow, "data-and-control", "Create lane mask using active.lane.mask intrinsic, and use " "it for both data and control flow"), clEnumValN(TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck, "data-and-control-without-rt-check", "Similar to data-and-control, but remove the runtime check"), clEnumValN(TailFoldingStyle::DataWithEVL, "data-with-evl", "Use predicated EVL instructions for tail folding. If EVL " "is unsupported, fallback to data-without-lane-mask.")))
static cl::opt< bool > EnableEpilogueVectorization("enable-epilogue-vectorization", cl::init(true), cl::Hidden, cl::desc("Enable vectorization of epilogue loops."))
static ScalarEpilogueLowering getScalarEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI, TargetTransformInfo *TTI, TargetLibraryInfo *TLI, LoopVectorizationLegality &LVL, InterleavedAccessInfo *IAI)
static void preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L, const SCEV2ValueTy &ExpandedSCEVs, EpilogueLoopVectorizationInfo &EPI)
Prepare Plan for vectorizing the epilogue loop.
static cl::opt< bool > PreferPredicatedReductionSelect("prefer-predicated-reduction-select", cl::init(false), cl::Hidden, cl::desc("Prefer predicating a reduction operation over an after loop select."))
static VPWidenIntOrFpInductionRecipe * createWidenInductionRecipes(PHINode *Phi, Instruction *PhiOrTrunc, VPValue *Start, const InductionDescriptor &IndDesc, VPlan &Plan, ScalarEvolution &SE, Loop &OrigLoop)
Creates a VPWidenIntOrFpInductionRecpipe for Phi.
static cl::opt< bool > PreferInLoopReductions("prefer-inloop-reductions", cl::init(false), cl::Hidden, cl::desc("Prefer in-loop vector reductions, " "overriding the targets preference."))
static cl::opt< bool > EnableLoadStoreRuntimeInterleave("enable-loadstore-runtime-interleave", cl::init(true), cl::Hidden, cl::desc("Enable runtime interleaving until load/store ports are saturated"))
static cl::opt< bool > VPlanBuildStressTest("vplan-build-stress-test", cl::init(false), cl::Hidden, cl::desc("Build VPlan for every supported loop nest in the function and bail " "out right after the build (stress test the VPlan H-CFG construction " "in the VPlan-native vectorization path)."))
static bool hasIrregularType(Type *Ty, const DataLayout &DL)
A helper function that returns true if the given type is irregular.
static cl::opt< bool > LoopVectorizeWithBlockFrequency("loop-vectorize-with-block-frequency", cl::init(true), cl::Hidden, cl::desc("Enable the use of the block frequency analysis to access PGO " "heuristics minimizing code growth in cold regions and being more " "aggressive in hot regions."))
static std::optional< ElementCount > getSmallBestKnownTC(PredicatedScalarEvolution &PSE, Loop *L, bool CanUseConstantMax=true)
Returns "best known" trip count, which is either a valid positive trip count or std::nullopt when an ...
static Value * getExpandedStep(const InductionDescriptor &ID, const SCEV2ValueTy &ExpandedSCEVs)
Return the expanded step for ID using ExpandedSCEVs to look up SCEV expansion results.
static bool useActiveLaneMask(TailFoldingStyle Style)
static bool hasReplicatorRegion(VPlan &Plan)
static bool isIndvarOverflowCheckKnownFalse(const LoopVectorizationCostModel *Cost, ElementCount VF, std::optional< unsigned > UF=std::nullopt)
For the given VF and UF and maximum trip count computed for the loop, return whether the induction va...
static void addFullyUnrolledInstructionsToIgnore(Loop *L, const LoopVectorizationLegality::InductionList &IL, SmallPtrSetImpl< Instruction * > &InstsToIgnore)
Knowing that loop L executes a single vector iteration, add instructions that will get simplified and...
static cl::opt< PreferPredicateTy::Option > PreferPredicateOverEpilogue("prefer-predicate-over-epilogue", cl::init(PreferPredicateTy::ScalarEpilogue), cl::Hidden, cl::desc("Tail-folding and predication preferences over creating a scalar " "epilogue loop."), cl::values(clEnumValN(PreferPredicateTy::ScalarEpilogue, "scalar-epilogue", "Don't tail-predicate loops, create scalar epilogue"), clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue, "predicate-else-scalar-epilogue", "prefer tail-folding, create scalar epilogue if tail " "folding fails."), clEnumValN(PreferPredicateTy::PredicateOrDontVectorize, "predicate-dont-vectorize", "prefers tail-folding, don't attempt vectorization if " "tail-folding fails.")))
static cl::opt< bool > EnableInterleavedMemAccesses("enable-interleaved-mem-accesses", cl::init(false), cl::Hidden, cl::desc("Enable vectorization on interleaved memory accesses in a loop"))
static cl::opt< bool > EnableMaskedInterleavedMemAccesses("enable-masked-interleaved-mem-accesses", cl::init(false), cl::Hidden, cl::desc("Enable vectorization on masked interleaved memory accesses in a loop"))
An interleave-group may need masking if it resides in a block that needs predication,...
static cl::opt< bool > ForceOrderedReductions("force-ordered-reductions", cl::init(false), cl::Hidden, cl::desc("Enable the vectorisation of loops with in-order (strict) " "FP reductions"))
static void cse(BasicBlock *BB)
Perform cse of induction variable instructions.
static const SCEV * getAddressAccessSCEV(Value *Ptr, LoopVectorizationLegality *Legal, PredicatedScalarEvolution &PSE, const Loop *TheLoop)
Gets Address Access SCEV after verifying that the access pattern is loop invariant except the inducti...
static cl::opt< cl::boolOrDefault > ForceSafeDivisor("force-widen-divrem-via-safe-divisor", cl::Hidden, cl::desc("Override cost based safe divisor widening for div/rem instructions"))
static InstructionCost calculateEarlyExitCost(VPCostContext &CostCtx, VPlan &Plan, ElementCount VF)
For loops with uncountable early exits, find the cost of doing work when exiting the loop early,...
static cl::opt< unsigned > ForceTargetMaxVectorInterleaveFactor("force-target-max-vector-interleave", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's max interleave factor for " "vectorized loops."))
static bool processLoopInVPlanNativePath(Loop *L, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, LoopVectorizationLegality *LVL, TargetTransformInfo *TTI, TargetLibraryInfo *TLI, DemandedBits *DB, AssumptionCache *AC, OptimizationRemarkEmitter *ORE, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, LoopVectorizeHints &Hints, LoopVectorizationRequirements &Requirements)
static bool useMaskedInterleavedAccesses(const TargetTransformInfo &TTI)
static cl::opt< unsigned > NumberOfStoresToPredicate("vectorize-num-stores-pred", cl::init(1), cl::Hidden, cl::desc("Max number of stores to be predicated behind an if."))
The number of stores in a loop that are allowed to need predication.
static cl::opt< unsigned > MaxNestedScalarReductionIC("max-nested-scalar-reduction-interleave", cl::init(2), cl::Hidden, cl::desc("The maximum interleave count to use when interleaving a scalar " "reduction in a nested loop."))
static cl::opt< unsigned > ForceTargetMaxScalarInterleaveFactor("force-target-max-scalar-interleave", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's max interleave factor for " "scalar loops."))
static void checkMixedPrecision(Loop *L, OptimizationRemarkEmitter *ORE)
static bool willGenerateVectors(VPlan &Plan, ElementCount VF, const TargetTransformInfo &TTI)
Check if any recipe of Plan will generate a vector value, which will be assigned a vector register.
static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks, VectorizationFactor &VF, Loop *L, PredicatedScalarEvolution &PSE, VPCostContext &CostCtx, VPlan &Plan, ScalarEpilogueLowering SEL, std::optional< unsigned > VScale)
This function determines whether or not it's still profitable to vectorize the loop given the extra w...
static void addExitUsersForFirstOrderRecurrences(VPlan &Plan, VFRange &Range)
Handle users in the exit block for first order reductions in the original exit block.
static void fixScalarResumeValuesFromBypass(BasicBlock *BypassBlock, Loop *L, VPlan &BestEpiPlan, LoopVectorizationLegality &LVL, const SCEV2ValueTy &ExpandedSCEVs, Value *MainVectorTripCount)
static cl::opt< bool > MaximizeBandwidth("vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden, cl::desc("Maximize bandwidth when selecting vectorization factor which " "will be determined by the smallest type in loop."))
static OptimizationRemarkAnalysis createLVAnalysis(const char *PassName, StringRef RemarkName, Loop *TheLoop, Instruction *I, DebugLoc DL={})
Create an analysis remark that explains why vectorization failed.
#define F(x, y, z)
Definition MD5.cpp:55
#define I(x, y, z)
Definition MD5.cpp:58
mir Rename Register Operands
This file implements a map that provides insertion order iteration.
This file contains the declarations for metadata subclasses.
#define T
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
#define P(N)
This file contains the declarations for profiling metadata utility functions.
const SmallVectorImpl< MachineOperand > & Cond
static BinaryOperator * CreateMul(Value *S1, Value *S2, const Twine &Name, BasicBlock::iterator InsertBefore, Value *FlagsOp)
static BinaryOperator * CreateAdd(Value *S1, Value *S2, const Twine &Name, BasicBlock::iterator InsertBefore, Value *FlagsOp)
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
This file contains some templates that are useful if you are working with the STL at all.
#define OP(OPC)
Definition Instruction.h:46
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition Statistic.h:171
#define LLVM_DEBUG(...)
Definition Debug.h:114
#define DEBUG_WITH_TYPE(TYPE,...)
DEBUG_WITH_TYPE macro - This macro should be used by passes to emit debug information.
Definition Debug.h:72
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")
This pass exposes codegen information to IR-level passes.
LocallyHashedType DenseMapInfo< LocallyHashedType >::Empty
This file implements the TypeSwitch template, which mimics a switch() statement whose cases are type ...
This file contains the declarations of different VPlan-related auxiliary helpers.
This file provides utility VPlan to VPlan transformations.
This file declares the class VPlanVerifier, which contains utility functions to check the consistency...
This file contains the declarations of the Vectorization Plan base classes:
static const char PassName[]
Value * RHS
Value * LHS
static const uint32_t IV[8]
Definition blake3_impl.h:83
A manager for alias analyses.
Class for arbitrary precision integers.
Definition APInt.h:78
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition APInt.h:234
uint64_t getZExtValue() const
Get zero extended value.
Definition APInt.h:1540
unsigned getActiveBits() const
Compute the number of active bits in the value.
Definition APInt.h:1512
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:41
size_t size() const
size - Get the array size.
Definition ArrayRef.h:147
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
LLVM_ABI unsigned getVScaleRangeMin() const
Returns the minimum value for the vscale_range attribute.
LLVM Basic Block Representation.
Definition BasicBlock.h:62
iterator begin()
Instruction iterator methods.
Definition BasicBlock.h:459
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
Definition BasicBlock.h:528
LLVM_ABI const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const Function * getParent() const
Return the enclosing method, or null if none.
Definition BasicBlock.h:213
LLVM_ABI InstListType::const_iterator getFirstNonPHIIt() const
Returns an iterator to the first instruction in this block that is not a PHINode instruction.
LLVM_ABI const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
LLVM_ABI const BasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this basic block belongs to.
LLVM_ABI LLVMContext & getContext() const
Get the context in which this basic block lives.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition BasicBlock.h:233
BinaryOps getOpcode() const
Definition InstrTypes.h:374
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Conditional or Unconditional Branch instruction.
bool isConditional() const
static BranchInst * Create(BasicBlock *IfTrue, InsertPosition InsertBefore=nullptr)
BasicBlock * getSuccessor(unsigned i) const
Represents analyses that only rely on functions' control flow.
Definition Analysis.h:73
bool isNoBuiltin() const
Return true if the call should not be treated as a call to a builtin.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Value * getArgOperand(unsigned i) const
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
unsigned arg_size() const
This class represents a function call, abstracting a target machine's calling convention.
static Type * makeCmpResultType(Type *opnd_type)
Create a result type for fcmp/icmp.
Definition InstrTypes.h:984
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:678
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:701
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:703
@ ICMP_NE
not equal
Definition InstrTypes.h:700
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:704
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
Definition InstrTypes.h:791
An abstraction over a floating-point predicate, and a pack of an integer predicate with samesign info...
This is the shared class of boolean and integer constants.
Definition Constants.h:87
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:63
A debug info location.
Definition DebugLoc.h:124
static DebugLoc getTemporary()
Definition DebugLoc.h:161
static DebugLoc getUnknown()
Definition DebugLoc.h:162
An analysis that produces DemandedBits for a function.
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
Definition DenseMap.h:187
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)
Definition DenseMap.h:229
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
Definition DenseMap.h:156
void insert_range(Range &&R)
Inserts range of 'std::pair<KeyT, ValueT>' values into the map.
Definition DenseMap.h:267
Implements a dense probed hash-table based set.
Definition DenseSet.h:269
Analysis pass which computes a DominatorTree.
Definition Dominators.h:284
void changeImmediateDominator(DomTreeNodeBase< NodeT > *N, DomTreeNodeBase< NodeT > *NewIDom)
changeImmediateDominator - This method is used to update the dominator tree information when a node's...
void eraseNode(NodeT *BB)
eraseNode - Removes a node from the dominator tree.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition Dominators.h:165
constexpr bool isVector() const
One or more elements.
Definition TypeSize.h:324
static constexpr ElementCount getScalable(ScalarTy MinVal)
Definition TypeSize.h:312
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition TypeSize.h:309
static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)
Definition TypeSize.h:315
constexpr bool isScalar() const
Exactly one element.
Definition TypeSize.h:320
BasicBlock * getAdditionalBypassBlock() const
Return the additional bypass block which targets the scalar loop by skipping the epilogue loop after ...
BasicBlock * createVectorizedLoopSkeleton() final
Implements the interface for creating a vectorized skeleton using the epilogue loop strategy (i....
EpilogueVectorizerEpilogueLoop(Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, const TargetTransformInfo *TTI, AssumptionCache *AC, EpilogueLoopVectorizationInfo &EPI, LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, GeneratedRTChecks &Checks, VPlan &Plan)
void printDebugTracesAtStart() override
Allow subclasses to override and print debug traces before/after vplan execution, when trace informat...
BasicBlock * emitMinimumVectorEpilogueIterCountCheck(BasicBlock *VectorPH, BasicBlock *Bypass, BasicBlock *Insert)
Emits an iteration count bypass check after the main vector loop has finished to see if there are any...
A specialized derived class of inner loop vectorizer that performs vectorization of main loops in the...
void introduceCheckBlockInVPlan(BasicBlock *CheckIRBB)
Introduces a new VPIRBasicBlock for CheckIRBB to Plan between the vector preheader and its predecesso...
BasicBlock * emitIterationCountCheck(BasicBlock *VectorPH, BasicBlock *Bypass, bool ForEpilogue)
Emits an iteration count bypass check once for the main loop (when ForEpilogue is false) and once for...
EpilogueVectorizerMainLoop(Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, const TargetTransformInfo *TTI, AssumptionCache *AC, EpilogueLoopVectorizationInfo &EPI, LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, GeneratedRTChecks &Check, VPlan &Plan)
Value * createIterationCountCheck(BasicBlock *VectorPH, ElementCount VF, unsigned UF) const
void printDebugTracesAtStart() override
Allow subclasses to override and print debug traces before/after vplan execution, when trace informat...
BasicBlock * createVectorizedLoopSkeleton() final
Implements the interface for creating a vectorized skeleton using the main loop strategy (i....
Convenience struct for specifying and reasoning about fast-math flags.
Definition FMF.h:22
Class to represent function types.
param_iterator param_begin() const
param_iterator param_end() const
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition Function.h:209
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition Function.cpp:762
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition Function.cpp:727
Represents flags for the getelementptr instruction/expression.
static GEPNoWrapFlags none()
Common base class shared among various IRBuilders.
Definition IRBuilder.h:114
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
Definition IRBuilder.h:345
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2780
A struct for saving information about induction variables.
const SCEV * getStep() const
InductionKind
This enum represents the kinds of inductions that we support.
@ IK_NoInduction
Not an induction variable.
@ IK_FpInduction
Floating point induction variable.
@ IK_PtrInduction
Pointer induction var. Step = C.
@ IK_IntInduction
Integer induction variable. Step = C.
const SmallVectorImpl< Instruction * > & getCastInsts() const
Returns a reference to the type cast instructions in the induction update chain, that are redundant w...
Value * getStartValue() const
InnerLoopAndEpilogueVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, const TargetTransformInfo *TTI, AssumptionCache *AC, EpilogueLoopVectorizationInfo &EPI, LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, GeneratedRTChecks &Checks, VPlan &Plan, ElementCount VecWidth, ElementCount MinProfitableTripCount, unsigned UnrollFactor)
EpilogueLoopVectorizationInfo & EPI
Holds and updates state information required to vectorize the main loop and its epilogue in two separ...
InnerLoopVectorizer vectorizes loops which contain only one basic block to a specified vectorization ...
virtual void printDebugTracesAtStart()
Allow subclasses to override and print debug traces before/after vplan execution, when trace informat...
Value * TripCount
Trip count of the original loop.
const TargetTransformInfo * TTI
Target Transform Info.
LoopVectorizationCostModel * Cost
The profitablity analysis.
BlockFrequencyInfo * BFI
BFI and PSI are used to check for profile guided size optimizations.
Value * getTripCount() const
Returns the original loop trip count.
friend class LoopVectorizationPlanner
PredicatedScalarEvolution & PSE
A wrapper around ScalarEvolution used to add runtime SCEV checks.
LoopInfo * LI
Loop Info.
ProfileSummaryInfo * PSI
DominatorTree * DT
Dominator Tree.
void setTripCount(Value *TC)
Used to set the trip count after ILV's construction and after the preheader block has been executed.
void fixVectorizedLoop(VPTransformState &State)
Fix the vectorized code, taking care of header phi's, and more.
virtual BasicBlock * createVectorizedLoopSkeleton()
Creates a basic block for the scalar preheader.
virtual void printDebugTracesAtEnd()
AssumptionCache * AC
Assumption Cache.
InnerLoopVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, const TargetTransformInfo *TTI, AssumptionCache *AC, ElementCount VecWidth, unsigned UnrollFactor, LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, GeneratedRTChecks &RTChecks, VPlan &Plan)
IRBuilder Builder
The builder that we use.
void fixNonInductionPHIs(VPTransformState &State)
Fix the non-induction PHIs in Plan.
VPBasicBlock * VectorPHVPBB
The vector preheader block of Plan, used as target for check blocks introduced during skeleton creati...
unsigned UF
The vectorization unroll factor to use.
GeneratedRTChecks & RTChecks
Structure to hold information about generated runtime checks, responsible for cleaning the checks,...
virtual ~InnerLoopVectorizer()=default
ElementCount VF
The vectorization SIMD factor to use.
Loop * OrigLoop
The original loop.
BasicBlock * createScalarPreheader(StringRef Prefix)
Create and return a new IR basic block for the scalar preheader whose name is prefixed with Prefix.
InstSimplifyFolder - Use InstructionSimplify to fold operations to existing values.
static InstructionCost getInvalid(CostType Val=0)
static InstructionCost getMax()
CostType getValue() const
This function is intended to be used as sparingly as possible, since the class provides the full rang...
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
LLVM_ABI void moveBefore(InstListType::iterator InsertPos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
bool isBinaryOp() const
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Instruction * user_back()
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
const char * getOpcodeName() const
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Class to represent integer types.
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition Type.cpp:319
LLVM_ABI APInt getMask() const
For example, this is 0xFF for an 8 bit integer, 0xFFFF for i16, etc.
Definition Type.cpp:343
The group of interleaved loads/stores sharing the same stride and close to each other.
uint32_t getFactor() const
InstTy * getMember(uint32_t Index) const
Get the member with the given index Index.
InstTy * getInsertPos() const
uint32_t getNumMembers() const
Drive the analysis of interleaved memory accesses in the loop.
bool requiresScalarEpilogue() const
Returns true if an interleaved group that may access memory out-of-bounds requires a scalar epilogue ...
LLVM_ABI void analyzeInterleaving(bool EnableMaskedInterleavedGroup)
Analyze the interleaved accesses and collect them in interleave groups.
An instruction for reading from memory.
Type * getPointerOperandType() const
This analysis provides dependence information for the memory accesses of a loop.
Drive the analysis of memory accesses in the loop.
const RuntimePointerChecking * getRuntimePointerChecking() const
unsigned getNumRuntimePointerChecks() const
Number of memchecks required to prove independence of otherwise may-alias pointers.
Analysis pass that exposes the LoopInfo for a function.
Definition LoopInfo.h:569
bool contains(const LoopT *L) const
Return true if the specified loop is contained within in this loop.
BlockT * getLoopLatch() const
If there is a single latch block for this loop, return it.
bool isInnermost() const
Return true if the loop does not contain any (natural) loops.
BlockT * getHeader() const
iterator_range< block_iterator > blocks() const
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
Store the result of a depth first search within basic blocks contained by a single loop.
RPOIterator beginRPO() const
Reverse iterate over the cached postorder blocks.
void perform(const LoopInfo *LI)
Traverse the loop blocks and store the DFS result.
RPOIterator endRPO() const
Wrapper class to LoopBlocksDFS that provides a standard begin()/end() interface for the DFS reverse p...
void perform(const LoopInfo *LI)
Traverse the loop blocks and store the DFS result.
void removeBlock(BlockT *BB)
This method completely removes BB from all data structures, including all of the Loop objects it is n...
LoopVectorizationCostModel - estimates the expected speedups due to vectorization.
SmallPtrSet< Type *, 16 > ElementTypesInLoop
All element types found in the loop.
bool isLegalMaskedLoad(Type *DataType, Value *Ptr, Align Alignment, unsigned AddressSpace) const
Returns true if the target machine supports masked load operation for the given DataType and kind of ...
LoopVectorizationCostModel(ScalarEpilogueLowering SEL, Loop *L, PredicatedScalarEvolution &PSE, LoopInfo *LI, LoopVectorizationLegality *Legal, const TargetTransformInfo &TTI, const TargetLibraryInfo *TLI, DemandedBits *DB, AssumptionCache *AC, OptimizationRemarkEmitter *ORE, const Function *F, const LoopVectorizeHints *Hints, InterleavedAccessInfo &IAI, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI)
void collectElementTypesForWidening()
Collect all element types in the loop for which widening is needed.
bool canVectorizeReductions(ElementCount VF) const
Returns true if the target machine supports all of the reduction variables found for the given VF.
bool isLegalMaskedStore(Type *DataType, Value *Ptr, Align Alignment, unsigned AddressSpace) const
Returns true if the target machine supports masked store operation for the given DataType and kind of...
bool isEpilogueVectorizationProfitable(const ElementCount VF, const unsigned IC) const
Returns true if epilogue vectorization is considered profitable, and false otherwise.
bool isPredicatedInst(Instruction *I) const
Returns true if I is an instruction that needs to be predicated at runtime.
void collectValuesToIgnore()
Collect values we want to ignore in the cost model.
void collectInLoopReductions()
Split reductions into those that happen in the loop, and those that happen outside.
std::pair< unsigned, unsigned > getSmallestAndWidestTypes()
bool isUniformAfterVectorization(Instruction *I, ElementCount VF) const
Returns true if I is known to be uniform after vectorization.
void collectNonVectorizedAndSetWideningDecisions(ElementCount VF)
Collect values that will not be widened, including Uniforms, Scalars, and Instructions to Scalarize f...
PredicatedScalarEvolution & PSE
Predicated scalar evolution analysis.
const LoopVectorizeHints * Hints
Loop Vectorize Hint.
std::optional< unsigned > getMaxSafeElements() const
Return maximum safe number of elements to be processed per vector iteration, which do not prevent sto...
const TargetTransformInfo & TTI
Vector target information.
LoopVectorizationLegality * Legal
Vectorization legality.
std::optional< InstructionCost > getReductionPatternCost(Instruction *I, ElementCount VF, Type *VectorTy) const
Return the cost of instructions in an inloop reduction pattern, if I is part of that pattern.
InstructionCost getInstructionCost(Instruction *I, ElementCount VF)
Returns the execution time cost of an instruction for a given vector width.
DemandedBits * DB
Demanded bits analysis.
bool interleavedAccessCanBeWidened(Instruction *I, ElementCount VF) const
Returns true if I is a memory instruction in an interleaved-group of memory accesses that can be vect...
const TargetLibraryInfo * TLI
Target Library Info.
bool memoryInstructionCanBeWidened(Instruction *I, ElementCount VF)
Returns true if I is a memory instruction with consecutive memory access that can be widened.
const InterleaveGroup< Instruction > * getInterleavedAccessGroup(Instruction *Instr) const
Get the interleaved access group that Instr belongs to.
InstructionCost getVectorIntrinsicCost(CallInst *CI, ElementCount VF) const
Estimate cost of an intrinsic call instruction CI if it were vectorized with factor VF.
bool OptForSize
Whether this loop should be optimized for size based on function attribute or profile information.
bool useMaxBandwidth(TargetTransformInfo::RegisterKind RegKind)
bool isScalarAfterVectorization(Instruction *I, ElementCount VF) const
Returns true if I is known to be scalar after vectorization.
bool isOptimizableIVTruncate(Instruction *I, ElementCount VF)
Return True if instruction I is an optimizable truncate whose operand is an induction variable.
FixedScalableVFPair computeMaxVF(ElementCount UserVF, unsigned UserIC)
bool shouldConsiderRegPressureForVF(ElementCount VF)
Loop * TheLoop
The loop that we evaluate.
TTI::TargetCostKind CostKind
The kind of cost that we are calculating.
TailFoldingStyle getTailFoldingStyle(bool IVUpdateMayOverflow=true) const
Returns the TailFoldingStyle that is best for the current loop.
InterleavedAccessInfo & InterleaveInfo
The interleave access information contains groups of interleaved accesses with the same stride and cl...
SmallPtrSet< const Value *, 16 > ValuesToIgnore
Values to ignore in the cost model.
void setVectorizedCallDecision(ElementCount VF)
A call may be vectorized in different ways depending on whether we have vectorized variants available...
void invalidateCostModelingDecisions()
Invalidates decisions already taken by the cost model.
bool isAccessInterleaved(Instruction *Instr) const
Check if Instr belongs to any interleaved access group.
bool selectUserVectorizationFactor(ElementCount UserVF)
Setup cost-based decisions for user vectorization factor.
std::optional< unsigned > getVScaleForTuning() const
Return the value of vscale used for tuning the cost model.
OptimizationRemarkEmitter * ORE
Interface to emit optimization remarks.
LoopInfo * LI
Loop Info analysis.
bool requiresScalarEpilogue(bool IsVectorizing) const
Returns true if we're required to use a scalar epilogue for at least the final iteration of the origi...
SmallPtrSet< const Value *, 16 > VecValuesToIgnore
Values to ignore in the cost model when VF > 1.
bool isInLoopReduction(PHINode *Phi) const
Returns true if the Phi is part of an inloop reduction.
bool isProfitableToScalarize(Instruction *I, ElementCount VF) const
void setWideningDecision(const InterleaveGroup< Instruction > *Grp, ElementCount VF, InstWidening W, InstructionCost Cost)
Save vectorization decision W and Cost taken by the cost model for interleaving group Grp and vector ...
const MapVector< Instruction *, uint64_t > & getMinimalBitwidths() const
CallWideningDecision getCallWideningDecision(CallInst *CI, ElementCount VF) const
bool isLegalGatherOrScatter(Value *V, ElementCount VF)
Returns true if the target machine can represent V as a masked gather or scatter operation.
bool canTruncateToMinimalBitwidth(Instruction *I, ElementCount VF) const
bool shouldConsiderInvariant(Value *Op)
Returns true if Op should be considered invariant and if it is trivially hoistable.
bool foldTailByMasking() const
Returns true if all loop blocks should be masked to fold tail loop.
bool foldTailWithEVL() const
Returns true if VP intrinsics with explicit vector length support should be generated in the tail fol...
bool usePredicatedReductionSelect() const
Returns true if the predicated reduction select should be used to set the incoming value for the redu...
bool blockNeedsPredicationForAnyReason(BasicBlock *BB) const
Returns true if the instructions in this block requires predication for any reason,...
void setCallWideningDecision(CallInst *CI, ElementCount VF, InstWidening Kind, Function *Variant, Intrinsic::ID IID, std::optional< unsigned > MaskPos, InstructionCost Cost)
void setTailFoldingStyles(bool IsScalableVF, unsigned UserIC)
Selects and saves TailFoldingStyle for 2 options - if IV update may overflow or not.
AssumptionCache * AC
Assumption cache.
void setWideningDecision(Instruction *I, ElementCount VF, InstWidening W, InstructionCost Cost)
Save vectorization decision W and Cost taken by the cost model for instruction I and vector width VF.
InstWidening
Decision that was taken during cost calculation for memory instruction.
bool isScalarWithPredication(Instruction *I, ElementCount VF) const
Returns true if I is an instruction which requires predication and for which our chosen predication s...
InstructionCost getVectorCallCost(CallInst *CI, ElementCount VF) const
Estimate cost of a call instruction CI if it were vectorized with factor VF.
bool useOrderedReductions(const RecurrenceDescriptor &RdxDesc) const
Returns true if we should use strict in-order reductions for the given RdxDesc.
std::pair< InstructionCost, InstructionCost > getDivRemSpeculationCost(Instruction *I, ElementCount VF) const
Return the costs for our two available strategies for lowering a div/rem operation which requires spe...
bool isDivRemScalarWithPredication(InstructionCost ScalarCost, InstructionCost SafeDivisorCost) const
Given costs for both strategies, return true if the scalar predication lowering should be used for di...
InstructionCost expectedCost(ElementCount VF)
Returns the expected execution cost.
void setCostBasedWideningDecision(ElementCount VF)
Memory access instruction may be vectorized in more than one way.
InstWidening getWideningDecision(Instruction *I, ElementCount VF) const
Return the cost model decision for the given instruction I and vector width VF.
FixedScalableVFPair MaxPermissibleVFWithoutMaxBW
The highest VF possible for this loop, without using MaxBandwidth.
bool isScalarEpilogueAllowed() const
Returns true if a scalar epilogue is not allowed due to optsize or a loop hint annotation.
InstructionCost getWideningCost(Instruction *I, ElementCount VF)
Return the vectorization cost for the given instruction I and vector width VF.
void collectInstsToScalarize(ElementCount VF)
Collects the instructions to scalarize for each predicated instruction in the loop.
LoopVectorizationLegality checks if it is legal to vectorize a loop, and to what vectorization factor...
MapVector< PHINode *, InductionDescriptor > InductionList
InductionList saves induction variables and maps them to the induction descriptor.
const SmallPtrSetImpl< const Instruction * > & getPotentiallyFaultingLoads() const
Returns potentially faulting loads.
bool canVectorize(bool UseVPlanNativePath)
Returns true if it is legal to vectorize this loop.
bool canVectorizeFPMath(bool EnableStrictReductions)
Returns true if it is legal to vectorize the FP math operations in this loop.
PHINode * getPrimaryInduction()
Returns the primary induction variable.
const SmallVector< BasicBlock *, 4 > & getCountableExitingBlocks() const
Returns all exiting blocks with a countable exit, i.e.
const InductionList & getInductionVars() const
Returns the induction variables found in the loop.
bool hasUncountableEarlyExit() const
Returns true if the loop has exactly one uncountable early exit, i.e.
bool hasHistograms() const
Returns a list of all known histogram operations in the loop.
const LoopAccessInfo * getLAI() const
Planner drives the vectorization process after having passed Legality checks.
VectorizationFactor selectEpilogueVectorizationFactor(const ElementCount MaxVF, unsigned IC)
VPlan & getPlanFor(ElementCount VF) const
Return the VPlan for VF.
Definition VPlan.cpp:1611
VectorizationFactor planInVPlanNativePath(ElementCount UserVF)
Use the VPlan-native path to plan how to best vectorize, return the best VF and its cost.
void updateLoopMetadataAndProfileInfo(Loop *VectorLoop, VPBasicBlock *HeaderVPBB, const VPlan &Plan, bool VectorizingEpilogue, MDNode *OrigLoopID, std::optional< unsigned > OrigAverageTripCount, unsigned OrigLoopInvocationWeight, unsigned EstimatedVFxUF, bool DisableRuntimeUnroll)
Update loop metadata and profile info for both the scalar remainder loop and VectorLoop,...
Definition VPlan.cpp:1662
void buildVPlans(ElementCount MinVF, ElementCount MaxVF)
Build VPlans for power-of-2 VF's between MinVF and MaxVF inclusive, according to the information gath...
Definition VPlan.cpp:1595
VectorizationFactor computeBestVF()
Compute and return the most profitable vectorization factor.
DenseMap< const SCEV *, Value * > executePlan(ElementCount VF, unsigned UF, VPlan &BestPlan, InnerLoopVectorizer &LB, DominatorTree *DT, bool VectorizingEpilogue)
Generate the IR code for the vectorized loop captured in VPlan BestPlan according to the best selecte...
unsigned selectInterleaveCount(VPlan &Plan, ElementCount VF, InstructionCost LoopCost)
void emitInvalidCostRemarks(OptimizationRemarkEmitter *ORE)
Emit remarks for recipes with invalid costs in the available VPlans.
static bool getDecisionAndClampRange(const std::function< bool(ElementCount)> &Predicate, VFRange &Range)
Test a Predicate on a Range of VF's.
Definition VPlan.cpp:1576
void printPlans(raw_ostream &O)
Definition VPlan.cpp:1760
void plan(ElementCount UserVF, unsigned UserIC)
Build VPlans for the specified UserVF and UserIC if they are non-zero or all applicable candidate VFs...
void addMinimumIterationCheck(VPlan &Plan, ElementCount VF, unsigned UF, ElementCount MinProfitableTripCount) const
Create a check to Plan to see if the vector loop should be executed based on its trip count.
bool hasPlanWithVF(ElementCount VF) const
Look through the existing plans and return true if we have one with vectorization factor VF.
This holds vectorization requirements that must be verified late in the process.
Utility class for getting and setting loop vectorizer hints in the form of loop metadata.
bool allowVectorization(Function *F, Loop *L, bool VectorizeOnlyWhenForced) const
void emitRemarkWithHints() const
Dumps all the hint information.
const char * vectorizeAnalysisPassName() const
If hints are provided that force vectorization, use the AlwaysPrint pass name to force the frontend t...
This class emits a version of the loop where run-time checks ensure that may-alias pointers can't ove...
Represents a single loop in the control flow graph.
Definition LoopInfo.h:40
DebugLoc getStartLoc() const
Return the debug location of the start of this loop.
Definition LoopInfo.cpp:632
Metadata node.
Definition Metadata.h:1077
This class implements a map that also provides access to all stored values in a deterministic order.
Definition MapVector.h:36
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition MapVector.h:115
Function * getFunction(StringRef Name) const
Look up the specified function in the module symbol table.
Definition Module.cpp:230
Diagnostic information for optimization analysis remarks related to pointer aliasing.
Diagnostic information for optimization analysis remarks related to floating-point non-commutativity.
Diagnostic information for optimization analysis remarks.
The optimization diagnostic interface.
LLVM_ABI void emit(DiagnosticInfoOptimizationBase &OptDiag)
Output the remark via the diagnostic handler and to the optimization record file.
Diagnostic information for missed-optimization remarks.
Diagnostic information for applied optimization remarks.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
op_range incoming_values()
void setIncomingValueForBlock(const BasicBlock *BB, Value *V)
Set every incoming value(s) for block BB to V.
Value * getIncomingValueForBlock(const BasicBlock *BB) const
unsigned getNumIncomingValues() const
Return the number of incoming edges.
An interface layer with SCEV used to manage how we see SCEV expressions for values in the context of ...
ScalarEvolution * getSE() const
Returns the ScalarEvolution analysis used.
LLVM_ABI const SCEVPredicate & getPredicate() const
LLVM_ABI unsigned getSmallConstantMaxTripCount()
Returns the upper bound of the loop trip count as a normal unsigned value, or 0 if the trip count is ...
LLVM_ABI const SCEV * getBackedgeTakenCount()
Get the (predicated) backedge count for the analyzed loop.
LLVM_ABI const SCEV * getSCEV(Value *V)
Returns the SCEV expression of V, in the context of the current SCEV predicate.
A set of analyses that are preserved following a run of a transformation pass.
Definition Analysis.h:112
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition Analysis.h:118
PreservedAnalyses & preserveSet()
Mark an analysis set as preserved.
Definition Analysis.h:151
PreservedAnalyses & preserve()
Mark an analysis as preserved.
Definition Analysis.h:132
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
Analysis providing profile information.
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
static bool isFMulAddIntrinsic(Instruction *I)
Returns true if the instruction is a call to the llvm.fmuladd intrinsic.
FastMathFlags getFastMathFlags() const
Instruction * getLoopExitInstr() const
static LLVM_ABI unsigned getOpcode(RecurKind Kind)
Returns the opcode corresponding to the RecurrenceKind.
Type * getRecurrenceType() const
Returns the type of the recurrence.
const SmallPtrSet< Instruction *, 8 > & getCastInsts() const
Returns a reference to the instructions used for type-promoting the recurrence.
unsigned getMinWidthCastToRecurrenceTypeInBits() const
Returns the minimum width used by the recurrence in bits.
TrackingVH< Value > getRecurrenceStartValue() const
LLVM_ABI SmallVector< Instruction *, 4 > getReductionOpChain(PHINode *Phi, Loop *L) const
Attempts to find a chain of operations from Phi to LoopExitInst that can be treated as a set of reduc...
static bool isAnyOfRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
bool isSigned() const
Returns true if all source operands of the recurrence are SExtInsts.
RecurKind getRecurrenceKind() const
bool isOrdered() const
Expose an ordered FP reduction to the instance users.
static LLVM_ABI bool isFloatingPointRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is a floating point kind.
static bool isFindIVRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
Value * getSentinelValue() const
Returns the sentinel value for FindFirstIV & FindLastIV recurrences to replace the start value.
static bool isMinMaxRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is any min/max kind.
std::optional< ArrayRef< PointerDiffInfo > > getDiffChecks() const
const SmallVectorImpl< RuntimePointerCheck > & getChecks() const
Returns the checks that generateChecks created.
This class uses information about analyze scalars to rewrite expressions in canonical form.
ScalarEvolution * getSE()
bool isInsertedInstruction(Instruction *I) const
Return true if the specified instruction was inserted by the code rewriter.
LLVM_ABI Value * expandCodeForPredicate(const SCEVPredicate *Pred, Instruction *Loc)
Generates a code sequence that evaluates this predicate.
void eraseDeadInstructions(Value *Root)
Remove inserted instructions that are dead, e.g.
virtual bool isAlwaysTrue() const =0
Returns true if the predicate is always true.
This class represents an analyzed expression in the program.
LLVM_ABI bool isZero() const
Return true if the expression is a constant zero.
LLVM_ABI Type * getType() const
Return the LLVM type of this SCEV expression.
Analysis pass that exposes the ScalarEvolution for a function.
The main scalar evolution driver.
LLVM_ABI const SCEV * getURemExpr(const SCEV *LHS, const SCEV *RHS)
Represents an unsigned remainder expression based on unsigned division.
LLVM_ABI const SCEV * getBackedgeTakenCount(const Loop *L, ExitCountKind Kind=Exact)
If the specified loop has a predictable backedge-taken count, return it, otherwise return a SCEVCould...
LLVM_ABI const SCEV * getConstant(ConstantInt *V)
LLVM_ABI const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
LLVM_ABI const SCEV * getTripCountFromExitCount(const SCEV *ExitCount)
A version of getTripCountFromExitCount below which always picks an evaluation type which can not resu...
const SCEV * getOne(Type *Ty)
Return a SCEV for the constant 1 of a specific type.
LLVM_ABI void forgetLoop(const Loop *L)
This method should be called by the client when it has changed a loop in a way that may effect Scalar...
LLVM_ABI bool isLoopInvariant(const SCEV *S, const Loop *L)
Return true if the value of the given SCEV is unchanging in the specified loop.
LLVM_ABI bool isSCEVable(Type *Ty) const
Test if values of the given type are analyzable within the SCEV framework.
LLVM_ABI const SCEV * getElementCount(Type *Ty, ElementCount EC, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
LLVM_ABI void forgetValue(Value *V)
This method should be called by the client when it has changed a value in a way that may effect its v...
LLVM_ABI void forgetBlockAndLoopDispositions(Value *V=nullptr)
Called when the client has changed the disposition of values in a loop or block.
const SCEV * getMinusOne(Type *Ty)
Return a SCEV for the constant -1 of a specific type.
LLVM_ABI void forgetLcssaPhiWithNewPredecessor(Loop *L, PHINode *V)
Forget LCSSA phi node V of loop L to which a new predecessor was added, such that it may no longer be...
LLVM_ABI unsigned getSmallConstantTripCount(const Loop *L)
Returns the exact trip count of the loop if we can compute it, and the result is a small constant.
APInt getUnsignedRangeMax(const SCEV *S)
Determine the max of the unsigned range for a particular SCEV.
LLVM_ABI const SCEV * applyLoopGuards(const SCEV *Expr, const Loop *L)
Try to apply information from loop guards for L to Expr.
LLVM_ABI const SCEV * getAddExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap, unsigned Depth=0)
Get a canonical add expression, or something simpler if possible.
LLVM_ABI bool isKnownPredicate(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
Test if the given expression is known to satisfy the condition described by Pred, LHS,...
This class represents the LLVM 'select' instruction.
A vector that has set insertion semantics.
Definition SetVector.h:59
size_type size() const
Determine the number of elements in the SetVector.
Definition SetVector.h:104
void insert_range(Range &&R)
Definition SetVector.h:193
size_type count(const key_type &key) const
Count the number of elements of a given key in the SetVector.
Definition SetVector.h:279
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition SetVector.h:168
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
Definition SetVector.h:356
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
Definition StringRef.h:55
Analysis pass providing the TargetTransformInfo.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
static LLVM_ABI PartialReductionExtendKind getPartialReductionExtendKind(Instruction *I)
Get the kind of extension that an instruction represents.
static LLVM_ABI OperandValueInfo getOperandInfo(const Value *V)
Collect properties of V used in cost analysis, e.g. OP_PowerOf2.
TargetCostKind
The kind of cost model.
@ TCK_RecipThroughput
Reciprocal throughput.
@ TCK_CodeSize
Instruction code size.
@ TCK_SizeAndLatency
The weighted sum of size and latency.
@ TCK_Latency
The latency of instruction.
@ TCC_Free
Expected to fold away in lowering.
LLVM_ABI InstructionCost getInstructionCost(const User *U, ArrayRef< const Value * > Operands, TargetCostKind CostKind) const
Estimate the cost of a given IR user when lowered.
@ SK_Splice
Concatenates elements from the first input vector with elements of the second input vector.
@ SK_Broadcast
Broadcast element 0 to all other elements.
@ SK_Reverse
Reverse the order of the vector.
CastContextHint
Represents a hint about the context in which a cast is used.
@ Reversed
The cast is used with a reversed load/store.
@ Masked
The cast is used with a masked load/store.
@ None
The cast is not used with a load/store of any kind.
@ Normal
The cast is used with a normal load/store.
@ Interleave
The cast is used with an interleaved load/store.
@ GatherScatter
The cast is used with a gather/scatter.
This class represents a truncation of integer types.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
This class implements a switch-like dispatch statement for a value of 'T' using dyn_cast functionalit...
Definition TypeSwitch.h:87
TypeSwitch< T, ResultT > & Case(CallableT &&caseFn)
Add a case on the given type.
Definition TypeSwitch.h:96
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:45
LLVM_ABI unsigned getIntegerBitWidth() const
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:273
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:297
static LLVM_ABI Type * getVoidTy(LLVMContext &C)
Definition Type.cpp:281
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:352
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:198
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition Type.h:128
LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.cpp:231
static LLVM_ABI IntegerType * getInt1Ty(LLVMContext &C)
Definition Type.cpp:294
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
Definition Type.h:184
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:240
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:139
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
op_range operands()
Definition User.h:292
LLVM_ABI bool replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
Definition User.cpp:21
iterator_range< op_iterator > op_range
Definition User.h:281
Value * getOperand(unsigned i) const
Definition User.h:232
static SmallVector< VFInfo, 8 > getMappings(const CallInst &CI)
Retrieve all the VFInfo instances associated to the CallInst CI.
Definition VectorUtils.h:74
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
Definition VPlan.h:3760
void appendRecipe(VPRecipeBase *Recipe)
Augment the existing recipes of a VPBasicBlock with an additional Recipe as the last recipe.
Definition VPlan.h:3835
RecipeListTy::iterator iterator
Instruction iterators...
Definition VPlan.h:3787
iterator end()
Definition VPlan.h:3797
iterator begin()
Recipe iterator methods.
Definition VPlan.h:3795
iterator_range< iterator > phis()
Returns an iterator range over the PHI-like recipes in the block.
Definition VPlan.h:3848
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
Definition VPlan.cpp:246
VPRegionBlock * getEnclosingLoopRegion()
Definition VPlan.cpp:619
void insert(VPRecipeBase *Recipe, iterator InsertPt)
Definition VPlan.h:3826
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
Definition VPlan.h:81
VPRegionBlock * getParent()
Definition VPlan.h:173
const VPBasicBlock * getExitingBasicBlock() const
Definition VPlan.cpp:190
void setName(const Twine &newName)
Definition VPlan.h:166
size_t getNumSuccessors() const
Definition VPlan.h:219
void swapSuccessors()
Swap successors of the block. The block must have exactly 2 successors.
Definition VPlan.h:322
size_t getNumPredecessors() const
Definition VPlan.h:220
VPlan * getPlan()
Definition VPlan.cpp:165
VPBlockBase * getSinglePredecessor() const
Definition VPlan.h:215
const VPBasicBlock * getEntryBasicBlock() const
Definition VPlan.cpp:170
VPBlockBase * getSingleSuccessor() const
Definition VPlan.h:209
const VPBlocksTy & getSuccessors() const
Definition VPlan.h:198
static auto blocksOnly(const T &Range)
Return an iterator range over Range which only includes BlockTy blocks.
Definition VPlanUtils.h:228
static void insertOnEdge(VPBlockBase *From, VPBlockBase *To, VPBlockBase *BlockPtr)
Inserts BlockPtr on the edge between From and To.
Definition VPlanUtils.h:249
static void connectBlocks(VPBlockBase *From, VPBlockBase *To, unsigned PredIdx=-1u, unsigned SuccIdx=-1u)
Connect VPBlockBases From and To bi-directionally.
Definition VPlanUtils.h:187
static void reassociateBlocks(VPBlockBase *Old, VPBlockBase *New)
Reassociate all the blocks connected to Old so that they now point to New.
Definition VPlanUtils.h:214
VPlan-based builder utility analogous to IRBuilder.
VPDerivedIVRecipe * createDerivedIV(InductionDescriptor::InductionKind Kind, FPMathOperator *FPBinOp, VPValue *Start, VPValue *Current, VPValue *Step, const Twine &Name="")
Convert the input value Current to the corresponding value of an induction with Start and Step values...
VPPhi * createScalarPhi(ArrayRef< VPValue * > IncomingValues, DebugLoc DL, const Twine &Name="")
VPInstruction * createNaryOp(unsigned Opcode, ArrayRef< VPValue * > Operands, Instruction *Inst=nullptr, const Twine &Name="")
Create an N-ary operation with Opcode, Operands and set Inst as its underlying Instruction.
VPInstruction * createScalarCast(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy, DebugLoc DL)
unsigned getNumDefinedValues() const
Returns the number of values defined by the VPDef.
Definition VPlanValue.h:424
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
Definition VPlanValue.h:397
void execute(VPTransformState &State) override
Generate the transformed value of the induction at offset StartValue (1.
VPValue * getStepValue() const
Definition VPlan.h:3637
VPValue * getStartValue() const
Definition VPlan.h:3636
A pure virtual base class for all recipes modeling header phis, including phis for first order recurr...
Definition VPlan.h:1973
virtual VPValue * getBackedgeValue()
Returns the incoming value from the loop backedge.
Definition VPlan.h:2021
VPValue * getStartValue()
Returns the start value of the phi, if one is set.
Definition VPlan.h:2010
A recipe representing a sequence of load -> update -> store as part of a histogram operation.
Definition VPlan.h:1678
A special type of VPBasicBlock that wraps an existing IR basic block.
Definition VPlan.h:3913
Helper to manage IR metadata for recipes.
Definition VPlan.h:939
This is a concrete Recipe that models a single VPlan-level instruction.
Definition VPlan.h:980
@ ComputeAnyOfResult
Compute the final result of a AnyOf reduction with select(cmp(),x,y), where one of (x,...
Definition VPlan.h:1013
@ ResumeForEpilogue
Explicit user for the resume phi of the canonical induction in the main VPlan, used by the epilogue v...
Definition VPlan.h:1060
@ FirstOrderRecurrenceSplice
Definition VPlan.h:986
@ ReductionStartVector
Start vector for reductions with 3 operands: the original start value, the identity value for the red...
Definition VPlan.h:1051
unsigned getOpcode() const
Definition VPlan.h:1116
VPInterleaveRecipe is a recipe for transforming an interleave group of load or stores into one wide l...
Definition VPlan.h:2572
In what follows, the term "input IR" refers to code that is fed into the vectorizer whereas the term ...
A recipe for forming partial reductions.
Definition VPlan.h:2749
detail::zippy< llvm::detail::zip_first, VPUser::const_operand_range, const_incoming_blocks_range > incoming_values_and_blocks() const
Returns an iterator range over pairs of incoming values and corresponding incoming blocks.
Definition VPlan.h:1287
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
Definition VPlan.h:394
VPBasicBlock * getParent()
Definition VPlan.h:415
DebugLoc getDebugLoc() const
Returns the debug location of the recipe.
Definition VPlan.h:482
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Helper class to create VPRecipies from IR instructions.
VPRecipeBase * tryToCreateWidenRecipe(VPSingleDefRecipe *R, VFRange &Range)
Create and return a widened recipe for R if one can be created within the given VF Range.
VPValue * getBlockInMask(VPBasicBlock *VPBB) const
Returns the entry mask for block VPBB or null if the mask is all-true.
VPValue * getVPValueOrAddLiveIn(Value *V)
std::optional< unsigned > getScalingForReduction(const Instruction *ExitInst)
void collectScaledReductions(VFRange &Range)
Find all possible partial reductions in the loop and track all of those that are valid so recipes can...
VPReplicateRecipe * handleReplication(Instruction *I, ArrayRef< VPValue * > Operands, VFRange &Range)
Build a VPReplicationRecipe for I using Operands.
VPRecipeBase * tryToCreatePartialReduction(Instruction *Reduction, ArrayRef< VPValue * > Operands, unsigned ScaleFactor)
Create and return a partial reduction recipe for a reduction instruction along with binary operation ...
A recipe for handling reduction phis.
Definition VPlan.h:2327
bool isInLoop() const
Returns true, if the phi is part of an in-loop reduction.
Definition VPlan.h:2387
RecurKind getRecurrenceKind() const
Returns the recurrence kind of the reduction.
Definition VPlan.h:2381
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
Definition VPlan.h:3948
const VPBlockBase * getEntry() const
Definition VPlan.h:3984
VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...
Definition VPlan.h:2852
VPSingleDef is a base class for recipes for modeling a sequence of one or more output IR that define ...
Definition VPlan.h:521
Instruction * getUnderlyingInstr()
Returns the underlying instruction.
Definition VPlan.h:586
An analysis for type-inference for VPValues.
Type * inferScalarType(const VPValue *V)
Infer the type of V. Returns the scalar type of V.
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
Definition VPlanValue.h:199
void setOperand(unsigned I, VPValue *New)
Definition VPlanValue.h:243
VPValue * getOperand(unsigned N) const
Definition VPlanValue.h:238
void addOperand(VPValue *Operand)
Definition VPlanValue.h:232
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
Definition VPlan.cpp:135
Value * getLiveInIRValue() const
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
Definition VPlanValue.h:176
Value * getUnderlyingValue() const
Return the underlying Value attached to this VPValue.
Definition VPlanValue.h:85
void replaceAllUsesWith(VPValue *New)
Definition VPlan.cpp:1412
user_iterator user_begin()
Definition VPlanValue.h:130
unsigned getNumUsers() const
Definition VPlanValue.h:113
void replaceUsesWithIf(VPValue *New, llvm::function_ref< bool(VPUser &U, unsigned Idx)> ShouldReplace)
Go through the uses list for this VPValue and make each use point to New if the callback ShouldReplac...
Definition VPlan.cpp:1416
user_range users()
Definition VPlanValue.h:134
A recipe to compute a pointer to the last element of each part of a widened memory access for widened...
Definition VPlan.h:1837
VPWidenCastRecipe is a recipe to create vector cast instructions.
Definition VPlan.h:1479
A recipe for handling GEP instructions.
Definition VPlan.h:1765
Base class for widened induction (VPWidenIntOrFpInductionRecipe and VPWidenPointerInductionRecipe),...
Definition VPlan.h:2038
VPValue * getStepValue()
Returns the step value of the induction.
Definition VPlan.h:2066
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
Definition VPlan.h:2083
A recipe for handling phi nodes of integer and floating-point inductions, producing their vector valu...
Definition VPlan.h:2113
A common base class for widening memory operations.
Definition VPlan.h:3129
A recipe for widened phis.
Definition VPlan.h:2249
VPWidenRecipe is a recipe for producing a widened instruction using the opcode and operands of the re...
Definition VPlan.h:1436
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
Definition VPlan.h:4051
bool hasVF(ElementCount VF) const
Definition VPlan.h:4260
VPBasicBlock * getEntry()
Definition VPlan.h:4150
VPValue & getVectorTripCount()
The vector trip count.
Definition VPlan.h:4240
VPValue & getVF()
Returns the VF of the vector loop region.
Definition VPlan.h:4243
VPValue * getTripCount() const
The trip count of the original loop.
Definition VPlan.h:4212
iterator_range< SmallSetVector< ElementCount, 2 >::iterator > vectorFactors() const
Returns an iterator range over all VFs of the plan.
Definition VPlan.h:4267
bool hasUF(unsigned UF) const
Definition VPlan.h:4278
ArrayRef< VPIRBasicBlock * > getExitBlocks() const
Return an ArrayRef containing VPIRBasicBlocks wrapping the exit blocks of the original scalar loop.
Definition VPlan.h:4202
LLVM_ABI_FOR_TEST VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
Definition VPlan.cpp:1046
bool hasEarlyExit() const
Returns true if the VPlan is based on a loop with an early exit.
Definition VPlan.h:4423
InstructionCost cost(ElementCount VF, VPCostContext &Ctx)
Return the cost of this plan.
Definition VPlan.cpp:1028
void resetTripCount(VPValue *NewTripCount)
Resets the trip count for the VPlan.
Definition VPlan.h:4226
VPBasicBlock * getMiddleBlock()
Returns the 'middle' block of the plan, that is the block that selects whether to execute the scalar ...
Definition VPlan.h:4175
VPValue * getOrAddLiveIn(Value *V)
Gets the live-in VPValue for V or adds a new live-in (if none exists yet) for V.
Definition VPlan.h:4302
bool hasScalarVFOnly() const
Definition VPlan.h:4271
VPBasicBlock * getScalarPreheader() const
Return the VPBasicBlock for the preheader of the scalar loop.
Definition VPlan.h:4193
void execute(VPTransformState *State)
Generate the IR code for this VPlan.
Definition VPlan.cpp:952
VPCanonicalIVPHIRecipe * getCanonicalIV()
Returns the canonical induction recipe of the vector loop.
Definition VPlan.h:4356
VPIRBasicBlock * getScalarHeader() const
Return the VPIRBasicBlock wrapping the header of the scalar loop.
Definition VPlan.h:4198
VPBasicBlock * getVectorPreheader()
Returns the preheader of the vector loop region, if one exists, or null otherwise.
Definition VPlan.h:4155
VPlan * duplicate()
Clone the current VPlan, update all VPValues of the new VPlan and cloned recipes to refer to the clon...
Definition VPlan.cpp:1188
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:256
LLVM_ABI bool hasOneUser() const
Return true if there is exactly one user of this value.
Definition Value.cpp:166
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
Definition Value.cpp:390
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition Value.h:439
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:546
iterator_range< user_iterator > users()
Definition Value.h:426
LLVM_ABI LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.cpp:1101
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:322
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
std::pair< iterator, bool > insert(const ValueT &V)
Definition DenseSet.h:194
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
Definition DenseSet.h:169
constexpr bool hasKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns true if there exists a value X where RHS.multiplyCoefficientBy(X) will result in a value whos...
Definition TypeSize.h:269
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:200
static constexpr bool isKnownLE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:230
constexpr bool isNonZero() const
Definition TypeSize.h:156
constexpr ScalarTy getKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns a value X where RHS.multiplyCoefficientBy(X) will result in a value whose quantity matches ou...
Definition TypeSize.h:277
static constexpr bool isKnownLT(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:216
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition TypeSize.h:169
constexpr LeafTy multiplyCoefficientBy(ScalarTy RHS) const
Definition TypeSize.h:256
constexpr bool isFixed() const
Returns true if the quantity is not scaled by vscale.
Definition TypeSize.h:172
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition TypeSize.h:166
constexpr bool isZero() const
Definition TypeSize.h:154
static constexpr bool isKnownGT(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:223
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
Definition TypeSize.h:252
static constexpr bool isKnownGE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:237
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
Definition ilist_node.h:34
self_iterator getIterator()
Definition ilist_node.h:130
IteratorT end() const
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition raw_ostream.h:53
A raw_ostream that writes to an std::string.
Changed
This provides a very simple, boring adaptor for a begin and end iterator into a range type.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Entry
Definition COFF.h:862
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition CallingConv.h:24
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
Definition CallingConv.h:76
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
@ BasicBlock
Various leaf nodes.
Definition ISDOpcodes.h:81
std::variant< std::monostate, Loc::Single, Loc::Multi, Loc::MMI, Loc::EntryValue > Variant
Alias for the std::variant specialization base class of DbgVariable.
Definition DwarfDebug.h:189
SpecificConstantMatch m_ZeroInt()
Convenience matchers for specific integer values.
BinaryOp_match< SpecificConstantMatch, SrcTy, TargetOpcode::G_SUB > m_Neg(const SrcTy &&Src)
Matches a register negated by a G_SUB.
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
class_match< BinaryOperator > m_BinOp()
Match an arbitrary binary operation and ignore it.
OneOps_match< OpTy, Instruction::Freeze > m_Freeze(const OpTy &Op)
Matches FreezeInst.
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
bool match(Val *V, const Pattern &P)
bind_ty< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
BinaryOp_match< LHS, RHS, Instruction::Mul > m_Mul(const LHS &L, const RHS &R)
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
SpecificCmpClass_match< LHS, RHS, ICmpInst > m_SpecificICmp(CmpPredicate MatchPred, const LHS &L, const RHS &R)
class_match< CmpInst > m_Cmp()
Matches any compare instruction and ignore it.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
match_combine_or< CastInst_match< OpTy, ZExtInst >, CastInst_match< OpTy, SExtInst > > m_ZExtOrSExt(const OpTy &Op)
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
MatchFunctor< Val, Pattern > match_fn(const Pattern &P)
A match functor that can be used as a UnaryPredicate in functional algorithms like all_of.
class_match< const SCEVVScale > m_SCEVVScale()
bind_cst_ty m_scev_APInt(const APInt *&C)
Match an SCEV constant and bind it to an APInt.
specificloop_ty m_SpecificLoop(const Loop *L)
cst_pred_ty< is_specific_signed_cst > m_scev_SpecificSInt(int64_t V)
Match an SCEV constant with a plain signed integer (sign-extended value will be matched)
SCEVAffineAddRec_match< Op0_t, Op1_t, class_match< const Loop > > m_scev_AffineAddRec(const Op0_t &Op0, const Op1_t &Op1)
SCEVBinaryExpr_match< SCEVMulExpr, Op0_t, Op1_t > m_scev_Mul(const Op0_t &Op0, const Op1_t &Op1)
bool match(const SCEV *S, const Pattern &P)
class_match< const SCEV > m_SCEV()
match_combine_or< AllRecipe_match< Instruction::ZExt, Op0_t >, AllRecipe_match< Instruction::SExt, Op0_t > > m_ZExtOrSExt(const Op0_t &Op0)
VPInstruction_match< VPInstruction::ExtractLastElement, Op0_t > m_ExtractLastElement(const Op0_t &Op0)
class_match< VPValue > m_VPValue()
Match an arbitrary VPValue and ignore it.
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
Add a small namespace to avoid name clashes with the classes used in the streaming interface.
DiagnosticInfoOptimizationBase::Argument NV
NodeAddr< InstrNode * > Instr
Definition RDFGraph.h:389
NodeAddr< PhiNode * > Phi
Definition RDFGraph.h:390
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
bool isSingleScalar(const VPValue *VPV)
Returns true if VPV is a single scalar, either because it produces the same value for all lanes or on...
Definition VPlanUtils.h:44
VPValue * getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr)
Get or create a VPValue that corresponds to the expansion of Expr.
VPBasicBlock * getFirstLoopHeader(VPlan &Plan, VPDominatorTree &VPDT)
Returns the header block of the first, top-level loop, or null if none exist.
const SCEV * getSCEVExprForVPValue(VPValue *V, ScalarEvolution &SE)
Return the SCEV expression for V.
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI bool simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI, ScalarEvolution *SE, AssumptionCache *AC, MemorySSAUpdater *MSSAU, bool PreserveLCSSA)
Simplify each loop in a loop nest recursively.
LLVM_ABI void ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI, Instruction *I)
Replace the instruction specified by BI with the instruction specified by I.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:318
@ Offset
Definition DWP.cpp:477
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
Definition STLExtras.h:831
FunctionAddr VTableAddr Value
Definition InstrProf.h:137
LLVM_ABI Value * addRuntimeChecks(Instruction *Loc, Loop *TheLoop, const SmallVectorImpl< RuntimePointerCheck > &PointerChecks, SCEVExpander &Expander, bool HoistRuntimeChecks=false)
Add code that checks at runtime if the accessed arrays in PointerChecks overlap.
auto cast_if_present(const Y &Val)
cast_if_present<X> - Functionally identical to cast, except that a null value is accepted.
Definition Casting.h:689
LLVM_ABI bool RemoveRedundantDbgInstrs(BasicBlock *BB)
Try to remove redundant dbg.value instructions from given basic block.
cl::opt< bool > VerifyEachVPlan
LLVM_ABI std::optional< unsigned > getLoopEstimatedTripCount(Loop *L, unsigned *EstimatedLoopInvocationWeight=nullptr)
Return either:
static void reportVectorization(OptimizationRemarkEmitter *ORE, Loop *TheLoop, VectorizationFactor VF, unsigned IC)
Report successful vectorization of the loop.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1705
unsigned getLoadStoreAddressSpace(const Value *I)
A helper function that returns the address space of the pointer operand of load or store instruction.
LLVM_ABI Intrinsic::ID getMinMaxReductionIntrinsicOp(Intrinsic::ID RdxID)
Returns the min/max intrinsic used when expanding a min/max reduction.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1657
LLVM_ABI_FOR_TEST bool verifyVPlanIsValid(const VPlan &Plan, bool VerifyLate=false)
Verify invariants for general VPlans.
LLVM_ABI Intrinsic::ID getVectorIntrinsicIDForCall(const CallInst *CI, const TargetLibraryInfo *TLI)
Returns intrinsic ID for call.
InstructionCost Cost
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:649
LLVM_ABI bool verifyFunction(const Function &F, raw_ostream *OS=nullptr)
Check a function for errors, useful for use when debugging a pass.
const Value * getLoadStorePointerOperand(const Value *V)
A helper function that returns the pointer operand of a load or store instruction.
OuterAnalysisManagerProxy< ModuleAnalysisManager, Function > ModuleAnalysisManagerFunctionProxy
Provide the ModuleAnalysisManager to Function proxy.
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
LLVM_ABI bool formLCSSARecursively(Loop &L, const DominatorTree &DT, const LoopInfo *LI, ScalarEvolution *SE)
Put a loop nest into LCSSA form.
Definition LCSSA.cpp:449
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition STLExtras.h:2116
LLVM_ABI bool shouldOptimizeForSize(const MachineFunction *MF, ProfileSummaryInfo *PSI, const MachineBlockFrequencyInfo *BFI, PGSOQueryType QueryType=PGSOQueryType::Other)
Returns true if machine function MF is suggested to be size-optimized based on the profile.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:634
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:293
Align getLoadStoreAlignment(const Value *I)
A helper function that returns the alignment of load or store instruction.
iterator_range< df_iterator< VPBlockShallowTraversalWrapper< VPBlockBase * > > > vp_depth_first_shallow(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order.
Definition VPlanCFG.h:216
LLVM_ABI bool VerifySCEV
LLVM_ABI bool isSafeToSpeculativelyExecute(const Instruction *I, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr, bool UseVariableInfo=true, bool IgnoreUBImplyingAttrs=true)
Return true if the instruction does not have any effects besides calculating the result and does not ...
bool isa_and_nonnull(const Y &Val)
Definition Casting.h:682
iterator_range< df_iterator< VPBlockDeepTraversalWrapper< VPBlockBase * > > > vp_depth_first_deep(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order while traversing t...
Definition VPlanCFG.h:243
SmallVector< VPRegisterUsage, 8 > calculateRegisterUsageForPlan(VPlan &Plan, ArrayRef< ElementCount > VFs, const TargetTransformInfo &TTI, const SmallPtrSetImpl< const Value * > &ValuesToIgnore)
Estimate the register usage for Plan and vectorization factors in VFs by calculating the highest numb...
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:348
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:759
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1712
void collectEphemeralRecipesForVPlan(VPlan &Plan, DenseSet< VPRecipeBase * > &EphRecipes)
auto reverse(ContainerTy &&C)
Definition STLExtras.h:408
LLVM_ABI void setBranchWeights(Instruction &I, ArrayRef< uint32_t > Weights, bool IsExpected)
Create a new branch_weights metadata node and add or overwrite a prof metadata reference to instructi...
bool containsIrreducibleCFG(RPOTraversalT &RPOTraversal, const LoopInfoT &LI)
Return true if the control flow in RPOTraversal is irreducible.
Definition CFG.h:149
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:288
void sort(IteratorTy Start, IteratorTy End)
Definition STLExtras.h:1624
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:207
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1719
LLVM_ABI cl::opt< bool > EnableLoopVectorization
LLVM_ABI bool wouldInstructionBeTriviallyDead(const Instruction *I, const TargetLibraryInfo *TLI=nullptr)
Return true if the result produced by the instruction would have no side effects if it was not used.
Definition Local.cpp:421
FunctionAddr VTableAddr Count
Definition InstrProf.h:139
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
Type * toVectorizedTy(Type *Ty, ElementCount EC)
A helper for converting to vectorized types.
LLVM_ABI void llvm_unreachable_internal(const char *msg=nullptr, const char *file=nullptr, unsigned line=0)
This function calls abort(), and prints the optional message to stderr.
T * find_singleton(R &&Range, Predicate P, bool AllowRepeats=false)
Return the single value in Range that satisfies P(<member of Range> *, AllowRepeats)->T * returning n...
Definition STLExtras.h:1767
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
cl::opt< unsigned > ForceTargetInstructionCost
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:548
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
Definition Format.h:126
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
Definition STLExtras.h:325
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
Definition MathExtras.h:405
bool canVectorizeTy(Type *Ty)
Returns true if Ty is a valid vector element type, void, or an unpacked literal struct where all elem...
TargetTransformInfo TTI
static void reportVectorizationInfo(const StringRef Msg, const StringRef ORETag, OptimizationRemarkEmitter *ORE, Loop *TheLoop, Instruction *I=nullptr, DebugLoc DL={})
Reports an informative message: print Msg for debugging purposes as well as an optimization remark.
LLVM_ABI bool isAssignmentTrackingEnabled(const Module &M)
Return true if assignment tracking is enabled for module M.
RecurKind
These are the kinds of recurrences that we support.
@ Or
Bitwise or logical OR of integers.
@ FMulAdd
Sum of float products with llvm.fmuladd(a * b + sum).
@ Sub
Subtraction of integers.
@ AddChainWithSubs
A chain of adds and subs.
LLVM_ABI Value * getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF)
Given information about an recurrence kind, return the identity for the @llvm.vector....
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition Alignment.h:155
LLVM_ABI void reportVectorizationFailure(const StringRef DebugMsg, const StringRef OREMsg, const StringRef ORETag, OptimizationRemarkEmitter *ORE, Loop *TheLoop, Instruction *I=nullptr)
Reports a vectorization failure: print DebugMsg for debugging purposes along with the corresponding o...
DWARFExpression::Operation Op
ScalarEpilogueLowering
@ CM_ScalarEpilogueNotAllowedLowTripLoop
@ CM_ScalarEpilogueNotNeededUsePredicate
@ CM_ScalarEpilogueNotAllowedOptSize
@ CM_ScalarEpilogueAllowed
@ CM_ScalarEpilogueNotAllowedUsePredicate
LLVM_ABI bool isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Return true if this function can prove that V does not have undef bits and is never poison.
ArrayRef(const T &OneElt) -> ArrayRef< T >
Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)
Return a value for Step multiplied by VF.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:565
LLVM_ABI BasicBlock * SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="", bool Before=false)
Split the specified block at the specified instruction.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1738
auto predecessors(const MachineBasicBlock *BB)
iterator_range< pointer_iterator< WrappedIteratorT > > make_pointer_range(RangeT &&Range)
Definition iterator.h:363
cl::opt< bool > EnableVPlanNativePath
Definition VPlan.cpp:56
Type * getLoadStoreType(const Value *I)
A helper function that returns the type of a load or store instruction.
ArrayRef< Type * > getContainedTypes(Type *const &Ty)
Returns the types contained in Ty.
LLVM_ABI Value * addDiffRuntimeChecks(Instruction *Loc, ArrayRef< PointerDiffInfo > Checks, SCEVExpander &Expander, function_ref< Value *(IRBuilderBase &, unsigned)> GetVF, unsigned IC)
bool pred_empty(const BasicBlock *BB)
Definition CFG.h:119
@ DataAndControlFlowWithoutRuntimeCheck
Use predicate to control both data and control flow, but modify the trip count so that a runtime over...
@ None
Don't use tail folding.
@ DataWithEVL
Use predicated EVL instructions for tail-folding.
@ DataAndControlFlow
Use predicate to control both data and control flow.
@ DataWithoutLaneMask
Same as Data, but avoids using the get.active.lane.mask intrinsic to calculate the mask and instead i...
@ Data
Use predicate only to mask operations on data in the loop.
unsigned getPredBlockCostDivisor(TargetTransformInfo::TargetCostKind CostKind)
A helper function that returns how much we should divide the cost of a predicated block by.
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
LLVM_ABI bool hasBranchWeightMD(const Instruction &I)
Checks if an instructions has Branch Weight Metadata.
hash_code hash_combine(const Ts &...args)
Combine values into a single hash_code.
Definition Hashing.h:592
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
Definition bit.h:299
Type * toVectorTy(Type *Scalar, ElementCount EC)
A helper function for converting Scalar types to vector types.
std::unique_ptr< VPlan > VPlanPtr
Definition VPlan.h:77
constexpr detail::IsaCheckPredicate< Types... > IsaPred
Function object wrapper for the llvm::isa type check.
Definition Casting.h:836
LLVM_ABI MapVector< Instruction *, uint64_t > computeMinimumValueSizes(ArrayRef< BasicBlock * > Blocks, DemandedBits &DB, const TargetTransformInfo *TTI=nullptr)
Compute a map of integer instructions to their minimum legal type size.
hash_code hash_combine_range(InputIteratorT first, InputIteratorT last)
Compute a hash_code for a sequence of values.
Definition Hashing.h:466
LLVM_ABI cl::opt< bool > EnableLoopInterleaving
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:853
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
A special type used by analysis passes to provide an address that identifies that particular analysis...
Definition Analysis.h:29
static LLVM_ABI void collectEphemeralValues(const Loop *L, AssumptionCache *AC, SmallPtrSetImpl< const Value * > &EphValues)
Collect a loop's ephemeral values (those used only by an assume or similar intrinsics in the loop).
An information struct used to provide DenseMap with the various necessary components for a given valu...
Encapsulate information regarding vectorization of a loop and its epilogue.
EpilogueLoopVectorizationInfo(ElementCount MVF, unsigned MUF, ElementCount EVF, unsigned EUF, VPlan &EpiloguePlan)
A class that represents two vectorization factors (initialized with 0 by default).
static FixedScalableVFPair getNone()
This holds details about a histogram operation – a load -> update -> store sequence where each lane i...
Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...
TargetLibraryInfo * TLI
LLVM_ABI LoopVectorizeResult runImpl(Function &F)
LLVM_ABI bool processLoop(Loop *L)
ProfileSummaryInfo * PSI
LoopAccessInfoManager * LAIs
LLVM_ABI void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
LLVM_ABI LoopVectorizePass(LoopVectorizeOptions Opts={})
BlockFrequencyInfo * BFI
ScalarEvolution * SE
AssumptionCache * AC
LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
OptimizationRemarkEmitter * ORE
TargetTransformInfo * TTI
Storage for information about made changes.
A chain of instructions that form a partial reduction.
Instruction * Reduction
The top-level binary operation that forms the reduction to a scalar after the loop body.
Instruction * ExtendA
The extension of each of the inner binary operation's operands.
A CRTP mix-in to automatically provide informational APIs needed for passes.
Definition PassManager.h:70
A marker analysis to determine if extra passes should be run after loop vectorization.
static LLVM_ABI AnalysisKey Key
Holds the VFShape for a specific scalar to vector function mapping.
std::optional< unsigned > getParamIndexForOptionalMask() const
Instruction Set Architecture.
Encapsulates information needed to describe a parameter.
A range of powers-of-2 vectorization factors with fixed start and adjustable end.
ElementCount End
Struct to hold various analysis needed for cost computations.
LoopVectorizationCostModel & CM
bool isLegacyUniformAfterVectorization(Instruction *I, ElementCount VF) const
Return true if I is considered uniform-after-vectorization in the legacy cost model for VF.
bool skipCostComputation(Instruction *UI, bool IsVector) const
Return true if the cost for UI shouldn't be computed, e.g.
InstructionCost getLegacyCost(Instruction *UI, ElementCount VF) const
Return the cost for UI with VF using the legacy cost model as fallback until computing the cost of al...
SmallPtrSet< Instruction *, 8 > SkipCostComputation
A recipe for handling first-order recurrence phis.
Definition VPlan.h:2292
A struct that represents some properties of the register usage of a loop.
VPTransformState holds information passed down when "executing" a VPlan, needed for generating the ou...
A recipe for widening select instructions.
Definition VPlan.h:1719
static void materializeBroadcasts(VPlan &Plan)
Add explicit broadcasts for live-ins and VPValues defined in Plan's entry block if they are used as v...
static void materializeBackedgeTakenCount(VPlan &Plan, VPBasicBlock *VectorPH)
Materialize the backedge-taken count to be computed explicitly using VPInstructions.
static LLVM_ABI_FOR_TEST std::unique_ptr< VPlan > buildVPlan0(Loop *TheLoop, LoopInfo &LI, Type *InductionTy, DebugLoc IVDL, PredicatedScalarEvolution &PSE)
Create a base VPlan0, serving as the common starting point for all later candidates.
static void optimizeInductionExitUsers(VPlan &Plan, DenseMap< VPValue *, VPValue * > &EndValues, ScalarEvolution &SE)
If there's a single exit block, optimize its phi recipes that use exiting IV values by feeding them p...
static LLVM_ABI_FOR_TEST void handleEarlyExits(VPlan &Plan, bool HasUncountableExit)
Update Plan to account for all early exits.
static void canonicalizeEVLLoops(VPlan &Plan)
Transform EVL loops to use variable-length stepping after region dissolution.
static void dropPoisonGeneratingRecipes(VPlan &Plan, const std::function< bool(BasicBlock *)> &BlockNeedsPredication)
Drop poison flags from recipes that may generate a poison value that is used after vectorization,...
static void createInterleaveGroups(VPlan &Plan, const SmallPtrSetImpl< const InterleaveGroup< Instruction > * > &InterleaveGroups, VPRecipeBuilder &RecipeBuilder, const bool &ScalarEpilogueAllowed)
static bool runPass(bool(*Transform)(VPlan &, ArgsTy...), VPlan &Plan, typename std::remove_reference< ArgsTy >::type &...Args)
Helper to run a VPlan transform Transform on VPlan, forwarding extra arguments to the transform.
static void addBranchWeightToMiddleTerminator(VPlan &Plan, ElementCount VF, std::optional< unsigned > VScaleForTuning)
Add branch weight metadata, if the Plan's middle block is terminated by a BranchOnCond recipe.
static void materializeBuildVectors(VPlan &Plan)
Add explicit Build[Struct]Vector recipes that combine multiple scalar values into single vectors.
static void unrollByUF(VPlan &Plan, unsigned UF)
Explicitly unroll Plan by UF.
static DenseMap< const SCEV *, Value * > expandSCEVs(VPlan &Plan, ScalarEvolution &SE)
Expand VPExpandSCEVRecipes in Plan's entry block.
static void convertToConcreteRecipes(VPlan &Plan)
Lower abstract recipes to concrete ones, that can be codegen'd.
static void addMinimumIterationCheck(VPlan &Plan, ElementCount VF, unsigned UF, ElementCount MinProfitableTripCount, bool RequiresScalarEpilogue, bool TailFolded, bool CheckNeededWithTailFolding, Loop *OrigLoop, const uint32_t *MinItersBypassWeights, DebugLoc DL, ScalarEvolution &SE)
static void convertToAbstractRecipes(VPlan &Plan, VPCostContext &Ctx, VFRange &Range)
This function converts initial recipes to the abstract recipes and clamps Range based on cost model f...
static void materializeConstantVectorTripCount(VPlan &Plan, ElementCount BestVF, unsigned BestUF, PredicatedScalarEvolution &PSE)
static DenseMap< VPBasicBlock *, VPValue * > introduceMasksAndLinearize(VPlan &Plan, bool FoldTail)
Predicate and linearize the control-flow in the only loop region of Plan.
static void addExplicitVectorLength(VPlan &Plan, const std::optional< unsigned > &MaxEVLSafeElements)
Add a VPEVLBasedIVPHIRecipe and related recipes to Plan and replaces all uses except the canonical IV...
static void replaceSymbolicStrides(VPlan &Plan, PredicatedScalarEvolution &PSE, const DenseMap< Value *, const SCEV * > &StridesMap)
Replace symbolic strides from StridesMap in Plan with constants when possible.
static bool handleMaxMinNumReductions(VPlan &Plan)
Check if Plan contains any FMaxNum or FMinNum reductions.
static void removeBranchOnConst(VPlan &Plan)
Remove BranchOnCond recipes with true or false conditions together with removing dead edges to their ...
static LLVM_ABI_FOR_TEST void createLoopRegions(VPlan &Plan)
Replace loops in Plan's flat CFG with VPRegionBlocks, turning Plan's flat CFG into a hierarchical CFG...
static void removeDeadRecipes(VPlan &Plan)
Remove dead recipes from Plan.
static void attachCheckBlock(VPlan &Plan, Value *Cond, BasicBlock *CheckBlock, bool AddBranchWeights)
Wrap runtime check block CheckBlock in a VPIRBB and Cond in a VPValue and connect the block to Plan,...
static void materializeVectorTripCount(VPlan &Plan, VPBasicBlock *VectorPHVPBB, bool TailByMasking, bool RequiresScalarEpilogue)
Materialize vector trip count computations to a set of VPInstructions.
static void simplifyRecipes(VPlan &Plan)
Perform instcombine-like simplifications on recipes in Plan.
static LLVM_ABI_FOR_TEST bool tryToConvertVPInstructionsToVPRecipes(VPlanPtr &Plan, function_ref< const InductionDescriptor *(PHINode *)> GetIntOrFpInductionDescriptor, const TargetLibraryInfo &TLI)
Replaces the VPInstructions in Plan with corresponding widen recipes.
static void replicateByVF(VPlan &Plan, ElementCount VF)
Replace each replicating VPReplicateRecipe and VPInstruction outside of any replicate region in Plan ...
static void clearReductionWrapFlags(VPlan &Plan)
Clear NSW/NUW flags from reduction instructions if necessary.
static void cse(VPlan &Plan)
Perform common-subexpression-elimination on Plan.
static void addActiveLaneMask(VPlan &Plan, bool UseActiveLaneMaskForControlFlow, bool DataAndControlFlowWithoutRuntimeCheck)
Replace (ICMP_ULE, wide canonical IV, backedge-taken-count) checks with an (active-lane-mask recipe,...
static void optimize(VPlan &Plan)
Apply VPlan-to-VPlan optimizations to Plan, including induction recipe optimizations,...
static void dissolveLoopRegions(VPlan &Plan)
Replace loop regions with explicit CFG.
static void narrowInterleaveGroups(VPlan &Plan, ElementCount VF, unsigned VectorRegWidth)
Try to convert a plan with interleave groups with VF elements to a plan with the interleave groups re...
static void truncateToMinimalBitwidths(VPlan &Plan, const MapVector< Instruction *, uint64_t > &MinBWs)
Insert truncates and extends for any truncated recipe.
static bool adjustFixedOrderRecurrences(VPlan &Plan, VPBuilder &Builder)
Try to have all users of fixed-order recurrences appear after the recipe defining their previous valu...
static void optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF, unsigned BestUF, PredicatedScalarEvolution &PSE)
Optimize Plan based on BestVF and BestUF.
static void materializeVFAndVFxUF(VPlan &Plan, VPBasicBlock *VectorPH, ElementCount VF)
Materialize VF and VFxUF to be computed explicitly using VPInstructions.
static LLVM_ABI_FOR_TEST void addMiddleCheck(VPlan &Plan, bool RequiresScalarEpilogueCheck, bool TailFolded)
If a check is needed to guard executing the scalar epilogue loop, it will be added to the middle bloc...
TODO: The following VectorizationFactor was pulled out of LoopVectorizationCostModel class.
InstructionCost Cost
Cost of the loop with that width.
ElementCount MinProfitableTripCount
The minimum trip count required to make vectorization profitable, e.g.
ElementCount Width
Vector width with best cost.
InstructionCost ScalarCost
Cost of the scalar loop.
static VectorizationFactor Disabled()
Width 1 means no vectorization, cost 0 means uncomputed cost.
static LLVM_ABI bool HoistRuntimeChecks