83#include "llvm/Config/llvm-config.h"
138#define DEBUG_TYPE "scalar-evolution"
141 "Number of loop exits with predictable exit counts");
143 "Number of loop exits without predictable exit counts");
145 "Number of loops with trip counts computed by force");
147#ifdef EXPENSIVE_CHECKS
155 cl::desc(
"Maximum number of iterations SCEV will "
156 "symbolically execute a constant "
162 cl::desc(
"Verify ScalarEvolution's backedge taken counts (slow)"));
165 cl::desc(
"Enable stricter verification with -verify-scev is passed"));
169 cl::desc(
"Verify IR correctness when making sensitive SCEV queries (slow)"),
174 cl::desc(
"Threshold for inlining multiplication operands into a SCEV"),
179 cl::desc(
"Threshold for inlining addition operands into a SCEV"),
183 "scalar-evolution-max-scev-compare-depth",
cl::Hidden,
184 cl::desc(
"Maximum depth of recursive SCEV complexity comparisons"),
188 "scalar-evolution-max-scev-operations-implication-depth",
cl::Hidden,
189 cl::desc(
"Maximum depth of recursive SCEV operations implication analysis"),
193 "scalar-evolution-max-value-compare-depth",
cl::Hidden,
194 cl::desc(
"Maximum depth of recursive value complexity comparisons"),
199 cl::desc(
"Maximum depth of recursive arithmetics"),
203 "scalar-evolution-max-constant-evolving-depth",
cl::Hidden,
208 cl::desc(
"Maximum depth of recursive SExt/ZExt/Trunc"),
213 cl::desc(
"Max coefficients in AddRec during evolving"),
218 cl::desc(
"Size of the expression which is considered huge"),
223 cl::desc(
"Threshold for switching to iteratively computing SCEV ranges"),
227 "scalar-evolution-max-loop-guard-collection-depth",
cl::Hidden,
228 cl::desc(
"Maximum depth for recursive loop guard collection"),
cl::init(1));
233 cl::desc(
"When printing analysis, include information on every instruction"));
236 "scalar-evolution-use-expensive-range-sharpening",
cl::Hidden,
238 cl::desc(
"Use more powerful methods of sharpening expression ranges. May "
239 "be costly in terms of compile time"));
242 "scalar-evolution-max-scc-analysis-depth",
cl::Hidden,
243 cl::desc(
"Maximum amount of nodes to process while searching SCEVUnknown "
244 "Phi strongly connected components"),
249 cl::desc(
"Handle <= and >= in finite loops"),
253 "scalar-evolution-use-context-for-no-wrap-flag-strenghening",
cl::Hidden,
254 cl::desc(
"Infer nuw/nsw flags using context where suitable"),
265#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
283 OS <<
"(ptrtoint " << *
Op->getType() <<
" " << *
Op <<
" to "
284 << *PtrToInt->
getType() <<
")";
290 OS <<
"(trunc " << *
Op->getType() <<
" " << *
Op <<
" to "
297 OS <<
"(zext " << *
Op->getType() <<
" " << *
Op <<
" to "
304 OS <<
"(sext " << *
Op->getType() <<
" " << *
Op <<
" to "
333 const char *OpStr =
nullptr;
346 OpStr =
" umin_seq ";
370 OS <<
"(" << *UDiv->
getLHS() <<
" /u " << *UDiv->
getRHS() <<
")";
377 OS <<
"***COULDNOTCOMPUTE***";
453 if (!
Mul)
return false;
457 if (!SC)
return false;
475 if (
const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(
ID, IP))
return S;
477 UniqueSCEVs.InsertNode(S, IP);
496 if (
const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(
ID, IP))
499 UniqueSCEVs.InsertNode(S, IP);
519 "Must be a non-bit-width-changing pointer-to-integer cast!");
531 "Cannot truncate non-integer value!");
538 "Cannot zero extend non-integer value!");
545 "Cannot sign extend non-integer value!");
550 SE->forgetMemoizedResults(
this);
553 SE->UniqueSCEVs.RemoveNode(
this);
559void SCEVUnknown::allUsesReplacedWith(
Value *New) {
561 SE->forgetMemoizedResults(
this);
564 SE->UniqueSCEVs.RemoveNode(
this);
586 if (LIsPointer != RIsPointer)
587 return (
int)LIsPointer - (int)RIsPointer;
592 return (
int)LID - (int)RID;
597 unsigned LArgNo = LA->getArgNo(), RArgNo =
RA->getArgNo();
598 return (
int)LArgNo - (int)RArgNo;
604 if (
auto L = LGV->getLinkage() - RGV->getLinkage())
607 const auto IsGVNameSemantic = [&](
const GlobalValue *GV) {
608 auto LT = GV->getLinkage();
615 if (IsGVNameSemantic(LGV) && IsGVNameSemantic(RGV))
616 return LGV->getName().compare(RGV->getName());
627 if (LParent != RParent) {
630 if (LDepth != RDepth)
631 return (
int)LDepth - (int)RDepth;
635 unsigned LNumOps = LInst->getNumOperands(),
636 RNumOps = RInst->getNumOperands();
637 if (LNumOps != RNumOps)
638 return (
int)LNumOps - (int)RNumOps;
640 for (
unsigned Idx :
seq(LNumOps)) {
642 RInst->getOperand(Idx),
Depth + 1);
656static std::optional<int>
666 return (
int)LType - (int)RType;
691 unsigned LBitWidth = LA.
getBitWidth(), RBitWidth =
RA.getBitWidth();
692 if (LBitWidth != RBitWidth)
693 return (
int)LBitWidth - (int)RBitWidth;
694 return LA.
ult(
RA) ? -1 : 1;
700 return LTy->getBitWidth() - RTy->getBitWidth();
711 if (LLoop != RLoop) {
713 assert(LHead != RHead &&
"Two loops share the same header?");
717 "No dominance between recurrences used by one SCEV?");
740 unsigned LNumOps = LOps.
size(), RNumOps = ROps.
size();
741 if (LNumOps != RNumOps)
742 return (
int)LNumOps - (int)RNumOps;
744 for (
unsigned i = 0; i != LNumOps; ++i) {
769 if (
Ops.size() < 2)
return;
774 return Complexity && *Complexity < 0;
776 if (
Ops.size() == 2) {
780 if (IsLessComplex(
RHS,
LHS))
787 return IsLessComplex(
LHS,
RHS);
794 for (
unsigned i = 0, e =
Ops.size(); i != e-2; ++i) {
800 for (
unsigned j = i+1; j != e &&
Ops[j]->getSCEVType() == Complexity; ++j) {
805 if (i == e-2)
return;
827template <
typename FoldT,
typename IsIdentityT,
typename IsAbsorberT>
831 IsIdentityT IsIdentity, IsAbsorberT IsAbsorber) {
833 for (
unsigned Idx = 0; Idx <
Ops.size();) {
841 Ops.erase(
Ops.begin() + Idx);
848 assert(Folded &&
"Must have folded value");
852 if (Folded && IsAbsorber(Folded->
getAPInt()))
856 if (Folded && !IsIdentity(Folded->
getAPInt()))
857 Ops.insert(
Ops.begin(), Folded);
859 return Ops.size() == 1 ?
Ops[0] :
nullptr;
934 APInt OddFactorial(W, 1);
936 for (
unsigned i = 3; i <= K; ++i) {
939 OddFactorial *= (i >> TwoFactors);
943 unsigned CalculationBits = W +
T;
957 for (
unsigned i = 1; i != K; ++i) {
990 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
1010 "getLosslessPtrToIntExpr() should self-recurse at most once.");
1014 if (!
Op->getType()->isPointerTy())
1025 if (
const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(
ID, IP))
1055 SCEV *S =
new (SCEVAllocator)
1057 UniqueSCEVs.InsertNode(S, IP);
1062 assert(
Depth == 0 &&
"getLosslessPtrToIntExpr() should not self-recurse for "
1063 "non-SCEVUnknown's.");
1075 class SCEVPtrToIntSinkingRewriter
1083 SCEVPtrToIntSinkingRewriter
Rewriter(SE);
1084 return Rewriter.visit(Scev);
1093 return Base::visit(S);
1118 "Should only reach pointer-typed SCEVUnknown's.");
1124 const SCEV *IntOp = SCEVPtrToIntSinkingRewriter::rewrite(
Op, *
this);
1126 "We must have succeeded in sinking the cast, "
1127 "and ending up with an integer-typed expression!");
1132 assert(Ty->isIntegerTy() &&
"Target type must be an integer type!");
1144 "This is not a truncating conversion!");
1146 "This is not a conversion to a SCEVable type!");
1147 assert(!
Op->getType()->isPointerTy() &&
"Can't truncate pointer!");
1155 if (
const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(
ID, IP))
return S;
1177 UniqueSCEVs.InsertNode(S, IP);
1189 unsigned numTruncs = 0;
1190 for (
unsigned i = 0, e = CommOp->getNumOperands(); i != e && numTruncs < 2;
1198 if (numTruncs < 2) {
1208 if (
const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(
ID, IP))
1215 for (
const SCEV *
Op : AddRec->operands())
1230 UniqueSCEVs.InsertNode(S, IP);
1270struct ExtendOpTraitsBase {
1271 typedef const SCEV *(ScalarEvolution::*GetExtendExprTy)(
const SCEV *,
Type *,
1276template <
typename ExtendOp>
struct ExtendOpTraits {
1292 static const GetExtendExprTy GetExtendExpr;
1294 static const SCEV *getOverflowLimitForStep(
const SCEV *Step,
1295 ICmpInst::Predicate *Pred,
1296 ScalarEvolution *SE) {
1301const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits<
1308 static const GetExtendExprTy GetExtendExpr;
1310 static const SCEV *getOverflowLimitForStep(
const SCEV *Step,
1311 ICmpInst::Predicate *Pred,
1312 ScalarEvolution *SE) {
1317const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits<
1329template <
typename ExtendOpTy>
1332 auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType;
1333 auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr;
1349 for (
auto It = DiffOps.
begin(); It != DiffOps.
end(); ++It)
1362 auto PreStartFlags =
1380 const SCEV *OperandExtendedStart =
1382 (SE->*GetExtendExpr)(Step, WideTy,
Depth));
1383 if ((SE->*GetExtendExpr)(Start, WideTy,
Depth) == OperandExtendedStart) {
1395 const SCEV *OverflowLimit =
1396 ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep(Step, &Pred, SE);
1398 if (OverflowLimit &&
1406template <
typename ExtendOpTy>
1410 auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr;
1418 (SE->*GetExtendExpr)(PreStart, Ty,
Depth));
1453template <
typename ExtendOpTy>
1454bool ScalarEvolution::proveNoWrapByVaryingStart(
const SCEV *Start,
1457 auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType;
1467 APInt StartAI = StartC->
getAPInt();
1469 for (
unsigned Delta : {-2, -1, 1, 2}) {
1470 const SCEV *PreStart =
getConstant(StartAI - Delta);
1472 FoldingSetNodeID
ID;
1474 ID.AddPointer(PreStart);
1475 ID.AddPointer(Step);
1479 static_cast<SCEVAddRecExpr *
>(UniqueSCEVs.FindNodeOrInsertPos(
ID, IP));
1483 if (PreAR && PreAR->getNoWrapFlags(WrapType)) {
1486 const SCEV *Limit = ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep(
1487 DeltaS, &Pred,
this);
1505 const unsigned BitWidth =
C.getBitWidth();
1523 const APInt &ConstantStart,
1542 auto &UserIDs = FoldCacheUser[
I.first->second];
1543 assert(
count(UserIDs,
ID) == 1 &&
"unexpected duplicates in UserIDs");
1544 for (
unsigned I = 0;
I != UserIDs.size(); ++
I)
1545 if (UserIDs[
I] ==
ID) {
1550 I.first->second = S;
1552 FoldCacheUser[S].push_back(
ID);
1558 "This is not an extending conversion!");
1560 "This is not a conversion to a SCEVable type!");
1561 assert(!
Op->getType()->isPointerTy() &&
"Can't extend pointer!");
1565 if (
const SCEV *S = FoldCache.lookup(
ID))
1577 "This is not an extending conversion!");
1579 assert(!
Op->getType()->isPointerTy() &&
"Can't extend pointer!");
1596 if (
const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(
ID, IP))
return S;
1600 UniqueSCEVs.InsertNode(S, IP);
1609 const SCEV *
X = ST->getOperand();
1623 if (AR->isAffine()) {
1624 const SCEV *Start = AR->getStart();
1625 const SCEV *Step = AR->getStepRecurrence(*
this);
1627 const Loop *L = AR->getLoop();
1631 if (AR->hasNoUnsignedWrap()) {
1652 const SCEV *CastedMaxBECount =
1656 if (MaxBECount == RecastedMaxBECount) {
1666 const SCEV *WideMaxBECount =
1668 const SCEV *OperandExtendedAdd =
1674 if (ZAdd == OperandExtendedAdd) {
1685 OperandExtendedAdd =
1691 if (ZAdd == OperandExtendedAdd) {
1712 !AC.assumptions().empty()) {
1714 auto NewFlags = proveNoUnsignedWrapViaInduction(AR);
1716 if (AR->hasNoUnsignedWrap()) {
1751 const APInt &
C = SC->getAPInt();
1755 const SCEV *SResidual =
1764 if (proveNoWrapByVaryingStart<SCEVZeroExtendExpr>(Start, Step, L)) {
1777 if (matchURem(
Op, LHS, RHS))
1789 if (SA->hasNoUnsignedWrap()) {
1793 for (
const auto *
Op : SA->operands())
1810 const SCEV *SResidual =
1822 if (SM->hasNoUnsignedWrap()) {
1826 for (
const auto *
Op : SM->operands())
1843 if (SM->getNumOperands() == 2)
1845 if (MulLHS->getAPInt().isPowerOf2())
1848 MulLHS->getAPInt().logBase2();
1863 for (
auto *Operand :
MinMax->operands())
1874 for (
auto *Operand :
MinMax->operands())
1881 if (
const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(
ID, IP))
return S;
1884 UniqueSCEVs.InsertNode(S, IP);
1892 "This is not an extending conversion!");
1894 "This is not a conversion to a SCEVable type!");
1895 assert(!
Op->getType()->isPointerTy() &&
"Can't extend pointer!");
1899 if (
const SCEV *S = FoldCache.lookup(
ID))
1911 "This is not an extending conversion!");
1913 assert(!
Op->getType()->isPointerTy() &&
"Can't extend pointer!");
1935 if (
const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(
ID, IP))
return S;
1940 UniqueSCEVs.InsertNode(S, IP);
1949 const SCEV *
X = ST->getOperand();
1960 if (SA->hasNoSignedWrap()) {
1964 for (
const auto *
Op : SA->operands())
1982 const SCEV *SResidual =
1996 if (AR->isAffine()) {
1997 const SCEV *Start = AR->getStart();
1998 const SCEV *Step = AR->getStepRecurrence(*
this);
2000 const Loop *L = AR->getLoop();
2004 if (AR->hasNoSignedWrap()) {
2026 const SCEV *CastedMaxBECount =
2030 if (MaxBECount == RecastedMaxBECount) {
2040 const SCEV *WideMaxBECount =
2042 const SCEV *OperandExtendedAdd =
2048 if (SAdd == OperandExtendedAdd) {
2059 OperandExtendedAdd =
2065 if (SAdd == OperandExtendedAdd) {
2085 auto NewFlags = proveNoSignedWrapViaInduction(AR);
2087 if (AR->hasNoSignedWrap()) {
2102 const APInt &
C = SC->getAPInt();
2106 const SCEV *SResidual =
2115 if (proveNoWrapByVaryingStart<SCEVSignExtendExpr>(Start, Step, L)) {
2134 for (
auto *Operand :
MinMax->operands())
2143 if (
const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(
ID, IP))
return S;
2146 UniqueSCEVs.InsertNode(S, IP);
2172 "This is not an extending conversion!");
2174 "This is not a conversion to a SCEVable type!");
2179 if (SC->getAPInt().isNegative())
2184 const SCEV *NewOp =
T->getOperand();
2203 for (
const SCEV *
Op : AR->operands())
2242 APInt &AccumulatedConstant,
2245 bool Interesting =
false;
2252 if (Scale != 1 || AccumulatedConstant != 0 ||
C->getValue()->isZero())
2254 AccumulatedConstant += Scale *
C->getAPInt();
2259 for (; i !=
Ops.size(); ++i) {
2269 Add->operands(), NewScale, SE);
2275 auto Pair = M.insert({
Key, NewScale});
2279 Pair.first->second += NewScale;
2287 std::pair<DenseMap<const SCEV *, APInt>::iterator,
bool> Pair =
2288 M.insert({
Ops[i], Scale});
2292 Pair.first->second += Scale;
2311 case Instruction::Add:
2314 case Instruction::Sub:
2317 case Instruction::Mul:
2331 const SCEV *
A = (this->*Extension)(
2333 const SCEV *LHSB = (this->*Extension)(LHS, WideTy, 0);
2334 const SCEV *RHSB = (this->*Extension)(RHS, WideTy, 0);
2342 if (BinOp == Instruction::Mul)
2348 APInt C = RHSC->getAPInt();
2349 unsigned NumBits =
C.getBitWidth();
2350 bool IsSub = (BinOp == Instruction::Sub);
2351 bool IsNegativeConst = (
Signed &&
C.isNegative());
2353 bool OverflowDown = IsSub ^ IsNegativeConst;
2355 if (IsNegativeConst) {
2368 APInt Limit = Min + Magnitude;
2374 APInt Limit = Max - Magnitude;
2379std::optional<SCEV::NoWrapFlags>
2384 return std::nullopt;
2393 bool Deduced =
false;
2395 if (OBO->
getOpcode() != Instruction::Add &&
2398 return std::nullopt;
2407 false, LHS, RHS, CtxI)) {
2414 true, LHS, RHS, CtxI)) {
2421 return std::nullopt;
2431 using namespace std::placeholders;
2438 assert(CanAnalyze &&
"don't call from other places!");
2445 auto IsKnownNonNegative = [&](
const SCEV *S) {
2455 if (SignOrUnsignWrap != SignOrUnsignMask &&
2462 return Instruction::Add;
2464 return Instruction::Mul;
2475 Opcode,
C, OBO::NoSignedWrap);
2483 Opcode,
C, OBO::NoUnsignedWrap);
2493 Ops[0]->isZero() && IsKnownNonNegative(
Ops[1]))
2500 if (UDiv->getOperand(1) ==
Ops[1])
2503 if (UDiv->getOperand(1) ==
Ops[0])
2519 "only nuw or nsw allowed");
2520 assert(!
Ops.empty() &&
"Cannot get empty add!");
2521 if (
Ops.size() == 1)
return Ops[0];
2524 for (
unsigned i = 1, e =
Ops.size(); i != e; ++i)
2526 "SCEVAddExpr operand types don't match!");
2528 Ops, [](
const SCEV *
Op) {
return Op->getType()->isPointerTy(); });
2529 assert(NumPtrs <= 1 &&
"add has at most one pointer operand");
2534 [](
const APInt &C1,
const APInt &C2) {
return C1 + C2; },
2535 [](
const APInt &
C) {
return C.isZero(); },
2536 [](
const APInt &
C) {
return false; });
2549 return getOrCreateAddExpr(
Ops, ComputeFlags(
Ops));
2554 if (
Add->getNoWrapFlags(OrigFlags) != OrigFlags)
2555 Add->setNoWrapFlags(ComputeFlags(
Ops));
2563 bool FoundMatch =
false;
2564 for (
unsigned i = 0, e =
Ops.size(); i != e-1; ++i)
2565 if (
Ops[i] ==
Ops[i+1]) {
2577 --i; e -=
Count - 1;
2587 auto FindTruncSrcType = [&]() ->
Type * {
2593 return T->getOperand()->getType();
2595 const auto *LastOp =
Mul->getOperand(
Mul->getNumOperands() - 1);
2597 return T->getOperand()->getType();
2601 if (
auto *SrcType = FindTruncSrcType()) {
2608 if (
T->getOperand()->getType() != SrcType) {
2617 for (
unsigned j = 0, f = M->getNumOperands(); j != f && Ok; ++j) {
2620 if (
T->getOperand()->getType() != SrcType) {
2648 if (
Ops.size() == 2) {
2658 auto C2 =
C->getAPInt();
2661 APInt ConstAdd = C1 + C2;
2662 auto AddFlags = AddExpr->getNoWrapFlags();
2702 if (
Ops.size() == 2) {
2704 if (
Mul &&
Mul->getNumOperands() == 2 &&
2705 Mul->getOperand(0)->isAllOnesValue()) {
2708 if (matchURem(
Mul->getOperand(1),
X,
Y) &&
X ==
Ops[1]) {
2719 if (Idx <
Ops.size()) {
2720 bool DeletedAdd =
false;
2731 Ops.erase(
Ops.begin()+Idx);
2734 CommonFlags =
maskFlags(CommonFlags,
Add->getNoWrapFlags());
2757 struct APIntCompare {
2758 bool operator()(
const APInt &LHS,
const APInt &RHS)
const {
2759 return LHS.ult(RHS);
2766 std::map<APInt, SmallVector<const SCEV *, 4>, APIntCompare> MulOpLists;
2767 for (
const SCEV *NewOp : NewOps)
2768 MulOpLists[M.find(NewOp)->second].push_back(NewOp);
2771 if (AccumulatedConstant != 0)
2773 for (
auto &MulOp : MulOpLists) {
2774 if (MulOp.first == 1) {
2776 }
else if (MulOp.first != 0) {
2785 if (
Ops.size() == 1)
2796 for (
unsigned MulOp = 0, e =
Mul->getNumOperands(); MulOp != e; ++MulOp) {
2797 const SCEV *MulOpSCEV =
Mul->getOperand(MulOp);
2800 for (
unsigned AddOp = 0, e =
Ops.size(); AddOp != e; ++AddOp)
2801 if (MulOpSCEV ==
Ops[AddOp]) {
2803 const SCEV *InnerMul =
Mul->getOperand(MulOp == 0);
2804 if (
Mul->getNumOperands() != 2) {
2808 Mul->operands().take_front(MulOp));
2816 if (
Ops.size() == 2)
return OuterMul;
2818 Ops.erase(
Ops.begin()+AddOp);
2819 Ops.erase(
Ops.begin()+Idx-1);
2821 Ops.erase(
Ops.begin()+Idx);
2822 Ops.erase(
Ops.begin()+AddOp-1);
2824 Ops.push_back(OuterMul);
2829 for (
unsigned OtherMulIdx = Idx+1;
2836 OMulOp != e; ++OMulOp)
2837 if (OtherMul->
getOperand(OMulOp) == MulOpSCEV) {
2839 const SCEV *InnerMul1 =
Mul->getOperand(MulOp == 0);
2840 if (
Mul->getNumOperands() != 2) {
2842 Mul->operands().take_front(MulOp));
2849 OtherMul->
operands().take_front(OMulOp));
2854 const SCEV *InnerMulSum =
2858 if (
Ops.size() == 2)
return OuterMul;
2859 Ops.erase(
Ops.begin()+Idx);
2860 Ops.erase(
Ops.begin()+OtherMulIdx-1);
2861 Ops.push_back(OuterMul);
2881 for (
unsigned i = 0, e =
Ops.size(); i != e; ++i)
2884 Ops.erase(
Ops.begin()+i);
2889 if (!LIOps.
empty()) {
2914 auto *DefI = getDefiningScopeBound(LIOps);
2916 if (!isGuaranteedToTransferExecutionTo(DefI, ReachI))
2928 if (
Ops.size() == 1)
return NewRec;
2931 for (
unsigned i = 0;; ++i)
2932 if (
Ops[i] == AddRec) {
2942 for (
unsigned OtherIdx = Idx+1;
2950 "AddRecExprs are not sorted in reverse dominance order?");
2957 if (OtherAddRec->getLoop() == AddRecLoop) {
2958 for (
unsigned i = 0, e = OtherAddRec->getNumOperands();
2960 if (i >= AddRecOps.
size()) {
2961 append_range(AddRecOps, OtherAddRec->operands().drop_front(i));
2965 AddRecOps[i], OtherAddRec->getOperand(i)};
2968 Ops.erase(
Ops.begin() + OtherIdx); --OtherIdx;
2983 return getOrCreateAddExpr(
Ops, ComputeFlags(
Ops));
2995 static_cast<SCEVAddExpr *
>(UniqueSCEVs.FindNodeOrInsertPos(
ID, IP));
2999 S =
new (SCEVAllocator)
3001 UniqueSCEVs.InsertNode(S, IP);
3011 FoldingSetNodeID
ID;
3013 for (
const SCEV *
Op :
Ops)
3018 static_cast<SCEVAddRecExpr *
>(UniqueSCEVs.FindNodeOrInsertPos(
ID, IP));
3020 const SCEV **
O = SCEVAllocator.Allocate<
const SCEV *>(
Ops.size());
3022 S =
new (SCEVAllocator)
3023 SCEVAddRecExpr(
ID.Intern(SCEVAllocator), O,
Ops.size(), L);
3024 UniqueSCEVs.InsertNode(S, IP);
3025 LoopUsers[
L].push_back(S);
3035 FoldingSetNodeID
ID;
3037 for (
const SCEV *
Op :
Ops)
3041 static_cast<SCEVMulExpr *
>(UniqueSCEVs.FindNodeOrInsertPos(
ID, IP));
3043 const SCEV **
O = SCEVAllocator.Allocate<
const SCEV *>(
Ops.size());
3045 S =
new (SCEVAllocator) SCEVMulExpr(
ID.Intern(SCEVAllocator),
3047 UniqueSCEVs.InsertNode(S, IP);
3056 if (j > 1 && k / j != i) Overflow =
true;
3072 if (n == 0 || n == k)
return 1;
3073 if (k > n)
return 0;
3079 for (
uint64_t i = 1; i <= k; ++i) {
3080 r =
umul_ov(r, n-(i-1), Overflow);
3089 struct FindConstantInAddMulChain {
3090 bool FoundConstant =
false;
3092 bool follow(
const SCEV *S) {
3097 bool isDone()
const {
3098 return FoundConstant;
3102 FindConstantInAddMulChain
F;
3104 ST.visitAll(StartExpr);
3105 return F.FoundConstant;
3113 "only nuw or nsw allowed");
3114 assert(!
Ops.empty() &&
"Cannot get empty mul!");
3115 if (
Ops.size() == 1)
return Ops[0];
3117 Type *ETy =
Ops[0]->getType();
3119 for (
unsigned i = 1, e =
Ops.size(); i != e; ++i)
3121 "SCEVMulExpr operand types don't match!");
3126 [](
const APInt &C1,
const APInt &C2) {
return C1 * C2; },
3127 [](
const APInt &
C) {
return C.isOne(); },
3128 [](
const APInt &
C) {
return C.isZero(); });
3139 return getOrCreateMulExpr(
Ops, ComputeFlags(
Ops));
3144 if (
Mul->getNoWrapFlags(OrigFlags) != OrigFlags)
3145 Mul->setNoWrapFlags(ComputeFlags(
Ops));
3150 if (
Ops.size() == 2) {
3167 if (
Ops[0]->isAllOnesValue()) {
3172 bool AnyFolded =
false;
3173 for (
const SCEV *AddOp :
Add->operands()) {
3200 AddRec->getNoWrapFlags(FlagsMask));
3223 APInt C1V = LHSC->getAPInt();
3233 const SCEV *NewMul =
nullptr;
3237 assert(C1V.
ugt(1) &&
"C1 <= 1 should have been folded earlier");
3252 if (Idx <
Ops.size()) {
3253 bool DeletedMul =
false;
3259 Ops.erase(
Ops.begin()+Idx);
3283 for (
unsigned i = 0, e =
Ops.size(); i != e; ++i)
3286 Ops.erase(
Ops.begin()+i);
3291 if (!LIOps.
empty()) {
3304 for (
unsigned i = 0, e = AddRec->
getNumOperands(); i != e; ++i) {
3320 if (
Ops.size() == 1)
return NewRec;
3323 for (
unsigned i = 0;; ++i)
3324 if (
Ops[i] == AddRec) {
3345 bool OpsModified =
false;
3346 for (
unsigned OtherIdx = Idx+1;
3360 bool Overflow =
false;
3367 for (
int y = x, ye = 2*x+1; y != ye && !Overflow; ++y) {
3371 z < ze && !Overflow; ++z) {
3374 if (LargerThan64Bits)
3375 Coeff =
umul_ov(Coeff1, Coeff2, Overflow);
3377 Coeff = Coeff1*Coeff2;
3392 if (
Ops.size() == 2)
return NewAddRec;
3393 Ops[Idx] = NewAddRec;
3394 Ops.erase(
Ops.begin() + OtherIdx); --OtherIdx;
3410 return getOrCreateMulExpr(
Ops, ComputeFlags(
Ops));
3418 "SCEVURemExpr operand types don't match!");
3423 if (RHSC->getValue()->isOne())
3424 return getZero(LHS->getType());
3427 if (RHSC->getAPInt().isPowerOf2()) {
3428 Type *FullTy = LHS->getType();
3445 assert(!LHS->getType()->isPointerTy() &&
3446 "SCEVUDivExpr operand can't be pointer!");
3447 assert(LHS->getType() == RHS->getType() &&
3448 "SCEVUDivExpr operand types don't match!");
3455 if (
const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(
ID, IP))
3463 if (RHSC->getValue()->isOne())
3468 if (!RHSC->getValue()->isZero()) {
3472 Type *Ty = LHS->getType();
3473 unsigned LZ = RHSC->getAPInt().countl_zero();
3477 if (!RHSC->getAPInt().isPowerOf2())
3485 const APInt &StepInt = Step->getAPInt();
3486 const APInt &DivInt = RHSC->getAPInt();
3487 if (!StepInt.
urem(DivInt) &&
3493 for (
const SCEV *
Op : AR->operands())
3501 if (StartC && !DivInt.
urem(StepInt) &&
3507 const APInt &StartRem = StartInt.
urem(StepInt);
3508 if (StartRem != 0) {
3509 const SCEV *NewLHS =
3512 if (LHS != NewLHS) {
3522 if (
const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(
ID, IP))
3531 for (
const SCEV *
Op : M->operands())
3535 for (
unsigned i = 0, e = M->getNumOperands(); i != e; ++i) {
3536 const SCEV *
Op = M->getOperand(i);
3548 if (
auto *DivisorConstant =
3550 bool Overflow =
false;
3552 DivisorConstant->getAPInt().
umul_ov(RHSC->getAPInt(), Overflow);
3563 for (
const SCEV *
Op :
A->operands())
3567 for (
unsigned i = 0, e =
A->getNumOperands(); i != e; ++i) {
3574 if (
Operands.size() ==
A->getNumOperands())
3581 return getConstant(LHSC->getAPInt().udiv(RHSC->getAPInt()));
3587 AE && AE->getNumOperands() == 2) {
3589 const APInt &NegC = VC->getAPInt();
3592 if (MME && MME->getNumOperands() == 2 &&
3595 MME->getOperand(1) == RHS)
3596 return getZero(LHS->getType());
3603 const SCEV *NewLHS, *NewRHS;
3611 if (
const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(
ID, IP))
return S;
3614 UniqueSCEVs.InsertNode(S, IP);
3644 if (!
Mul || !
Mul->hasNoUnsignedWrap())
3651 if (LHSCst == RHSCst) {
3659 APInt Factor =
gcd(LHSCst, RHSCst);
3677 for (
int i = 0, e =
Mul->getNumOperands(); i != e; ++i) {
3678 if (
Mul->getOperand(i) == RHS) {
3697 if (StepChrec->getLoop() == L) {
3716 "SCEVAddRecExpr operand types don't match!");
3717 assert(!
Op->getType()->isPointerTy() &&
"Step must be integer");
3721 "SCEVAddRecExpr operand is not available at loop entry!");
3739 const Loop *NestedLoop = NestedAR->getLoop();
3740 if (L->contains(NestedLoop)
3743 DT.dominates(L->getHeader(), NestedLoop->
getHeader()))) {
3745 Operands[0] = NestedAR->getStart();
3749 bool AllInvariant =
all_of(
3761 AllInvariant =
all_of(NestedOperands, [&](
const SCEV *
Op) {
3772 return getAddRecExpr(NestedOperands, NestedLoop, InnerFlags);
3782 return getOrCreateAddRecExpr(
Operands, L, Flags);
3800 if (!GEPI || !isSCEVExprNeverPoison(GEPI))
3811 bool FirstIter =
true;
3813 for (
const SCEV *IndexExpr : IndexExprs) {
3820 Offsets.push_back(FieldOffset);
3823 CurTy = STy->getTypeAtIndex(Index);
3828 "The first index of a GEP indexes a pointer");
3829 CurTy =
GEP->getSourceElementType();
3840 const SCEV *LocalOffset =
getMulExpr(IndexExpr, ElementSize, OffsetWrap);
3841 Offsets.push_back(LocalOffset);
3846 if (Offsets.empty())
3859 "GEP should not change type mid-flight.");
3863SCEV *ScalarEvolution::findExistingSCEVInCache(
SCEVTypes SCEVType,
3866 ID.AddInteger(SCEVType);
3870 return UniqueSCEVs.FindNodeOrInsertPos(
ID, IP);
3880 assert(SCEVMinMaxExpr::isMinMaxType(Kind) &&
"Not a SCEVMinMaxExpr!");
3881 assert(!
Ops.empty() &&
"Cannot get empty (u|s)(min|max)!");
3882 if (
Ops.size() == 1)
return Ops[0];
3885 for (
unsigned i = 1, e =
Ops.size(); i != e; ++i) {
3887 "Operand types don't match!");
3890 "min/max should be consistently pointerish");
3916 return IsSigned ?
C.isMinSignedValue() :
C.isMinValue();
3918 return IsSigned ?
C.isMaxSignedValue() :
C.isMaxValue();
3923 return IsSigned ?
C.isMaxSignedValue() :
C.isMaxValue();
3925 return IsSigned ?
C.isMinSignedValue() :
C.isMinValue();
3931 if (
const SCEV *S = findExistingSCEVInCache(Kind,
Ops)) {
3937 while (Idx <
Ops.size() &&
Ops[Idx]->getSCEVType() < Kind)
3942 if (Idx <
Ops.size()) {
3943 bool DeletedAny =
false;
3944 while (
Ops[Idx]->getSCEVType() == Kind) {
3946 Ops.erase(
Ops.begin()+Idx);
3964 for (
unsigned i = 0, e =
Ops.size() - 1; i != e; ++i) {
3965 if (
Ops[i] ==
Ops[i + 1] ||
3966 isKnownViaNonRecursiveReasoning(FirstPred,
Ops[i],
Ops[i + 1])) {
3969 Ops.erase(
Ops.begin() + i + 1,
Ops.begin() + i + 2);
3972 }
else if (isKnownViaNonRecursiveReasoning(SecondPred,
Ops[i],
3975 Ops.erase(
Ops.begin() + i,
Ops.begin() + i + 1);
3981 if (
Ops.size() == 1)
return Ops[0];
3983 assert(!
Ops.empty() &&
"Reduced smax down to nothing!");
3988 ID.AddInteger(Kind);
3992 const SCEV *ExistingSCEV = UniqueSCEVs.FindNodeOrInsertPos(
ID, IP);
3994 return ExistingSCEV;
3995 const SCEV **O = SCEVAllocator.Allocate<
const SCEV *>(
Ops.size());
3997 SCEV *S =
new (SCEVAllocator)
4000 UniqueSCEVs.InsertNode(S, IP);
4007class SCEVSequentialMinMaxDeduplicatingVisitor final
4008 :
public SCEVVisitor<SCEVSequentialMinMaxDeduplicatingVisitor,
4009 std::optional<const SCEV *>> {
4010 using RetVal = std::optional<const SCEV *>;
4018 bool canRecurseInto(
SCEVTypes Kind)
const {
4021 return RootKind == Kind || NonSequentialRootKind == Kind;
4024 RetVal visitAnyMinMaxExpr(
const SCEV *S) {
4026 "Only for min/max expressions.");
4029 if (!canRecurseInto(Kind))
4039 return std::nullopt;
4046 RetVal
visit(
const SCEV *S) {
4048 if (!SeenOps.
insert(S).second)
4049 return std::nullopt;
4050 return Base::visit(S);
4054 SCEVSequentialMinMaxDeduplicatingVisitor(ScalarEvolution &SE,
4056 : SE(SE), RootKind(RootKind),
4057 NonSequentialRootKind(
4058 SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(
4062 SmallVectorImpl<const SCEV *> &NewOps) {
4067 for (
const SCEV *
Op : OrigOps) {
4072 Ops.emplace_back(*NewOp);
4076 NewOps = std::move(
Ops);
4080 RetVal visitConstant(
const SCEVConstant *Constant) {
return Constant; }
4082 RetVal visitVScale(
const SCEVVScale *VScale) {
return VScale; }
4084 RetVal visitPtrToIntExpr(
const SCEVPtrToIntExpr *Expr) {
return Expr; }
4086 RetVal visitTruncateExpr(
const SCEVTruncateExpr *Expr) {
return Expr; }
4088 RetVal visitZeroExtendExpr(
const SCEVZeroExtendExpr *Expr) {
return Expr; }
4090 RetVal visitSignExtendExpr(
const SCEVSignExtendExpr *Expr) {
return Expr; }
4092 RetVal visitAddExpr(
const SCEVAddExpr *Expr) {
return Expr; }
4094 RetVal visitMulExpr(
const SCEVMulExpr *Expr) {
return Expr; }
4096 RetVal visitUDivExpr(
const SCEVUDivExpr *Expr) {
return Expr; }
4098 RetVal visitAddRecExpr(
const SCEVAddRecExpr *Expr) {
return Expr; }
4100 RetVal visitSMaxExpr(
const SCEVSMaxExpr *Expr) {
4101 return visitAnyMinMaxExpr(Expr);
4104 RetVal visitUMaxExpr(
const SCEVUMaxExpr *Expr) {
4105 return visitAnyMinMaxExpr(Expr);
4108 RetVal visitSMinExpr(
const SCEVSMinExpr *Expr) {
4109 return visitAnyMinMaxExpr(Expr);
4112 RetVal visitUMinExpr(
const SCEVUMinExpr *Expr) {
4113 return visitAnyMinMaxExpr(Expr);
4116 RetVal visitSequentialUMinExpr(
const SCEVSequentialUMinExpr *Expr) {
4117 return visitAnyMinMaxExpr(Expr);
4120 RetVal visitUnknown(
const SCEVUnknown *Expr) {
return Expr; }
4122 RetVal visitCouldNotCompute(
const SCEVCouldNotCompute *Expr) {
return Expr; }
4164struct SCEVPoisonCollector {
4165 bool LookThroughMaybePoisonBlocking;
4166 SmallPtrSet<const SCEVUnknown *, 4> MaybePoison;
4167 SCEVPoisonCollector(
bool LookThroughMaybePoisonBlocking)
4168 : LookThroughMaybePoisonBlocking(LookThroughMaybePoisonBlocking) {}
4170 bool follow(
const SCEV *S) {
4171 if (!LookThroughMaybePoisonBlocking &&
4181 bool isDone()
const {
return false; }
4191 SCEVPoisonCollector PC1(
true);
4196 if (PC1.MaybePoison.empty())
4202 SCEVPoisonCollector PC2(
false);
4212 SCEVPoisonCollector PC(
false);
4235 while (!Worklist.
empty()) {
4237 if (!Visited.
insert(V).second)
4241 if (Visited.
size() > 16)
4246 if (PoisonVals.
contains(V) || ::isGuaranteedNotToBePoison(V))
4257 if (PDI->isDisjoint())
4264 II &&
II->getIntrinsicID() == Intrinsic::vscale)
4271 if (
I->hasPoisonGeneratingAnnotations())
4282 assert(SCEVSequentialMinMaxExpr::isSequentialMinMaxType(Kind) &&
4283 "Not a SCEVSequentialMinMaxExpr!");
4284 assert(!
Ops.empty() &&
"Cannot get empty (u|s)(min|max)!");
4285 if (
Ops.size() == 1)
4289 for (
unsigned i = 1, e =
Ops.size(); i != e; ++i) {
4291 "Operand types don't match!");
4294 "min/max should be consistently pointerish");
4302 if (
const SCEV *S = findExistingSCEVInCache(Kind,
Ops))
4309 SCEVSequentialMinMaxDeduplicatingVisitor Deduplicator(*
this, Kind);
4319 bool DeletedAny =
false;
4320 while (Idx <
Ops.size()) {
4321 if (
Ops[Idx]->getSCEVType() != Kind) {
4326 Ops.erase(
Ops.begin() + Idx);
4327 Ops.insert(
Ops.begin() + Idx, SMME->operands().begin(),
4328 SMME->operands().end());
4336 const SCEV *SaturationPoint;
4347 for (
unsigned i = 1, e =
Ops.size(); i != e; ++i) {
4348 if (!isGuaranteedNotToCauseUB(
Ops[i]))
4360 Ops.erase(
Ops.begin() + i);
4365 if (isKnownViaNonRecursiveReasoning(Pred,
Ops[i - 1],
Ops[i])) {
4366 Ops.erase(
Ops.begin() + i);
4374 ID.AddInteger(Kind);
4378 const SCEV *ExistingSCEV = UniqueSCEVs.FindNodeOrInsertPos(
ID, IP);
4380 return ExistingSCEV;
4382 const SCEV **O = SCEVAllocator.Allocate<
const SCEV *>(
Ops.size());
4384 SCEV *S =
new (SCEVAllocator)
4387 UniqueSCEVs.InsertNode(S, IP);
4435 if (
Size.isScalable())
4456 "Cannot get offset for structure containing scalable vector types");
4470 if (
SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(
ID, IP)) {
4472 "Stale SCEVUnknown in uniquing map!");
4478 UniqueSCEVs.InsertNode(S, IP);
4492 return Ty->isIntOrPtrTy();
4499 if (Ty->isPointerTy())
4510 if (Ty->isIntegerTy())
4514 assert(Ty->isPointerTy() &&
"Unexpected non-pointer non-integer type!");
4526 bool PreciseA, PreciseB;
4527 auto *ScopeA = getDefiningScopeBound({
A}, PreciseA);
4528 auto *ScopeB = getDefiningScopeBound({
B}, PreciseB);
4529 if (!PreciseA || !PreciseB)
4532 return (ScopeA == ScopeB) || DT.dominates(ScopeA, ScopeB) ||
4533 DT.dominates(ScopeB, ScopeA);
4537 return CouldNotCompute.get();
4540bool ScalarEvolution::checkValidity(
const SCEV *S)
const {
4543 return SU && SU->getValue() ==
nullptr;
4546 return !ContainsNulls;
4551 if (
I != HasRecMap.end())
4556 HasRecMap.insert({S, FoundAddRec});
4564 if (
SI == ExprValueMap.
end())
4566 return SI->second.getArrayRef();
4572void ScalarEvolution::eraseValueFromMap(
Value *V) {
4574 if (
I != ValueExprMap.end()) {
4575 auto EVIt = ExprValueMap.find(
I->second);
4576 bool Removed = EVIt->second.remove(V);
4578 assert(Removed &&
"Value not in ExprValueMap?");
4579 ValueExprMap.erase(
I);
4583void ScalarEvolution::insertValueToMap(
Value *V,
const SCEV *S) {
4587 auto It = ValueExprMap.find_as(V);
4588 if (It == ValueExprMap.end()) {
4590 ExprValueMap[S].insert(V);
4601 return createSCEVIter(V);
4608 if (
I != ValueExprMap.end()) {
4609 const SCEV *S =
I->second;
4610 assert(checkValidity(S) &&
4611 "existing SCEV has not been properly invalidated");
4624 Type *Ty = V->getType();
4632 if (!
Add ||
Add->getNumOperands() != 2 ||
4633 !
Add->getOperand(0)->isAllOnesValue())
4646 assert(!V->getType()->isPointerTy() &&
"Can't negate pointer");
4659 return (
const SCEV *)
nullptr;
4665 if (
const SCEV *Replaced = MatchMinMaxNegation(MME))
4669 Type *Ty = V->getType();
4675 assert(
P->getType()->isPointerTy());
4688 const SCEV **PtrOp =
nullptr;
4689 for (
const SCEV *&AddOp :
Ops) {
4690 if (AddOp->getType()->isPointerTy()) {
4691 assert(!PtrOp &&
"Cannot have multiple pointer ops");
4709 return getZero(LHS->getType());
4714 if (RHS->getType()->isPointerTy()) {
4715 if (!LHS->getType()->isPointerTy() ||
4725 const bool RHSIsNotMinSigned =
4756 Type *SrcTy = V->getType();
4757 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4758 "Cannot truncate or zero extend with non-integer arguments!");
4768 Type *SrcTy = V->getType();
4769 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4770 "Cannot truncate or zero extend with non-integer arguments!");
4780 Type *SrcTy = V->getType();
4781 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4782 "Cannot noop or zero extend with non-integer arguments!");
4784 "getNoopOrZeroExtend cannot truncate!");
4792 Type *SrcTy = V->getType();
4793 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4794 "Cannot noop or sign extend with non-integer arguments!");
4796 "getNoopOrSignExtend cannot truncate!");
4804 Type *SrcTy = V->getType();
4805 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4806 "Cannot noop or any extend with non-integer arguments!");
4808 "getNoopOrAnyExtend cannot truncate!");
4816 Type *SrcTy = V->getType();
4817 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4818 "Cannot truncate or noop with non-integer arguments!");
4820 "getTruncateOrNoop cannot extend!");
4828 const SCEV *PromotedLHS = LHS;
4829 const SCEV *PromotedRHS = RHS;
4849 assert(!
Ops.empty() &&
"At least one operand must be!");
4851 if (
Ops.size() == 1)
4855 Type *MaxType =
nullptr;
4856 for (
const auto *S :
Ops)
4861 assert(MaxType &&
"Failed to find maximum type!");
4865 for (
const auto *S :
Ops)
4874 if (!V->getType()->isPointerTy())
4879 V = AddRec->getStart();
4881 const SCEV *PtrOp =
nullptr;
4882 for (
const SCEV *AddOp :
Add->operands()) {
4883 if (AddOp->getType()->isPointerTy()) {
4884 assert(!PtrOp &&
"Cannot have multiple pointer ops");
4888 assert(PtrOp &&
"Must have pointer op");
4900 for (
User *U :
I->users()) {
4902 if (Visited.
insert(UserInsn).second)
4916 static const SCEV *rewrite(
const SCEV *S,
const Loop *L, ScalarEvolution &SE,
4917 bool IgnoreOtherLoops =
true) {
4920 if (
Rewriter.hasSeenLoopVariantSCEVUnknown())
4922 return Rewriter.hasSeenOtherLoops() && !IgnoreOtherLoops
4927 const SCEV *visitUnknown(
const SCEVUnknown *Expr) {
4929 SeenLoopVariantSCEVUnknown =
true;
4933 const SCEV *visitAddRecExpr(
const SCEVAddRecExpr *Expr) {
4937 SeenOtherLoops =
true;
4941 bool hasSeenLoopVariantSCEVUnknown() {
return SeenLoopVariantSCEVUnknown; }
4943 bool hasSeenOtherLoops() {
return SeenOtherLoops; }
4946 explicit SCEVInitRewriter(
const Loop *L, ScalarEvolution &SE)
4947 : SCEVRewriteVisitor(SE),
L(
L) {}
4950 bool SeenLoopVariantSCEVUnknown =
false;
4951 bool SeenOtherLoops =
false;
4960 static const SCEV *rewrite(
const SCEV *S,
const Loop *L, ScalarEvolution &SE) {
4961 SCEVPostIncRewriter
Rewriter(L, SE);
4963 return Rewriter.hasSeenLoopVariantSCEVUnknown()
4968 const SCEV *visitUnknown(
const SCEVUnknown *Expr) {
4970 SeenLoopVariantSCEVUnknown =
true;
4974 const SCEV *visitAddRecExpr(
const SCEVAddRecExpr *Expr) {
4978 SeenOtherLoops =
true;
4982 bool hasSeenLoopVariantSCEVUnknown() {
return SeenLoopVariantSCEVUnknown; }
4984 bool hasSeenOtherLoops() {
return SeenOtherLoops; }
4987 explicit SCEVPostIncRewriter(
const Loop *L, ScalarEvolution &SE)
4988 : SCEVRewriteVisitor(SE),
L(
L) {}
4991 bool SeenLoopVariantSCEVUnknown =
false;
4992 bool SeenOtherLoops =
false;
4998class SCEVBackedgeConditionFolder
5001 static const SCEV *rewrite(
const SCEV *S,
const Loop *L,
5002 ScalarEvolution &SE) {
5003 bool IsPosBECond =
false;
5004 Value *BECond =
nullptr;
5005 if (BasicBlock *Latch =
L->getLoopLatch()) {
5009 "Both outgoing branches should not target same header!");
5016 SCEVBackedgeConditionFolder
Rewriter(L, BECond, IsPosBECond, SE);
5020 const SCEV *visitUnknown(
const SCEVUnknown *Expr) {
5021 const SCEV *
Result = Expr;
5026 switch (
I->getOpcode()) {
5027 case Instruction::Select: {
5029 std::optional<const SCEV *> Res =
5030 compareWithBackedgeCondition(
SI->getCondition());
5038 std::optional<const SCEV *> Res = compareWithBackedgeCondition(
I);
5049 explicit SCEVBackedgeConditionFolder(
const Loop *L,
Value *BECond,
5050 bool IsPosBECond, ScalarEvolution &SE)
5051 : SCEVRewriteVisitor(SE),
L(
L), BackedgeCond(BECond),
5052 IsPositiveBECond(IsPosBECond) {}
5054 std::optional<const SCEV *> compareWithBackedgeCondition(
Value *IC);
5058 Value *BackedgeCond =
nullptr;
5060 bool IsPositiveBECond;
5063std::optional<const SCEV *>
5064SCEVBackedgeConditionFolder::compareWithBackedgeCondition(
Value *IC) {
5069 if (BackedgeCond == IC)
5072 return std::nullopt;
5077 static const SCEV *rewrite(
const SCEV *S,
const Loop *L,
5078 ScalarEvolution &SE) {
5084 const SCEV *visitUnknown(
const SCEVUnknown *Expr) {
5091 const SCEV *visitAddRecExpr(
const SCEVAddRecExpr *Expr) {
5098 bool isValid() {
return Valid; }
5101 explicit SCEVShiftRewriter(
const Loop *L, ScalarEvolution &SE)
5102 : SCEVRewriteVisitor(SE),
L(
L) {}
5111ScalarEvolution::proveNoWrapViaConstantRanges(
const SCEVAddRecExpr *AR) {
5115 using OBO = OverflowingBinaryOperator;
5123 const APInt &BECountAP = BECountMax->getAPInt();
5124 unsigned NoOverflowBitWidth =
5136 Instruction::Add, IncRange, OBO::NoSignedWrap);
5137 if (NSWRegion.contains(AddRecRange))
5146 Instruction::Add, IncRange, OBO::NoUnsignedWrap);
5147 if (NUWRegion.contains(AddRecRange))
5155ScalarEvolution::proveNoSignedWrapViaInduction(
const SCEVAddRecExpr *AR) {
5165 if (!SignedWrapViaInductionTried.insert(AR).second)
5190 AC.assumptions().empty())
5198 const SCEV *OverflowLimit =
5200 if (OverflowLimit &&
5208ScalarEvolution::proveNoUnsignedWrapViaInduction(
const SCEVAddRecExpr *AR) {
5218 if (!UnsignedWrapViaInductionTried.insert(AR).second)
5244 AC.assumptions().empty())
5279 explicit BinaryOp(Operator *
Op)
5283 IsNSW = OBO->hasNoSignedWrap();
5284 IsNUW = OBO->hasNoUnsignedWrap();
5288 explicit BinaryOp(
unsigned Opcode,
Value *
LHS,
Value *
RHS,
bool IsNSW =
false,
5290 : Opcode(Opcode),
LHS(
LHS),
RHS(
RHS), IsNSW(IsNSW), IsNUW(IsNUW) {}
5302 return std::nullopt;
5308 switch (
Op->getOpcode()) {
5309 case Instruction::Add:
5310 case Instruction::Sub:
5311 case Instruction::Mul:
5312 case Instruction::UDiv:
5313 case Instruction::URem:
5314 case Instruction::And:
5315 case Instruction::AShr:
5316 case Instruction::Shl:
5317 return BinaryOp(
Op);
5319 case Instruction::Or: {
5322 return BinaryOp(Instruction::Add,
Op->getOperand(0),
Op->getOperand(1),
5324 return BinaryOp(
Op);
5327 case Instruction::Xor:
5331 if (RHSC->getValue().isSignMask())
5332 return BinaryOp(Instruction::Add,
Op->getOperand(0),
Op->getOperand(1));
5334 if (V->getType()->isIntegerTy(1))
5335 return BinaryOp(Instruction::Add,
Op->getOperand(0),
Op->getOperand(1));
5336 return BinaryOp(
Op);
5338 case Instruction::LShr:
5347 if (SA->getValue().ult(
BitWidth)) {
5349 ConstantInt::get(SA->getContext(),
5351 return BinaryOp(Instruction::UDiv,
Op->getOperand(0),
X);
5354 return BinaryOp(
Op);
5356 case Instruction::ExtractValue: {
5358 if (EVI->getNumIndices() != 1 || EVI->getIndices()[0] != 0)
5366 bool Signed = WO->isSigned();
5369 return BinaryOp(BinOp, WO->getLHS(), WO->getRHS());
5374 return BinaryOp(BinOp, WO->getLHS(), WO->getRHS(),
5385 if (
II->getIntrinsicID() == Intrinsic::loop_decrement_reg)
5386 return BinaryOp(Instruction::Sub,
II->getOperand(0),
II->getOperand(1));
5388 return std::nullopt;
5414 if (
Op == SymbolicPHI)
5419 if (SourceBits != NewBits)
5432 if (
X != SymbolicPHI)
5434 Signed = SExt !=
nullptr;
5442 if (!L || L->getHeader() != PN->
getParent())
5500std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5501ScalarEvolution::createAddRecFromPHIWithCastsImpl(
const SCEVUnknown *SymbolicPHI) {
5509 assert(L &&
"Expecting an integer loop header phi");
5514 Value *BEValueV =
nullptr, *StartValueV =
nullptr;
5515 for (
unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
5516 Value *
V = PN->getIncomingValue(i);
5517 if (
L->contains(PN->getIncomingBlock(i))) {
5520 }
else if (BEValueV != V) {
5524 }
else if (!StartValueV) {
5526 }
else if (StartValueV != V) {
5527 StartValueV =
nullptr;
5531 if (!BEValueV || !StartValueV)
5532 return std::nullopt;
5534 const SCEV *BEValue =
getSCEV(BEValueV);
5541 return std::nullopt;
5545 unsigned FoundIndex =
Add->getNumOperands();
5546 Type *TruncTy =
nullptr;
5548 for (
unsigned i = 0, e =
Add->getNumOperands(); i != e; ++i)
5551 if (FoundIndex == e) {
5556 if (FoundIndex ==
Add->getNumOperands())
5557 return std::nullopt;
5561 for (
unsigned i = 0, e =
Add->getNumOperands(); i != e; ++i)
5562 if (i != FoundIndex)
5563 Ops.push_back(
Add->getOperand(i));
5569 return std::nullopt;
5622 const SCEV *StartVal =
getSCEV(StartValueV);
5623 const SCEV *PHISCEV =
5650 auto getExtendedExpr = [&](
const SCEV *Expr,
5651 bool CreateSignExtend) ->
const SCEV * {
5654 const SCEV *ExtendedExpr =
5657 return ExtendedExpr;
5665 auto PredIsKnownFalse = [&](
const SCEV *Expr,
5666 const SCEV *ExtendedExpr) ->
bool {
5667 return Expr != ExtendedExpr &&
5671 const SCEV *StartExtended = getExtendedExpr(StartVal,
Signed);
5672 if (PredIsKnownFalse(StartVal, StartExtended)) {
5674 return std::nullopt;
5679 const SCEV *AccumExtended = getExtendedExpr(Accum,
true);
5680 if (PredIsKnownFalse(Accum, AccumExtended)) {
5682 return std::nullopt;
5685 auto AppendPredicate = [&](
const SCEV *Expr,
5686 const SCEV *ExtendedExpr) ->
void {
5687 if (Expr != ExtendedExpr &&
5695 AppendPredicate(StartVal, StartExtended);
5696 AppendPredicate(Accum, AccumExtended);
5704 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>> PredRewrite =
5705 std::make_pair(NewAR, Predicates);
5707 PredicatedSCEVRewrites[{SymbolicPHI,
L}] = PredRewrite;
5711std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5716 return std::nullopt;
5719 auto I = PredicatedSCEVRewrites.find({SymbolicPHI, L});
5720 if (
I != PredicatedSCEVRewrites.end()) {
5721 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>> Rewrite =
5724 if (Rewrite.first == SymbolicPHI)
5725 return std::nullopt;
5729 assert(!(Rewrite.second).empty() &&
"Expected to find Predicates");
5733 std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5734 Rewrite = createAddRecFromPHIWithCastsImpl(SymbolicPHI);
5739 PredicatedSCEVRewrites[{SymbolicPHI, L}] = {SymbolicPHI, Predicates};
5740 return std::nullopt;
5757 auto areExprsEqual = [&](
const SCEV *Expr1,
const SCEV *Expr2) ->
bool {
5758 if (Expr1 != Expr2 &&
5759 !Preds->implies(SE.getEqualPredicate(Expr1, Expr2), SE) &&
5760 !Preds->implies(SE.getEqualPredicate(Expr2, Expr1), SE))
5777const SCEV *ScalarEvolution::createSimpleAffineAddRec(
PHINode *PN,
5779 Value *StartValueV) {
5782 assert(BEValueV && StartValueV);
5788 if (BO->Opcode != Instruction::Add)
5791 const SCEV *Accum =
nullptr;
5792 if (BO->LHS == PN && L->isLoopInvariant(BO->RHS))
5794 else if (BO->RHS == PN && L->isLoopInvariant(BO->LHS))
5808 insertValueToMap(PN, PHISCEV);
5813 proveNoWrapViaConstantRanges(AR)));
5821 "Accum is defined outside L, but is not invariant?");
5822 if (isAddRecNeverPoison(BEInst, L))
5829const SCEV *ScalarEvolution::createAddRecFromPHI(
PHINode *PN) {
5830 const Loop *
L = LI.getLoopFor(PN->
getParent());
5837 Value *BEValueV =
nullptr, *StartValueV =
nullptr;
5843 }
else if (BEValueV != V) {
5847 }
else if (!StartValueV) {
5849 }
else if (StartValueV != V) {
5850 StartValueV =
nullptr;
5854 if (!BEValueV || !StartValueV)
5857 assert(ValueExprMap.find_as(PN) == ValueExprMap.end() &&
5858 "PHI node already processed?");
5862 if (
auto *S = createSimpleAffineAddRec(PN, BEValueV, StartValueV))
5867 insertValueToMap(PN, SymbolicName);
5871 const SCEV *BEValue =
getSCEV(BEValueV);
5881 unsigned FoundIndex =
Add->getNumOperands();
5882 for (
unsigned i = 0, e =
Add->getNumOperands(); i != e; ++i)
5883 if (
Add->getOperand(i) == SymbolicName)
5884 if (FoundIndex == e) {
5889 if (FoundIndex !=
Add->getNumOperands()) {
5892 for (
unsigned i = 0, e =
Add->getNumOperands(); i != e; ++i)
5893 if (i != FoundIndex)
5894 Ops.push_back(SCEVBackedgeConditionFolder::rewrite(
Add->getOperand(i),
5906 if (BO->Opcode == Instruction::Add && BO->LHS == PN) {
5913 if (
GEP->getOperand(0) == PN) {
5914 GEPNoWrapFlags NW =
GEP->getNoWrapFlags();
5932 const SCEV *StartVal =
getSCEV(StartValueV);
5933 const SCEV *PHISCEV =
getAddRecExpr(StartVal, Accum, L, Flags);
5938 forgetMemoizedResults(SymbolicName);
5939 insertValueToMap(PN, PHISCEV);
5944 proveNoWrapViaConstantRanges(AR)));
5969 const SCEV *Shifted = SCEVShiftRewriter::rewrite(BEValue, L, *
this);
5970 const SCEV *
Start = SCEVInitRewriter::rewrite(Shifted, L, *
this,
false);
5972 isGuaranteedNotToCauseUB(Shifted) &&
::impliesPoison(Shifted, Start)) {
5973 const SCEV *StartVal =
getSCEV(StartValueV);
5974 if (Start == StartVal) {
5978 forgetMemoizedResults(SymbolicName);
5979 insertValueToMap(PN, Shifted);
5989 eraseValueFromMap(PN);
6009 Use &LeftUse =
Merge->getOperandUse(0);
6010 Use &RightUse =
Merge->getOperandUse(1);
6027const SCEV *ScalarEvolution::createNodeFromSelectLikePHI(
PHINode *PN) {
6029 [&](
BasicBlock *BB) {
return DT.isReachableFromEntry(BB); };
6044 assert(IDom &&
"At least the entry block should dominate PN");
6053 return createNodeForSelectOrPHI(PN,
Cond,
LHS,
RHS);
6069ScalarEvolution::createNodeForPHIWithIdenticalOperands(
PHINode *PN) {
6070 BinaryOperator *CommonInst =
nullptr;
6081 CommonInst = IncomingInst;
6088 const SCEV *CommonSCEV =
getSCEV(CommonInst);
6089 bool SCEVExprsIdentical =
6091 [
this, CommonSCEV](
Value *V) { return CommonSCEV == getSCEV(V); });
6092 return SCEVExprsIdentical ? CommonSCEV :
nullptr;
6095const SCEV *ScalarEvolution::createNodeForPHI(
PHINode *PN) {
6096 if (
const SCEV *S = createAddRecFromPHI(PN))
6106 if (
const SCEV *S = createNodeForPHIWithIdenticalOperands(PN))
6109 if (
const SCEV *S = createNodeFromSelectLikePHI(PN))
6118 struct FindClosure {
6119 const SCEV *OperandToFind;
6125 bool canRecurseInto(
SCEVTypes Kind)
const {
6128 return RootKind == Kind || NonSequentialRootKind == Kind ||
6133 : OperandToFind(OperandToFind), RootKind(RootKind),
6134 NonSequentialRootKind(
6138 bool follow(
const SCEV *S) {
6139 Found = S == OperandToFind;
6141 return !isDone() && canRecurseInto(S->
getSCEVType());
6144 bool isDone()
const {
return Found; }
6147 FindClosure FC(OperandToFind, RootKind);
6152std::optional<const SCEV *>
6153ScalarEvolution::createNodeForSelectOrPHIInstWithICmpInstCond(
Type *Ty,
6163 switch (ICI->getPredicate()) {
6177 bool Signed = ICI->isSigned();
6178 const SCEV *LA =
getSCEV(TrueVal);
6186 if (LA == LS &&
RA == RS)
6188 if (LA == RS &&
RA == LS)
6191 auto CoerceOperand = [&](
const SCEV *
Op) ->
const SCEV * {
6192 if (
Op->getType()->isPointerTy()) {
6203 LS = CoerceOperand(LS);
6204 RS = CoerceOperand(RS);
6228 const SCEV *TrueValExpr =
getSCEV(TrueVal);
6229 const SCEV *FalseValExpr =
getSCEV(FalseVal);
6243 X = ZExt->getOperand();
6245 const SCEV *FalseValExpr =
getSCEV(FalseVal);
6256 return std::nullopt;
6259static std::optional<const SCEV *>
6261 const SCEV *TrueExpr,
const SCEV *FalseExpr) {
6265 "Unexpected operands of a select.");
6277 return std::nullopt;
6292static std::optional<const SCEV *>
6296 return std::nullopt;
6299 const auto *SETrue = SE->
getSCEV(TrueVal);
6300 const auto *SEFalse = SE->
getSCEV(FalseVal);
6304const SCEV *ScalarEvolution::createNodeForSelectOrPHIViaUMinSeq(
6306 assert(
Cond->getType()->isIntegerTy(1) &&
"Select condition is not an i1?");
6308 V->getType() ==
TrueVal->getType() &&
6309 "Types of select hands and of the result must match.");
6312 if (!
V->getType()->isIntegerTy(1))
6315 if (std::optional<const SCEV *> S =
6328 return getSCEV(CI->isOne() ? TrueVal : FalseVal);
6332 if (std::optional<const SCEV *> S =
6333 createNodeForSelectOrPHIInstWithICmpInstCond(
I->getType(), ICI,
6339 return createNodeForSelectOrPHIViaUMinSeq(V,
Cond, TrueVal, FalseVal);
6345 assert(
GEP->getSourceElementType()->isSized() &&
6346 "GEP source element type must be sized");
6349 for (
Value *Index :
GEP->indices())
6354APInt ScalarEvolution::getConstantMultipleImpl(
const SCEV *S) {
6356 auto GetShiftedByZeros = [
BitWidth](uint32_t TrailingZeros) {
6359 : APInt::getOneBitSet(
BitWidth, TrailingZeros);
6361 auto GetGCDMultiple = [
this](
const SCEVNAryExpr *
N) {
6364 for (
unsigned I = 1,
E =
N->getNumOperands();
I <
E && Res != 1; ++
I)
6382 return GetShiftedByZeros(TZ);
6392 return GetShiftedByZeros(TZ);
6396 if (
M->hasNoUnsignedWrap()) {
6399 for (
const SCEV *Operand :
M->operands().drop_front())
6407 for (
const SCEV *Operand :
M->operands())
6409 return GetShiftedByZeros(TZ);
6414 if (
N->hasNoUnsignedWrap())
6415 return GetGCDMultiple(
N);
6418 for (
const SCEV *Operand :
N->operands().drop_front())
6420 return GetShiftedByZeros(TZ);
6433 .countMinTrailingZeros();
6434 return GetShiftedByZeros(Known);
6443 auto I = ConstantMultipleCache.find(S);
6444 if (
I != ConstantMultipleCache.end())
6447 APInt Result = getConstantMultipleImpl(S);
6448 auto InsertPair = ConstantMultipleCache.insert({S, Result});
6449 assert(InsertPair.second &&
"Should insert a new key");
6450 return InsertPair.first->second;
6466 if (
MDNode *MD =
I->getMetadata(LLVMContext::MD_range))
6469 if (std::optional<ConstantRange>
Range = CB->getRange())
6473 if (std::optional<ConstantRange>
Range =
A->getRange())
6476 return std::nullopt;
6483 UnsignedRanges.erase(AddRec);
6484 SignedRanges.erase(AddRec);
6485 ConstantMultipleCache.erase(AddRec);
6490getRangeForUnknownRecurrence(
const SCEVUnknown *U) {
6516 Value *Start, *Step;
6523 assert(L && L->getHeader() ==
P->getParent());
6536 case Instruction::AShr:
6537 case Instruction::LShr:
6538 case Instruction::Shl:
6553 KnownStep.getBitWidth() ==
BitWidth);
6556 auto MaxShiftAmt = KnownStep.getMaxValue();
6558 bool Overflow =
false;
6559 auto TotalShift = MaxShiftAmt.umul_ov(TCAP, Overflow);
6566 case Instruction::AShr: {
6574 if (KnownStart.isNonNegative())
6577 KnownStart.getMaxValue() + 1);
6578 if (KnownStart.isNegative())
6581 KnownEnd.getMaxValue() + 1);
6584 case Instruction::LShr: {
6593 KnownStart.getMaxValue() + 1);
6595 case Instruction::Shl: {
6599 if (TotalShift.ult(KnownStart.countMinLeadingZeros()))
6600 return ConstantRange(KnownStart.getMinValue(),
6601 KnownEnd.getMaxValue() + 1);
6609ScalarEvolution::getRangeRefIter(
const SCEV *S,
6610 ScalarEvolution::RangeSignHint SignHint) {
6611 DenseMap<const SCEV *, ConstantRange> &Cache =
6612 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? UnsignedRanges
6615 SmallPtrSet<const SCEV *, 8> Seen;
6619 auto AddToWorklist = [&WorkList, &Seen, &Cache](
const SCEV *Expr) {
6620 if (!Seen.
insert(Expr).second)
6653 for (
unsigned I = 0;
I != WorkList.
size(); ++
I) {
6654 const SCEV *
P = WorkList[
I];
6658 for (
const SCEV *
Op :
P->operands())
6664 if (!PendingPhiRangesIter.insert(
P).second)
6671 if (!WorkList.
empty()) {
6676 getRangeRef(
P, SignHint);
6680 PendingPhiRangesIter.erase(
P);
6684 return getRangeRef(S, SignHint, 0);
6691 const SCEV *S, ScalarEvolution::RangeSignHint SignHint,
unsigned Depth) {
6692 DenseMap<const SCEV *, ConstantRange> &Cache =
6693 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? UnsignedRanges
6701 if (
I != Cache.
end())
6705 return setRange(
C, SignHint, ConstantRange(
C->getAPInt()));
6710 return getRangeRefIter(S, SignHint);
6713 ConstantRange ConservativeResult(
BitWidth,
true);
6714 using OBO = OverflowingBinaryOperator;
6718 if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED) {
6722 ConservativeResult =
6729 ConservativeResult = ConstantRange(
6745 ConservativeResult.intersectWith(
X.truncate(
BitWidth), RangeType));
6752 ConservativeResult.intersectWith(
X.zeroExtend(
BitWidth), RangeType));
6759 ConservativeResult.intersectWith(
X.signExtend(
BitWidth), RangeType));
6763 ConstantRange
X = getRangeRef(PtrToInt->
getOperand(), SignHint,
Depth + 1);
6764 return setRange(PtrToInt, SignHint,
X);
6768 ConstantRange
X = getRangeRef(
Add->getOperand(0), SignHint,
Depth + 1);
6769 unsigned WrapType = OBO::AnyWrap;
6770 if (
Add->hasNoSignedWrap())
6771 WrapType |= OBO::NoSignedWrap;
6772 if (
Add->hasNoUnsignedWrap())
6773 WrapType |= OBO::NoUnsignedWrap;
6775 X =
X.addWithNoWrap(getRangeRef(
Op, SignHint,
Depth + 1), WrapType,
6777 return setRange(
Add, SignHint,
6778 ConservativeResult.intersectWith(
X, RangeType));
6782 ConstantRange
X = getRangeRef(
Mul->getOperand(0), SignHint,
Depth + 1);
6784 X =
X.multiply(getRangeRef(
Op, SignHint,
Depth + 1));
6785 return setRange(
Mul, SignHint,
6786 ConservativeResult.intersectWith(
X, RangeType));
6790 ConstantRange
X = getRangeRef(UDiv->
getLHS(), SignHint,
Depth + 1);
6791 ConstantRange
Y = getRangeRef(UDiv->
getRHS(), SignHint,
Depth + 1);
6792 return setRange(UDiv, SignHint,
6793 ConservativeResult.intersectWith(
X.udiv(
Y), RangeType));
6801 if (!UnsignedMinValue.
isZero())
6802 ConservativeResult = ConservativeResult.intersectWith(
6803 ConstantRange(UnsignedMinValue, APInt(
BitWidth, 0)), RangeType);
6812 bool AllNonNeg =
true;
6813 bool AllNonPos =
true;
6814 for (
unsigned i = 1, e = AddRec->
getNumOperands(); i != e; ++i) {
6821 ConservativeResult = ConservativeResult.intersectWith(
6826 ConservativeResult = ConservativeResult.intersectWith(
6835 const SCEV *MaxBEScev =
6849 auto RangeFromAffine = getRangeForAffineAR(
6851 ConservativeResult =
6852 ConservativeResult.intersectWith(RangeFromAffine, RangeType);
6854 auto RangeFromFactoring = getRangeViaFactoring(
6856 ConservativeResult =
6857 ConservativeResult.intersectWith(RangeFromFactoring, RangeType);
6863 const SCEV *SymbolicMaxBECount =
6868 auto RangeFromAffineNew = getRangeForAffineNoSelfWrappingAR(
6869 AddRec, SymbolicMaxBECount,
BitWidth, SignHint);
6870 ConservativeResult =
6871 ConservativeResult.intersectWith(RangeFromAffineNew, RangeType);
6876 return setRange(AddRec, SignHint, std::move(ConservativeResult));
6886 ID = Intrinsic::umax;
6889 ID = Intrinsic::smax;
6893 ID = Intrinsic::umin;
6896 ID = Intrinsic::smin;
6903 ConstantRange
X = getRangeRef(NAry->getOperand(0), SignHint,
Depth + 1);
6904 for (
unsigned i = 1, e = NAry->getNumOperands(); i != e; ++i)
6906 ID, {
X, getRangeRef(NAry->getOperand(i), SignHint,
Depth + 1)});
6907 return setRange(S, SignHint,
6908 ConservativeResult.intersectWith(
X, RangeType));
6917 ConservativeResult =
6918 ConservativeResult.intersectWith(*MDRange, RangeType);
6923 auto CR = getRangeForUnknownRecurrence(U);
6924 ConservativeResult = ConservativeResult.intersectWith(CR);
6935 if (
U->getType()->isPointerTy()) {
6938 unsigned ptrSize = DL.getPointerTypeSizeInBits(
U->getType());
6939 int ptrIdxDiff = ptrSize -
BitWidth;
6940 if (ptrIdxDiff > 0 && ptrSize >
BitWidth && NS > (
unsigned)ptrIdxDiff)
6953 ConservativeResult = ConservativeResult.intersectWith(
6957 ConservativeResult = ConservativeResult.intersectWith(
6962 if (
U->getType()->isPointerTy() && SignHint == HINT_RANGE_UNSIGNED) {
6965 bool CanBeNull, CanBeFreed;
6966 uint64_t DerefBytes =
6967 V->getPointerDereferenceableBytes(DL, CanBeNull, CanBeFreed);
6977 uint64_t
Align =
U->getValue()->getPointerAlignment(DL).value();
6978 uint64_t Rem = MaxVal.
urem(Align);
6983 ConservativeResult = ConservativeResult.intersectWith(
6991 if (PendingPhiRanges.insert(Phi).second) {
6992 ConstantRange RangeFromOps(
BitWidth,
false);
6994 for (
const auto &
Op :
Phi->operands()) {
6996 RangeFromOps = RangeFromOps.unionWith(OpRange);
6998 if (RangeFromOps.isFullSet())
7001 ConservativeResult =
7002 ConservativeResult.intersectWith(RangeFromOps, RangeType);
7003 bool Erased = PendingPhiRanges.erase(Phi);
7004 assert(Erased &&
"Failed to erase Phi properly?");
7011 if (
II->getIntrinsicID() == Intrinsic::vscale) {
7013 ConservativeResult = ConservativeResult.difference(Disallowed);
7016 return setRange(U, SignHint, std::move(ConservativeResult));
7022 return setRange(S, SignHint, std::move(ConservativeResult));
7031 const APInt &MaxBECount,
7038 if (Step == 0 || MaxBECount == 0)
7045 return ConstantRange::getFull(
BitWidth);
7061 return ConstantRange::getFull(
BitWidth);
7073 APInt MovedBoundary = Descending ? (StartLower - std::move(
Offset))
7074 : (StartUpper + std::move(
Offset));
7079 if (StartRange.
contains(MovedBoundary))
7080 return ConstantRange::getFull(
BitWidth);
7083 Descending ? std::move(MovedBoundary) : std::move(StartLower);
7085 Descending ? std::move(StartUpper) : std::move(MovedBoundary);
7094 const APInt &MaxBECount) {
7098 "mismatched bit widths");
7107 StepSRange.
getSignedMin(), StartSRange, MaxBECount,
true);
7109 StartSRange, MaxBECount,
7121ConstantRange ScalarEvolution::getRangeForAffineNoSelfWrappingAR(
7123 ScalarEvolution::RangeSignHint SignHint) {
7124 assert(AddRec->
isAffine() &&
"Non-affine AddRecs are not suppored!\n");
7126 "This only works for non-self-wrapping AddRecs!");
7127 const bool IsSigned = SignHint == HINT_RANGE_SIGNED;
7131 return ConstantRange::getFull(
BitWidth);
7139 return ConstantRange::getFull(
BitWidth);
7143 const SCEV *MaxItersWithoutWrap =
getUDivExpr(RangeWidth, StepAbs);
7145 MaxItersWithoutWrap))
7146 return ConstantRange::getFull(
BitWidth);
7167 ConstantRange StartRange = getRangeRef(Start, SignHint);
7168 ConstantRange EndRange = getRangeRef(End, SignHint);
7169 ConstantRange RangeBetween = StartRange.
unionWith(EndRange);
7173 return RangeBetween;
7178 return ConstantRange::getFull(
BitWidth);
7181 isKnownPredicateViaConstantRanges(LEPred, Start, End))
7182 return RangeBetween;
7184 isKnownPredicateViaConstantRanges(GEPred, Start, End))
7185 return RangeBetween;
7186 return ConstantRange::getFull(
BitWidth);
7191 const APInt &MaxBECount) {
7198 "mismatched bit widths");
7200 struct SelectPattern {
7201 Value *Condition =
nullptr;
7205 explicit SelectPattern(ScalarEvolution &SE,
unsigned BitWidth,
7207 std::optional<unsigned> CastOp;
7221 CastOp = SCast->getSCEVType();
7222 S = SCast->getOperand();
7225 using namespace llvm::PatternMatch;
7232 Condition =
nullptr;
7264 bool isRecognized() {
return Condition !=
nullptr; }
7267 SelectPattern StartPattern(*
this,
BitWidth, Start);
7268 if (!StartPattern.isRecognized())
7269 return ConstantRange::getFull(
BitWidth);
7271 SelectPattern StepPattern(*
this,
BitWidth, Step);
7272 if (!StepPattern.isRecognized())
7273 return ConstantRange::getFull(
BitWidth);
7275 if (StartPattern.Condition != StepPattern.Condition) {
7279 return ConstantRange::getFull(
BitWidth);
7290 const SCEV *TrueStart = this->
getConstant(StartPattern.TrueValue);
7291 const SCEV *TrueStep = this->
getConstant(StepPattern.TrueValue);
7292 const SCEV *FalseStart = this->
getConstant(StartPattern.FalseValue);
7293 const SCEV *FalseStep = this->
getConstant(StepPattern.FalseValue);
7295 ConstantRange TrueRange =
7296 this->getRangeForAffineAR(TrueStart, TrueStep, MaxBECount);
7297 ConstantRange FalseRange =
7298 this->getRangeForAffineAR(FalseStart, FalseStep, MaxBECount);
7320ScalarEvolution::getNonTrivialDefiningScopeBound(
const SCEV *S) {
7334 SmallPtrSet<const SCEV *, 16> Visited;
7336 auto pushOp = [&](
const SCEV *S) {
7337 if (!Visited.
insert(S).second)
7340 if (Visited.
size() > 30) {
7347 for (
const auto *S :
Ops)
7351 while (!Worklist.
empty()) {
7353 if (
auto *DefI = getNonTrivialDefiningScopeBound(S)) {
7354 if (!Bound || DT.dominates(Bound, DefI))
7361 return Bound ? Bound : &*F.getEntryBlock().begin();
7367 return getDefiningScopeBound(
Ops, Discard);
7370bool ScalarEvolution::isGuaranteedToTransferExecutionTo(
const Instruction *
A,
7372 if (
A->getParent() ==
B->getParent() &&
7377 auto *BLoop = LI.getLoopFor(
B->getParent());
7378 if (BLoop && BLoop->getHeader() ==
B->getParent() &&
7379 BLoop->getLoopPreheader() ==
A->getParent() &&
7381 A->getParent()->end()) &&
7388bool ScalarEvolution::isGuaranteedNotToBePoison(
const SCEV *
Op) {
7389 SCEVPoisonCollector PC(
true);
7391 return PC.MaybePoison.empty();
7394bool ScalarEvolution::isGuaranteedNotToCauseUB(
const SCEV *
Op) {
7400 return M && (!
isKnownNonZero(Op1) || !isGuaranteedNotToBePoison(Op1));
7404bool ScalarEvolution::isSCEVExprNeverPoison(
const Instruction *
I) {
7421 for (
const Use &
Op :
I->operands()) {
7427 auto *DefI = getDefiningScopeBound(SCEVOps);
7428 return isGuaranteedToTransferExecutionTo(DefI,
I);
7431bool ScalarEvolution::isAddRecNeverPoison(
const Instruction *
I,
const Loop *L) {
7433 if (isSCEVExprNeverPoison(
I))
7444 auto *ExitingBB =
L->getExitingBlock();
7448 SmallPtrSet<const Value *, 16> KnownPoison;
7457 while (!Worklist.
empty()) {
7460 for (
const Use &U :
Poison->uses()) {
7463 DT.dominates(PoisonUser->
getParent(), ExitingBB))
7467 if (KnownPoison.
insert(PoisonUser).second)
7475ScalarEvolution::LoopProperties
7476ScalarEvolution::getLoopProperties(
const Loop *L) {
7477 using LoopProperties = ScalarEvolution::LoopProperties;
7479 auto Itr = LoopPropertiesCache.find(L);
7480 if (Itr == LoopPropertiesCache.end()) {
7483 return !
SI->isSimple();
7493 return I->mayWriteToMemory();
7496 LoopProperties LP = {
true,
7499 for (
auto *BB :
L->getBlocks())
7500 for (
auto &
I : *BB) {
7502 LP.HasNoAbnormalExits =
false;
7503 if (HasSideEffects(&
I))
7504 LP.HasNoSideEffects =
false;
7505 if (!LP.HasNoAbnormalExits && !LP.HasNoSideEffects)
7509 auto InsertPair = LoopPropertiesCache.insert({
L, LP});
7510 assert(InsertPair.second &&
"We just checked!");
7511 Itr = InsertPair.first;
7524const SCEV *ScalarEvolution::createSCEVIter(
Value *V) {
7530 Stack.emplace_back(V,
true);
7531 Stack.emplace_back(V,
false);
7532 while (!Stack.empty()) {
7533 auto E = Stack.pop_back_val();
7534 Value *CurV = E.getPointer();
7540 const SCEV *CreatedSCEV =
nullptr;
7543 CreatedSCEV = createSCEV(CurV);
7548 CreatedSCEV = getOperandsToCreate(CurV,
Ops);
7552 insertValueToMap(CurV, CreatedSCEV);
7556 Stack.emplace_back(CurV,
true);
7558 Stack.emplace_back(
Op,
false);
7575 if (!DT.isReachableFromEntry(
I->getParent()))
7588 switch (BO->Opcode) {
7589 case Instruction::Add:
7590 case Instruction::Mul: {
7597 Ops.push_back(BO->
Op);
7601 Ops.push_back(BO->RHS);
7605 (BO->Opcode == Instruction::Add &&
7606 (NewBO->Opcode != Instruction::Add &&
7607 NewBO->Opcode != Instruction::Sub)) ||
7608 (BO->Opcode == Instruction::Mul &&
7609 NewBO->Opcode != Instruction::Mul)) {
7610 Ops.push_back(BO->LHS);
7615 if (BO->
Op && (BO->IsNSW || BO->IsNUW)) {
7618 Ops.push_back(BO->LHS);
7626 case Instruction::Sub:
7627 case Instruction::UDiv:
7628 case Instruction::URem:
7630 case Instruction::AShr:
7631 case Instruction::Shl:
7632 case Instruction::Xor:
7636 case Instruction::And:
7637 case Instruction::Or:
7641 case Instruction::LShr:
7648 Ops.push_back(BO->LHS);
7649 Ops.push_back(BO->RHS);
7653 switch (
U->getOpcode()) {
7654 case Instruction::Trunc:
7655 case Instruction::ZExt:
7656 case Instruction::SExt:
7657 case Instruction::PtrToInt:
7658 Ops.push_back(
U->getOperand(0));
7661 case Instruction::BitCast:
7663 Ops.push_back(
U->getOperand(0));
7668 case Instruction::SDiv:
7669 case Instruction::SRem:
7670 Ops.push_back(
U->getOperand(0));
7671 Ops.push_back(
U->getOperand(1));
7674 case Instruction::GetElementPtr:
7676 "GEP source element type must be sized");
7680 case Instruction::IntToPtr:
7683 case Instruction::PHI:
7687 case Instruction::Select: {
7689 auto CanSimplifyToUnknown = [
this,
U]() {
7707 if (CanSimplifyToUnknown())
7714 case Instruction::Call:
7715 case Instruction::Invoke:
7722 switch (
II->getIntrinsicID()) {
7723 case Intrinsic::abs:
7724 Ops.push_back(
II->getArgOperand(0));
7726 case Intrinsic::umax:
7727 case Intrinsic::umin:
7728 case Intrinsic::smax:
7729 case Intrinsic::smin:
7730 case Intrinsic::usub_sat:
7731 case Intrinsic::uadd_sat:
7732 Ops.push_back(
II->getArgOperand(0));
7733 Ops.push_back(
II->getArgOperand(1));
7735 case Intrinsic::start_loop_iterations:
7736 case Intrinsic::annotation:
7737 case Intrinsic::ptr_annotation:
7738 Ops.push_back(
II->getArgOperand(0));
7750const SCEV *ScalarEvolution::createSCEV(
Value *V) {
7759 if (!DT.isReachableFromEntry(
I->getParent()))
7774 switch (BO->Opcode) {
7775 case Instruction::Add: {
7801 if (BO->Opcode == Instruction::Sub)
7809 if (BO->Opcode == Instruction::Sub)
7816 if (!NewBO || (NewBO->Opcode != Instruction::Add &&
7817 NewBO->Opcode != Instruction::Sub)) {
7827 case Instruction::Mul: {
7848 if (!NewBO || NewBO->Opcode != Instruction::Mul) {
7857 case Instruction::UDiv:
7861 case Instruction::URem:
7865 case Instruction::Sub: {
7868 Flags = getNoWrapFlagsFromUB(BO->
Op);
7873 case Instruction::And:
7879 if (CI->isMinusOne())
7881 const APInt &
A = CI->getValue();
7887 unsigned LZ =
A.countl_zero();
7888 unsigned TZ =
A.countr_zero();
7893 APInt EffectiveMask =
7895 if ((LZ != 0 || TZ != 0) && !((~
A & ~Known.
Zero) & EffectiveMask)) {
7898 const SCEV *ShiftedLHS =
nullptr;
7902 unsigned MulZeros = OpC->getAPInt().countr_zero();
7903 unsigned GCD = std::min(MulZeros, TZ);
7908 auto *NewMul =
getMulExpr(MulOps, LHSMul->getNoWrapFlags());
7930 case Instruction::Or:
7939 case Instruction::Xor:
7942 if (CI->isMinusOne())
7951 if (LBO->getOpcode() == Instruction::And &&
7952 LCI->getValue() == CI->getValue())
7953 if (
const SCEVZeroExtendExpr *Z =
7956 const SCEV *Z0 =
Z->getOperand();
7963 if (CI->getValue().isMask(Z0TySize))
7969 APInt Trunc = CI->getValue().trunc(Z0TySize);
7978 case Instruction::Shl:
7996 auto MulFlags = getNoWrapFlagsFromUB(BO->
Op);
8004 ConstantInt *
X = ConstantInt::get(
8010 case Instruction::AShr:
8032 const SCEV *AddTruncateExpr =
nullptr;
8033 ConstantInt *ShlAmtCI =
nullptr;
8034 const SCEV *AddConstant =
nullptr;
8036 if (L &&
L->getOpcode() == Instruction::Add) {
8044 if (LShift && LShift->
getOpcode() == Instruction::Shl) {
8051 APInt AddOperand = AddOperandCI->
getValue().
ashr(AShrAmt);
8059 }
else if (L &&
L->getOpcode() == Instruction::Shl) {
8064 const SCEV *ShlOp0SCEV =
getSCEV(
L->getOperand(0));
8069 if (AddTruncateExpr && ShlAmtCI) {
8081 const APInt &ShlAmt = ShlAmtCI->
getValue();
8085 const SCEV *CompositeExpr =
8087 if (
L->getOpcode() != Instruction::Shl)
8088 CompositeExpr =
getAddExpr(CompositeExpr, AddConstant);
8097 switch (
U->getOpcode()) {
8098 case Instruction::Trunc:
8101 case Instruction::ZExt:
8104 case Instruction::SExt:
8114 if (BO->Opcode == Instruction::Sub && BO->IsNSW) {
8115 Type *Ty =
U->getType();
8123 case Instruction::BitCast:
8129 case Instruction::PtrToInt: {
8132 Type *DstIntTy =
U->getType();
8140 case Instruction::IntToPtr:
8144 case Instruction::SDiv:
8151 case Instruction::SRem:
8158 case Instruction::GetElementPtr:
8161 case Instruction::PHI:
8164 case Instruction::Select:
8165 return createNodeForSelectOrPHI(U,
U->getOperand(0),
U->getOperand(1),
8168 case Instruction::Call:
8169 case Instruction::Invoke:
8174 switch (
II->getIntrinsicID()) {
8175 case Intrinsic::abs:
8179 case Intrinsic::umax:
8183 case Intrinsic::umin:
8187 case Intrinsic::smax:
8191 case Intrinsic::smin:
8195 case Intrinsic::usub_sat: {
8196 const SCEV *
X =
getSCEV(
II->getArgOperand(0));
8197 const SCEV *
Y =
getSCEV(
II->getArgOperand(1));
8201 case Intrinsic::uadd_sat: {
8202 const SCEV *
X =
getSCEV(
II->getArgOperand(0));
8203 const SCEV *
Y =
getSCEV(
II->getArgOperand(1));
8207 case Intrinsic::start_loop_iterations:
8208 case Intrinsic::annotation:
8209 case Intrinsic::ptr_annotation:
8213 case Intrinsic::vscale:
8233 auto *ExitCountType = ExitCount->
getType();
8234 assert(ExitCountType->isIntegerTy());
8236 1 + ExitCountType->getScalarSizeInBits());
8249 auto CanAddOneWithoutOverflow = [&]() {
8251 getRangeRef(ExitCount, RangeSignHint::HINT_RANGE_UNSIGNED);
8262 if (EvalSize > ExitCountSize && CanAddOneWithoutOverflow())
8292 assert(ExitingBlock &&
"Must pass a non-null exiting block!");
8293 assert(L->isLoopExiting(ExitingBlock) &&
8294 "Exiting block must actually branch out of the loop!");
8303 const auto *MaxExitCount =
8311 L->getExitingBlocks(ExitingBlocks);
8313 std::optional<unsigned> Res;
8314 for (
auto *ExitingBB : ExitingBlocks) {
8318 Res = std::gcd(*Res, Multiple);
8320 return Res.value_or(1);
8324 const SCEV *ExitCount) {
8354 assert(ExitingBlock &&
"Must pass a non-null exiting block!");
8355 assert(L->isLoopExiting(ExitingBlock) &&
8356 "Exiting block must actually branch out of the loop!");
8366 return getBackedgeTakenInfo(L).getExact(ExitingBlock,
this);
8368 return getBackedgeTakenInfo(L).getSymbolicMax(ExitingBlock,
this);
8370 return getBackedgeTakenInfo(L).getConstantMax(ExitingBlock,
this);
8380 return getPredicatedBackedgeTakenInfo(L).getExact(ExitingBlock,
this,
8383 return getPredicatedBackedgeTakenInfo(L).getSymbolicMax(ExitingBlock,
this,
8386 return getPredicatedBackedgeTakenInfo(L).getConstantMax(ExitingBlock,
this,
8394 return getPredicatedBackedgeTakenInfo(L).getExact(L,
this, &Preds);
8401 return getBackedgeTakenInfo(L).getExact(L,
this);
8403 return getBackedgeTakenInfo(L).getConstantMax(
this);
8405 return getBackedgeTakenInfo(L).getSymbolicMax(L,
this);
8412 return getPredicatedBackedgeTakenInfo(L).getSymbolicMax(L,
this, &Preds);
8417 return getPredicatedBackedgeTakenInfo(L).getConstantMax(
this, &Preds);
8421 return getBackedgeTakenInfo(L).isConstantMaxOrZero(
this);
8431 for (
PHINode &PN : Header->phis())
8432 if (Visited.
insert(&PN).second)
8436ScalarEvolution::BackedgeTakenInfo &
8437ScalarEvolution::getPredicatedBackedgeTakenInfo(
const Loop *L) {
8438 auto &BTI = getBackedgeTakenInfo(L);
8439 if (BTI.hasFullInfo())
8442 auto Pair = PredicatedBackedgeTakenCounts.try_emplace(L);
8445 return Pair.first->second;
8447 BackedgeTakenInfo
Result =
8448 computeBackedgeTakenCount(L,
true);
8450 return PredicatedBackedgeTakenCounts.find(L)->second = std::move(Result);
8453ScalarEvolution::BackedgeTakenInfo &
8454ScalarEvolution::getBackedgeTakenInfo(
const Loop *L) {
8460 std::pair<DenseMap<const Loop *, BackedgeTakenInfo>::iterator,
bool> Pair =
8461 BackedgeTakenCounts.try_emplace(L);
8463 return Pair.first->second;
8468 BackedgeTakenInfo
Result = computeBackedgeTakenCount(L);
8475 if (
Result.hasAnyInfo()) {
8478 auto LoopUsersIt = LoopUsers.find(L);
8479 if (LoopUsersIt != LoopUsers.end())
8481 forgetMemoizedResults(ToForget);
8484 for (PHINode &PN :
L->getHeader()->phis())
8485 ConstantEvolutionLoopExitValue.erase(&PN);
8493 return BackedgeTakenCounts.find(L)->second = std::move(Result);
8502 BackedgeTakenCounts.clear();
8503 PredicatedBackedgeTakenCounts.clear();
8504 BECountUsers.clear();
8505 LoopPropertiesCache.clear();
8506 ConstantEvolutionLoopExitValue.clear();
8507 ValueExprMap.clear();
8508 ValuesAtScopes.clear();
8509 ValuesAtScopesUsers.clear();
8510 LoopDispositions.clear();
8511 BlockDispositions.clear();
8512 UnsignedRanges.clear();
8513 SignedRanges.clear();
8514 ExprValueMap.clear();
8516 ConstantMultipleCache.clear();
8517 PredicatedSCEVRewrites.clear();
8519 FoldCacheUser.clear();
8521void ScalarEvolution::visitAndClearUsers(
8525 while (!Worklist.
empty()) {
8532 if (It != ValueExprMap.
end()) {
8533 eraseValueFromMap(It->first);
8536 ConstantEvolutionLoopExitValue.erase(PN);
8550 while (!LoopWorklist.
empty()) {
8554 forgetBackedgeTakenCounts(CurrL,
false);
8555 forgetBackedgeTakenCounts(CurrL,
true);
8558 for (
auto I = PredicatedSCEVRewrites.begin();
8559 I != PredicatedSCEVRewrites.end();) {
8560 std::pair<const SCEV *, const Loop *> Entry =
I->first;
8561 if (Entry.second == CurrL)
8562 PredicatedSCEVRewrites.erase(
I++);
8567 auto LoopUsersItr = LoopUsers.find(CurrL);
8568 if (LoopUsersItr != LoopUsers.end())
8573 visitAndClearUsers(Worklist, Visited, ToForget);
8575 LoopPropertiesCache.erase(CurrL);
8578 LoopWorklist.
append(CurrL->begin(), CurrL->end());
8580 forgetMemoizedResults(ToForget);
8597 visitAndClearUsers(Worklist, Visited, ToForget);
8599 forgetMemoizedResults(ToForget);
8611 struct InvalidationRootCollector {
8615 InvalidationRootCollector(
Loop *L) : L(L) {}
8617 bool follow(
const SCEV *S) {
8623 if (L->contains(AddRec->
getLoop()))
8628 bool isDone()
const {
return false; }
8631 InvalidationRootCollector
C(L);
8633 forgetMemoizedResults(
C.Roots);
8646 BlockDispositions.clear();
8647 LoopDispositions.clear();
8664 while (!Worklist.
empty()) {
8666 bool LoopDispoRemoved = LoopDispositions.erase(Curr);
8667 bool BlockDispoRemoved = BlockDispositions.erase(Curr);
8668 if (!LoopDispoRemoved && !BlockDispoRemoved)
8670 auto Users = SCEVUsers.find(Curr);
8671 if (
Users != SCEVUsers.end())
8684const SCEV *ScalarEvolution::BackedgeTakenInfo::getExact(
8688 if (!isComplete() || ExitNotTaken.
empty())
8699 for (
const auto &ENT : ExitNotTaken) {
8700 const SCEV *BECount = ENT.ExactNotTaken;
8703 "We should only have known counts for exiting blocks that dominate "
8706 Ops.push_back(BECount);
8711 assert((Preds || ENT.hasAlwaysTruePredicate()) &&
8712 "Predicate should be always true!");
8721const ScalarEvolution::ExitNotTakenInfo *
8722ScalarEvolution::BackedgeTakenInfo::getExitNotTaken(
8723 const BasicBlock *ExitingBlock,
8724 SmallVectorImpl<const SCEVPredicate *> *Predicates)
const {
8725 for (
const auto &ENT : ExitNotTaken)
8726 if (ENT.ExitingBlock == ExitingBlock) {
8727 if (ENT.hasAlwaysTruePredicate())
8729 else if (Predicates) {
8739const SCEV *ScalarEvolution::BackedgeTakenInfo::getConstantMax(
8741 SmallVectorImpl<const SCEVPredicate *> *Predicates)
const {
8742 if (!getConstantMax())
8745 for (
const auto &ENT : ExitNotTaken)
8746 if (!ENT.hasAlwaysTruePredicate()) {
8754 "No point in having a non-constant max backedge taken count!");
8755 return getConstantMax();
8758const SCEV *ScalarEvolution::BackedgeTakenInfo::getSymbolicMax(
8760 SmallVectorImpl<const SCEVPredicate *> *Predicates) {
8768 for (
const auto &ENT : ExitNotTaken) {
8769 const SCEV *ExitCount = ENT.SymbolicMaxNotTaken;
8772 "We should only have known counts for exiting blocks that "
8778 assert((Predicates || ENT.hasAlwaysTruePredicate()) &&
8779 "Predicate should be always true!");
8782 if (ExitCounts.
empty())
8791bool ScalarEvolution::BackedgeTakenInfo::isConstantMaxOrZero(
8793 auto PredicateNotAlwaysTrue = [](
const ExitNotTakenInfo &ENT) {
8794 return !ENT.hasAlwaysTruePredicate();
8796 return MaxOrZero && !
any_of(ExitNotTaken, PredicateNotAlwaysTrue);
8812 this->ExactNotTaken = E = ConstantMaxNotTaken;
8813 this->SymbolicMaxNotTaken = SymbolicMaxNotTaken = ConstantMaxNotTaken;
8818 "Exact is not allowed to be less precise than Constant Max");
8821 "Exact is not allowed to be less precise than Symbolic Max");
8824 "Symbolic Max is not allowed to be less precise than Constant Max");
8827 "No point in having a non-constant max backedge taken count!");
8829 for (
const auto PredList : PredLists)
8830 for (
const auto *
P : PredList) {
8838 "Backedge count should be int");
8841 "Max backedge count should be int");
8854ScalarEvolution::BackedgeTakenInfo::BackedgeTakenInfo(
8856 bool IsComplete,
const SCEV *ConstantMax,
bool MaxOrZero)
8857 : ConstantMax(ConstantMax), IsComplete(IsComplete), MaxOrZero(MaxOrZero) {
8858 using EdgeExitInfo = ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo;
8860 ExitNotTaken.reserve(ExitCounts.
size());
8861 std::transform(ExitCounts.
begin(), ExitCounts.
end(),
8862 std::back_inserter(ExitNotTaken),
8863 [&](
const EdgeExitInfo &EEI) {
8864 BasicBlock *ExitBB = EEI.first;
8865 const ExitLimit &EL = EEI.second;
8866 return ExitNotTakenInfo(ExitBB, EL.ExactNotTaken,
8867 EL.ConstantMaxNotTaken, EL.SymbolicMaxNotTaken,
8872 "No point in having a non-constant max backedge taken count!");
8876ScalarEvolution::BackedgeTakenInfo
8877ScalarEvolution::computeBackedgeTakenCount(
const Loop *L,
8878 bool AllowPredicates) {
8880 L->getExitingBlocks(ExitingBlocks);
8882 using EdgeExitInfo = ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo;
8885 bool CouldComputeBECount =
true;
8887 const SCEV *MustExitMaxBECount =
nullptr;
8888 const SCEV *MayExitMaxBECount =
nullptr;
8889 bool MustExitMaxOrZero =
false;
8890 bool IsOnlyExit = ExitingBlocks.
size() == 1;
8902 if (ExitIfTrue == CI->
isZero())
8906 ExitLimit EL = computeExitLimit(L, ExitBB, IsOnlyExit, AllowPredicates);
8908 assert((AllowPredicates || EL.Predicates.empty()) &&
8909 "Predicated exit limit when predicates are not allowed!");
8914 ++NumExitCountsComputed;
8918 CouldComputeBECount =
false;
8925 "Exact is known but symbolic isn't?");
8926 ++NumExitCountsNotComputed;
8941 DT.dominates(ExitBB, Latch)) {
8942 if (!MustExitMaxBECount) {
8943 MustExitMaxBECount = EL.ConstantMaxNotTaken;
8944 MustExitMaxOrZero = EL.MaxOrZero;
8947 EL.ConstantMaxNotTaken);
8951 MayExitMaxBECount = EL.ConstantMaxNotTaken;
8954 EL.ConstantMaxNotTaken);
8958 const SCEV *MaxBECount = MustExitMaxBECount ? MustExitMaxBECount :
8962 bool MaxOrZero = (MustExitMaxOrZero && ExitingBlocks.size() == 1);
8968 for (
const auto &Pair : ExitCounts) {
8970 BECountUsers[Pair.second.ExactNotTaken].insert({
L, AllowPredicates});
8972 BECountUsers[Pair.second.SymbolicMaxNotTaken].insert(
8973 {
L, AllowPredicates});
8975 return BackedgeTakenInfo(std::move(ExitCounts), CouldComputeBECount,
8976 MaxBECount, MaxOrZero);
8979ScalarEvolution::ExitLimit
8980ScalarEvolution::computeExitLimit(
const Loop *L, BasicBlock *ExitingBlock,
8981 bool IsOnlyExit,
bool AllowPredicates) {
8982 assert(
L->contains(ExitingBlock) &&
"Exit count for non-loop block?");
8986 if (!Latch || !DT.dominates(ExitingBlock, Latch))
8994 "It should have one successor in loop and one exit block!");
9005 if (!
L->contains(SBB)) {
9010 assert(Exit &&
"Exiting block must have at least one exit");
9011 return computeExitLimitFromSingleExitSwitch(
9012 L, SI, Exit, IsOnlyExit);
9019 const Loop *L,
Value *ExitCond,
bool ExitIfTrue,
bool ControlsOnlyExit,
9020 bool AllowPredicates) {
9021 ScalarEvolution::ExitLimitCacheTy Cache(L, ExitIfTrue, AllowPredicates);
9022 return computeExitLimitFromCondCached(Cache, L, ExitCond, ExitIfTrue,
9023 ControlsOnlyExit, AllowPredicates);
9026std::optional<ScalarEvolution::ExitLimit>
9027ScalarEvolution::ExitLimitCache::find(
const Loop *L,
Value *ExitCond,
9028 bool ExitIfTrue,
bool ControlsOnlyExit,
9029 bool AllowPredicates) {
9031 (void)this->ExitIfTrue;
9032 (void)this->AllowPredicates;
9034 assert(this->L == L && this->ExitIfTrue == ExitIfTrue &&
9035 this->AllowPredicates == AllowPredicates &&
9036 "Variance in assumed invariant key components!");
9037 auto Itr = TripCountMap.find({ExitCond, ControlsOnlyExit});
9038 if (Itr == TripCountMap.end())
9039 return std::nullopt;
9043void ScalarEvolution::ExitLimitCache::insert(
const Loop *L,
Value *ExitCond,
9045 bool ControlsOnlyExit,
9046 bool AllowPredicates,
9048 assert(this->L == L && this->ExitIfTrue == ExitIfTrue &&
9049 this->AllowPredicates == AllowPredicates &&
9050 "Variance in assumed invariant key components!");
9052 auto InsertResult = TripCountMap.insert({{ExitCond, ControlsOnlyExit}, EL});
9053 assert(InsertResult.second &&
"Expected successful insertion!");
9058ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCondCached(
9059 ExitLimitCacheTy &Cache,
const Loop *L,
Value *ExitCond,
bool ExitIfTrue,
9060 bool ControlsOnlyExit,
bool AllowPredicates) {
9062 if (
auto MaybeEL = Cache.find(L, ExitCond, ExitIfTrue, ControlsOnlyExit,
9066 ExitLimit EL = computeExitLimitFromCondImpl(
9067 Cache, L, ExitCond, ExitIfTrue, ControlsOnlyExit, AllowPredicates);
9068 Cache.insert(L, ExitCond, ExitIfTrue, ControlsOnlyExit, AllowPredicates, EL);
9072ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCondImpl(
9073 ExitLimitCacheTy &Cache,
const Loop *L,
Value *ExitCond,
bool ExitIfTrue,
9074 bool ControlsOnlyExit,
bool AllowPredicates) {
9076 if (
auto LimitFromBinOp = computeExitLimitFromCondFromBinOp(
9077 Cache, L, ExitCond, ExitIfTrue, ControlsOnlyExit, AllowPredicates))
9078 return *LimitFromBinOp;
9084 computeExitLimitFromICmp(L, ExitCondICmp, ExitIfTrue, ControlsOnlyExit);
9085 if (EL.hasFullInfo() || !AllowPredicates)
9089 return computeExitLimitFromICmp(L, ExitCondICmp, ExitIfTrue,
9109 const WithOverflowInst *WO;
9124 auto EL = computeExitLimitFromICmp(L, Pred,
LHS,
getConstant(NewRHSC),
9125 ControlsOnlyExit, AllowPredicates);
9126 if (EL.hasAnyInfo())
9131 return computeExitCountExhaustively(L, ExitCond, ExitIfTrue);
9134std::optional<ScalarEvolution::ExitLimit>
9135ScalarEvolution::computeExitLimitFromCondFromBinOp(
9136 ExitLimitCacheTy &Cache,
const Loop *L,
Value *ExitCond,
bool ExitIfTrue,
9137 bool ControlsOnlyExit,
bool AllowPredicates) {
9146 return std::nullopt;
9151 bool EitherMayExit = IsAnd ^ ExitIfTrue;
9152 ExitLimit EL0 = computeExitLimitFromCondCached(
9153 Cache, L, Op0, ExitIfTrue, ControlsOnlyExit && !EitherMayExit,
9155 ExitLimit EL1 = computeExitLimitFromCondCached(
9156 Cache, L, Op1, ExitIfTrue, ControlsOnlyExit && !EitherMayExit,
9160 const Constant *NeutralElement = ConstantInt::get(ExitCond->
getType(), IsAnd);
9162 return Op1 == NeutralElement ? EL0 : EL1;
9164 return Op0 == NeutralElement ? EL1 : EL0;
9169 if (EitherMayExit) {
9179 ConstantMaxBECount = EL1.ConstantMaxNotTaken;
9181 ConstantMaxBECount = EL0.ConstantMaxNotTaken;
9184 EL1.ConstantMaxNotTaken);
9186 SymbolicMaxBECount = EL1.SymbolicMaxNotTaken;
9188 SymbolicMaxBECount = EL0.SymbolicMaxNotTaken;
9191 EL0.SymbolicMaxNotTaken, EL1.SymbolicMaxNotTaken, UseSequentialUMin);
9195 if (EL0.ExactNotTaken == EL1.ExactNotTaken)
9196 BECount = EL0.ExactNotTaken;
9209 SymbolicMaxBECount =
9211 return ExitLimit(BECount, ConstantMaxBECount, SymbolicMaxBECount,
false,
9215ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp(
9216 const Loop *L, ICmpInst *ExitCond,
bool ExitIfTrue,
bool ControlsOnlyExit,
9217 bool AllowPredicates) {
9229 ExitLimit EL = computeExitLimitFromICmp(L, Pred,
LHS,
RHS, ControlsOnlyExit,
9231 if (EL.hasAnyInfo())
9234 auto *ExhaustiveCount =
9235 computeExitCountExhaustively(L, ExitCond, ExitIfTrue);
9238 return ExhaustiveCount;
9240 return computeShiftCompareExitLimit(ExitCond->
getOperand(0),
9243ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp(
9244 const Loop *L, CmpPredicate Pred,
const SCEV *
LHS,
const SCEV *
RHS,
9245 bool ControlsOnlyExit,
bool AllowPredicates) {
9270 ConstantRange CompRange =
9286 auto *InnerLHS =
LHS;
9288 InnerLHS = ZExt->getOperand();
9335 if (EL.hasAnyInfo())
9352 if (EL.hasAnyInfo())
return EL;
9384 ExitLimit EL = howManyLessThans(
LHS,
RHS, L, IsSigned, ControlsOnlyExit,
9386 if (EL.hasAnyInfo())
9402 ExitLimit EL = howManyGreaterThans(
LHS,
RHS, L, IsSigned, ControlsOnlyExit,
9404 if (EL.hasAnyInfo())
9415ScalarEvolution::ExitLimit
9416ScalarEvolution::computeExitLimitFromSingleExitSwitch(
const Loop *L,
9418 BasicBlock *ExitingBlock,
9419 bool ControlsOnlyExit) {
9420 assert(!
L->contains(ExitingBlock) &&
"Not an exiting block!");
9423 if (
Switch->getDefaultDest() == ExitingBlock)
9427 "Default case must not exit the loop!");
9433 if (EL.hasAnyInfo())
9445 "Evaluation of SCEV at constant didn't fold correctly?");
9449ScalarEvolution::ExitLimit ScalarEvolution::computeShiftCompareExitLimit(
9459 const BasicBlock *Predecessor =
L->getLoopPredecessor();
9465 auto MatchPositiveShift =
9468 using namespace PatternMatch;
9470 ConstantInt *ShiftAmt;
9472 OutOpCode = Instruction::LShr;
9474 OutOpCode = Instruction::AShr;
9476 OutOpCode = Instruction::Shl;
9491 auto MatchShiftRecurrence =
9493 std::optional<Instruction::BinaryOps> PostShiftOpCode;
9508 if (MatchPositiveShift(
LHS, V, OpC)) {
9509 PostShiftOpCode = OpC;
9515 if (!PNOut || PNOut->getParent() !=
L->getHeader())
9518 Value *BEValue = PNOut->getIncomingValueForBlock(Latch);
9524 MatchPositiveShift(BEValue, OpLHS, OpCodeOut) &&
9531 (!PostShiftOpCode || *PostShiftOpCode == OpCodeOut);
9536 if (!MatchShiftRecurrence(
LHS, PN, OpCode))
9548 ConstantInt *StableValue =
nullptr;
9553 case Instruction::AShr: {
9561 StableValue = ConstantInt::get(Ty, 0);
9563 StableValue = ConstantInt::get(Ty, -1,
true);
9569 case Instruction::LShr:
9570 case Instruction::Shl:
9580 "Otherwise cannot be an operand to a branch instruction");
9582 if (
Result->isZeroValue()) {
9584 const SCEV *UpperBound =
9601 if (
const Function *
F = CI->getCalledFunction())
9610 if (!L->contains(
I))
return false;
9615 return L->getHeader() ==
I->getParent();
9691 if (!
I)
return nullptr;
9704 std::vector<Constant*>
Operands(
I->getNumOperands());
9706 for (
unsigned i = 0, e =
I->getNumOperands(); i != e; ++i) {
9715 if (!
C)
return nullptr;
9737 if (IncomingVal != CurrentVal) {
9740 IncomingVal = CurrentVal;
9752ScalarEvolution::getConstantEvolutionLoopExitValue(PHINode *PN,
9755 auto [
I,
Inserted] = ConstantEvolutionLoopExitValue.try_emplace(PN);
9764 DenseMap<Instruction *, Constant *> CurrentIterVals;
9766 assert(PN->
getParent() == Header &&
"Can't evaluate PHI not in loop header!");
9772 for (PHINode &
PHI : Header->phis()) {
9774 CurrentIterVals[&
PHI] = StartCST;
9776 if (!CurrentIterVals.
count(PN))
9777 return RetVal =
nullptr;
9783 "BEs is <= MaxBruteForceIterations which is an 'unsigned'!");
9786 unsigned IterationNum = 0;
9788 for (; ; ++IterationNum) {
9789 if (IterationNum == NumIterations)
9790 return RetVal = CurrentIterVals[PN];
9794 DenseMap<Instruction *, Constant *> NextIterVals;
9799 NextIterVals[PN] = NextPHI;
9801 bool StoppedEvolving = NextPHI == CurrentIterVals[PN];
9807 for (
const auto &
I : CurrentIterVals) {
9809 if (!
PHI ||
PHI == PN ||
PHI->getParent() != Header)
continue;
9814 for (
const auto &
I : PHIsToCompute) {
9815 PHINode *
PHI =
I.first;
9818 Value *BEValue =
PHI->getIncomingValueForBlock(Latch);
9821 if (NextPHI !=
I.second)
9822 StoppedEvolving =
false;
9827 if (StoppedEvolving)
9828 return RetVal = CurrentIterVals[PN];
9830 CurrentIterVals.swap(NextIterVals);
9834const SCEV *ScalarEvolution::computeExitCountExhaustively(
const Loop *L,
9844 DenseMap<Instruction *, Constant *> CurrentIterVals;
9846 assert(PN->
getParent() == Header &&
"Can't evaluate PHI not in loop header!");
9849 assert(Latch &&
"Should follow from NumIncomingValues == 2!");
9851 for (PHINode &
PHI : Header->phis()) {
9853 CurrentIterVals[&
PHI] = StartCST;
9855 if (!CurrentIterVals.
count(PN))
9863 for (
unsigned IterationNum = 0; IterationNum != MaxIterations;++IterationNum){
9870 if (CondVal->getValue() == uint64_t(ExitWhen)) {
9871 ++NumBruteForceTripCountsComputed;
9876 DenseMap<Instruction *, Constant *> NextIterVals;
9882 for (
const auto &
I : CurrentIterVals) {
9884 if (!
PHI ||
PHI->getParent() != Header)
continue;
9887 for (PHINode *
PHI : PHIsToCompute) {
9889 if (NextPHI)
continue;
9891 Value *BEValue =
PHI->getIncomingValueForBlock(Latch);
9894 CurrentIterVals.
swap(NextIterVals);
9905 for (
auto &LS : Values)
9907 return LS.second ? LS.second : V;
9912 const SCEV *
C = computeSCEVAtScope(V, L);
9913 for (
auto &LS :
reverse(ValuesAtScopes[V]))
9914 if (LS.first == L) {
9917 ValuesAtScopesUsers[
C].push_back({L, V});
9928 switch (V->getSCEVType()) {
9961 assert(!
C->getType()->isPointerTy() &&
9962 "Can only have one pointer, and it must be last");
9989ScalarEvolution::getWithOperands(
const SCEV *S,
9990 SmallVectorImpl<const SCEV *> &NewOps) {
10024const SCEV *ScalarEvolution::computeSCEVAtScope(
const SCEV *V,
const Loop *L) {
10025 switch (
V->getSCEVType()) {
10036 for (
unsigned i = 0, e = AddRec->
getNumOperands(); i != e; ++i) {
10047 for (++i; i !=
e; ++i)
10091 for (
unsigned i = 0, e =
Ops.size(); i != e; ++i) {
10093 if (OpAtScope !=
Ops[i]) {
10101 for (++i; i !=
e; ++i) {
10106 return getWithOperands(V, NewOps);
10121 const Loop *CurrLoop = this->LI[
I->getParent()];
10132 if (BackedgeTakenCount->
isZero()) {
10133 Value *InitValue =
nullptr;
10134 bool MultipleInitValues =
false;
10140 MultipleInitValues =
true;
10145 if (!MultipleInitValues && InitValue)
10154 unsigned InLoopPred =
10165 getConstantEvolutionLoopExitValue(PN, BTCC->getAPInt(), CurrLoop);
10181 bool MadeImprovement =
false;
10196 MadeImprovement |= OrigV != OpV;
10201 assert(
C->getType() ==
Op->getType() &&
"Type mismatch");
10206 if (!MadeImprovement)
10227const SCEV *ScalarEvolution::stripInjectiveFunctions(
const SCEV *S)
const {
10229 return stripInjectiveFunctions(ZExt->getOperand());
10231 return stripInjectiveFunctions(SExt->getOperand());
10250 assert(
A != 0 &&
"A must be non-zero.");
10286 APInt AD =
A.lshr(Mult2).trunc(BW - Mult2);
10306static std::optional<std::tuple<APInt, APInt, APInt, APInt, unsigned>>
10312 LLVM_DEBUG(
dbgs() << __func__ <<
": analyzing quadratic addrec: "
10313 << *AddRec <<
'\n');
10316 if (!LC || !MC || !
NC) {
10317 LLVM_DEBUG(
dbgs() << __func__ <<
": coefficients are not constant\n");
10318 return std::nullopt;
10324 assert(!
N.isZero() &&
"This is not a quadratic addrec");
10332 N =
N.sext(NewWidth);
10333 M = M.sext(NewWidth);
10334 L = L.sext(NewWidth);
10351 <<
"x + " <<
C <<
", coeff bw: " << NewWidth
10352 <<
", multiplied by " <<
T <<
'\n');
10361 std::optional<APInt>
Y) {
10363 unsigned W = std::max(
X->getBitWidth(),
Y->getBitWidth());
10366 return XW.
slt(YW) ? *
X : *
Y;
10369 return std::nullopt;
10370 return X ? *
X : *
Y;
10387 return std::nullopt;
10388 unsigned W =
X->getBitWidth();
10408static std::optional<APInt>
10414 return std::nullopt;
10417 LLVM_DEBUG(
dbgs() << __func__ <<
": solving for unsigned overflow\n");
10418 std::optional<APInt>
X =
10421 return std::nullopt;
10426 return std::nullopt;
10441static std::optional<APInt>
10445 "Starting value of addrec should be 0");
10446 LLVM_DEBUG(
dbgs() << __func__ <<
": solving boundary crossing for range "
10447 <<
Range <<
", addrec " << *AddRec <<
'\n');
10451 "Addrec's initial value should be in range");
10457 return std::nullopt;
10467 auto SolveForBoundary =
10468 [&](
APInt Bound) -> std::pair<std::optional<APInt>,
bool> {
10471 LLVM_DEBUG(
dbgs() <<
"SolveQuadraticAddRecRange: checking boundary "
10472 << Bound <<
" (before multiplying by " << M <<
")\n");
10475 std::optional<APInt> SO;
10478 "signed overflow\n");
10482 "unsigned overflow\n");
10483 std::optional<APInt> UO =
10486 auto LeavesRange = [&] (
const APInt &
X) {
10503 return {std::nullopt,
false};
10508 if (LeavesRange(*Min))
10509 return { Min,
true };
10510 std::optional<APInt> Max = Min == SO ? UO : SO;
10511 if (LeavesRange(*Max))
10512 return { Max,
true };
10515 return {std::nullopt,
true};
10522 auto SL = SolveForBoundary(
Lower);
10523 auto SU = SolveForBoundary(
Upper);
10526 if (!SL.second || !SU.second)
10527 return std::nullopt;
10570ScalarEvolution::ExitLimit ScalarEvolution::howFarToZero(
const SCEV *V,
10572 bool ControlsOnlyExit,
10573 bool AllowPredicates) {
10584 if (
C->getValue()->isZero())
return C;
10588 const SCEVAddRecExpr *AddRec =
10591 if (!AddRec && AllowPredicates)
10597 if (!AddRec || AddRec->
getLoop() != L)
10608 return ExitLimit(R, R, R,
false, Predicates);
10666 const SCEV *DistancePlusOne =
getAddExpr(Distance, One);
10692 const SCEV *
Exact =
10700 const SCEV *SymbolicMax =
10702 return ExitLimit(
Exact, ConstantMax, SymbolicMax,
false, Predicates);
10711 AllowPredicates ? &Predicates :
nullptr, *
this);
10719 return ExitLimit(
E, M, S,
false, Predicates);
10722ScalarEvolution::ExitLimit
10723ScalarEvolution::howFarToNonZero(
const SCEV *V,
const Loop *L) {
10731 if (!
C->getValue()->isZero())
10741std::pair<const BasicBlock *, const BasicBlock *>
10742ScalarEvolution::getPredecessorWithUniqueSuccessorForBB(
const BasicBlock *BB)
10753 if (
const Loop *L = LI.getLoopFor(BB))
10754 return {
L->getLoopPredecessor(),
L->getHeader()};
10756 return {
nullptr, BB};
10765 if (
A ==
B)
return true;
10780 if (ComputesEqualValues(AI, BI))
10789 if (!
Add ||
Add->getNumOperands() != 2)
10792 ME && ME->getNumOperands() == 2 && ME->getOperand(0)->isAllOnesValue()) {
10793 LHS =
Add->getOperand(1);
10794 RHS = ME->getOperand(1);
10798 ME && ME->getNumOperands() == 2 && ME->getOperand(0)->isAllOnesValue()) {
10799 LHS =
Add->getOperand(0);
10800 RHS = ME->getOperand(1);
10811 auto TrivialCase = [&](
bool TriviallyTrue) {
10820 const SCEV *NewLHS, *NewRHS;
10844 return TrivialCase(
false);
10845 return TrivialCase(
true);
10868 const APInt &
RA = RC->getAPInt();
10870 bool SimplifiedByConstantRange =
false;
10875 return TrivialCase(
true);
10877 return TrivialCase(
false);
10886 Changed = SimplifiedByConstantRange =
true;
10890 if (!SimplifiedByConstantRange) {
10907 assert(!
RA.isMinValue() &&
"Should have been caught earlier!");
10913 assert(!
RA.isMaxValue() &&
"Should have been caught earlier!");
10919 assert(!
RA.isMinSignedValue() &&
"Should have been caught earlier!");
10925 assert(!
RA.isMaxSignedValue() &&
"Should have been caught earlier!");
10937 return TrivialCase(
true);
10939 return TrivialCase(
false);
11044 auto NonRecursive = [
this, OrNegative](
const SCEV *S) {
11046 return C->getAPInt().isPowerOf2() ||
11047 (OrNegative &&
C->getAPInt().isNegatedPowerOf2());
11050 return isa<SCEVVScale>(S) && F.hasFnAttribute(Attribute::VScaleRange);
11053 if (NonRecursive(S))
11079 APInt C = Cst->getAPInt();
11080 return C.urem(M) == 0;
11088 const SCEV *SmodM =
11103 for (
auto *
A : Assumptions)
11104 if (
A->implies(
P, *
this))
11112std::pair<const SCEV *, const SCEV *>
11115 const SCEV *Start = SCEVInitRewriter::rewrite(S, L, *
this);
11117 return { Start, Start };
11119 const SCEV *
PostInc = SCEVPostIncRewriter::rewrite(S, L, *
this);
11128 getUsedLoops(LHS, LoopsUsed);
11129 getUsedLoops(RHS, LoopsUsed);
11131 if (LoopsUsed.
empty())
11136 for (
const auto *L1 : LoopsUsed)
11137 for (
const auto *L2 : LoopsUsed)
11138 assert((DT.dominates(L1->getHeader(), L2->getHeader()) ||
11139 DT.dominates(L2->getHeader(), L1->getHeader())) &&
11140 "Domination relationship is not a linear order");
11170 SplitRHS.second) &&
11182 if (isKnownPredicateViaSplitting(Pred, LHS, RHS))
11186 return isKnownViaNonRecursiveReasoning(Pred, LHS, RHS);
11196 return std::nullopt;
11211 if (KnownWithoutContext)
11212 return KnownWithoutContext;
11219 return std::nullopt;
11225 const Loop *L = LHS->getLoop();
11230std::optional<ScalarEvolution::MonotonicPredicateType>
11233 auto Result = getMonotonicPredicateTypeImpl(LHS, Pred);
11239 auto ResultSwapped =
11242 assert(*ResultSwapped != *Result &&
11243 "monotonicity should flip as we flip the predicate");
11250std::optional<ScalarEvolution::MonotonicPredicateType>
11251ScalarEvolution::getMonotonicPredicateTypeImpl(
const SCEVAddRecExpr *LHS,
11265 return std::nullopt;
11269 "Should be greater or less!");
11273 if (!LHS->hasNoUnsignedWrap())
11274 return std::nullopt;
11278 "Relational predicate is either signed or unsigned!");
11279 if (!
LHS->hasNoSignedWrap())
11280 return std::nullopt;
11282 const SCEV *Step =
LHS->getStepRecurrence(*
this);
11290 return std::nullopt;
11293std::optional<ScalarEvolution::LoopInvariantPredicate>
11300 return std::nullopt;
11307 if (!ArLHS || ArLHS->
getLoop() != L)
11308 return std::nullopt;
11312 return std::nullopt;
11338 return std::nullopt;
11375 return std::nullopt;
11378std::optional<ScalarEvolution::LoopInvariantPredicate>
11383 Pred, LHS, RHS, L, CtxI, MaxIter))
11391 for (
auto *
Op :
UMin->operands())
11393 Pred, LHS, RHS, L, CtxI,
Op))
11395 return std::nullopt;
11398std::optional<ScalarEvolution::LoopInvariantPredicate>
11413 return std::nullopt;
11420 if (!AR || AR->
getLoop() != L)
11421 return std::nullopt;
11425 return std::nullopt;
11431 if (Step != One && Step != MinusOne)
11432 return std::nullopt;
11438 return std::nullopt;
11444 return std::nullopt;
11452 if (Step == MinusOne)
11456 return std::nullopt;
11462bool ScalarEvolution::isKnownPredicateViaConstantRanges(
CmpPredicate Pred,
11468 auto CheckRange = [&](
bool IsSigned) {
11471 return RangeLHS.
icmp(Pred, RangeRHS);
11480 if (CheckRange(
true) || CheckRange(
false))
11489bool ScalarEvolution::isKnownPredicateViaNoOverflow(CmpPredicate Pred,
11496 auto MatchBinaryAddToConst = [
this](
const SCEV *
X,
const SCEV *
Y,
11497 APInt &OutC1, APInt &OutC2,
11499 const SCEV *XNonConstOp, *XConstOp;
11500 const SCEV *YNonConstOp, *YConstOp;
11504 if (!splitBinaryAdd(
X, XConstOp, XNonConstOp, XFlagsPresent)) {
11507 XFlagsPresent = ExpectedFlags;
11512 if (!splitBinaryAdd(
Y, YConstOp, YNonConstOp, YFlagsPresent)) {
11515 YFlagsPresent = ExpectedFlags;
11518 if (YNonConstOp != XNonConstOp)
11526 if ((YFlagsPresent & ExpectedFlags) != ExpectedFlags)
11529 (XFlagsPresent & ExpectedFlags) != ExpectedFlags) {
11589bool ScalarEvolution::isKnownPredicateViaSplitting(CmpPredicate Pred,
11611bool ScalarEvolution::isImpliedViaGuard(
const BasicBlock *BB, CmpPredicate Pred,
11612 const SCEV *
LHS,
const SCEV *
RHS) {
11617 return any_of(*BB, [&](
const Instruction &
I) {
11618 using namespace llvm::PatternMatch;
11623 isImpliedCond(Pred,
LHS,
RHS, Condition,
false);
11637 if (!L || !DT.isReachableFromEntry(L->getHeader()))
11642 "This cannot be done on broken IR!");
11645 if (isKnownViaNonRecursiveReasoning(Pred, LHS, RHS))
11654 if (LoopContinuePredicate && LoopContinuePredicate->
isConditional() &&
11655 isImpliedCond(Pred, LHS, RHS,
11657 LoopContinuePredicate->
getSuccessor(0) != L->getHeader()))
11662 if (WalkingBEDominatingConds)
11668 const auto &BETakenInfo = getBackedgeTakenInfo(L);
11669 const SCEV *LatchBECount = BETakenInfo.getExact(Latch,
this);
11676 const SCEV *LoopCounter =
11684 for (
auto &AssumeVH : AC.assumptions()) {
11691 if (isImpliedCond(Pred, LHS, RHS, CI->getArgOperand(0),
false))
11695 if (isImpliedViaGuard(Latch, Pred, LHS, RHS))
11698 for (
DomTreeNode *DTN = DT[Latch], *HeaderDTN = DT[L->getHeader()];
11699 DTN != HeaderDTN; DTN = DTN->getIDom()) {
11700 assert(DTN &&
"should reach the loop header before reaching the root!");
11703 if (isImpliedViaGuard(BB, Pred, LHS, RHS))
11711 if (!ContinuePredicate || !ContinuePredicate->
isConditional())
11725 assert(DT.dominates(DominatingEdge, Latch) &&
"should be!");
11727 if (isImpliedCond(Pred, LHS, RHS, Condition,
11741 if (!DT.isReachableFromEntry(BB))
11745 "This cannot be done on broken IR!");
11753 const bool ProvingStrictComparison =
11755 bool ProvedNonStrictComparison =
false;
11756 bool ProvedNonEquality =
false;
11759 if (!ProvedNonStrictComparison)
11760 ProvedNonStrictComparison = Fn(NonStrictPredicate);
11761 if (!ProvedNonEquality)
11763 if (ProvedNonStrictComparison && ProvedNonEquality)
11768 if (ProvingStrictComparison) {
11770 return isKnownViaNonRecursiveReasoning(
P, LHS, RHS);
11772 if (SplitAndProve(ProofFn))
11777 auto ProveViaCond = [&](
const Value *Condition,
bool Inverse) {
11779 if (isImpliedCond(Pred, LHS, RHS, Condition,
Inverse, CtxI))
11781 if (ProvingStrictComparison) {
11783 return isImpliedCond(
P, LHS, RHS, Condition,
Inverse, CtxI);
11785 if (SplitAndProve(ProofFn))
11794 const Loop *ContainingLoop = LI.getLoopFor(BB);
11796 if (ContainingLoop && ContainingLoop->
getHeader() == BB)
11800 for (std::pair<const BasicBlock *, const BasicBlock *> Pair(PredBB, BB);
11801 Pair.first; Pair = getPredecessorWithUniqueSuccessorForBB(Pair.first)) {
11813 for (
auto &AssumeVH : AC.assumptions()) {
11817 if (!DT.dominates(CI, BB))
11820 if (ProveViaCond(CI->getArgOperand(0),
false))
11826 F.getParent(), Intrinsic::experimental_guard);
11828 for (
const auto *GU : GuardDecl->users())
11830 if (Guard->getFunction() == BB->
getParent() && DT.dominates(Guard, BB))
11831 if (ProveViaCond(Guard->getArgOperand(0),
false))
11846 "LHS is not available at Loop Entry");
11848 "RHS is not available at Loop Entry");
11850 if (isKnownViaNonRecursiveReasoning(Pred, LHS, RHS))
11861 if (FoundCondValue ==
11865 if (!PendingLoopPredicates.insert(FoundCondValue).second)
11869 make_scope_exit([&]() { PendingLoopPredicates.erase(FoundCondValue); });
11872 const Value *Op0, *Op1;
11875 return isImpliedCond(Pred,
LHS,
RHS, Op0,
Inverse, CtxI) ||
11879 return isImpliedCond(Pred,
LHS,
RHS, Op0, Inverse, CtxI) ||
11880 isImpliedCond(Pred,
LHS,
RHS, Op1, Inverse, CtxI);
11884 if (!ICI)
return false;
11888 CmpPredicate FoundPred;
11897 return isImpliedCond(Pred,
LHS,
RHS, FoundPred, FoundLHS, FoundRHS, CtxI);
11900bool ScalarEvolution::isImpliedCond(CmpPredicate Pred,
const SCEV *
LHS,
11901 const SCEV *
RHS, CmpPredicate FoundPred,
11902 const SCEV *FoundLHS,
const SCEV *FoundRHS,
11903 const Instruction *CtxI) {
11913 auto *WideType = FoundLHS->
getType();
11925 TruncFoundLHS, TruncFoundRHS, CtxI))
11951 return isImpliedCondBalancedTypes(Pred,
LHS,
RHS, FoundPred, FoundLHS,
11955bool ScalarEvolution::isImpliedCondBalancedTypes(
11956 CmpPredicate Pred,
const SCEV *
LHS,
const SCEV *
RHS, CmpPredicate FoundPred,
11957 const SCEV *FoundLHS,
const SCEV *FoundRHS,
const Instruction *CtxI) {
11960 "Types should be balanced!");
11967 if (FoundLHS == FoundRHS)
11971 if (
LHS == FoundRHS ||
RHS == FoundLHS) {
11983 return isImpliedCondOperands(*
P,
LHS,
RHS, FoundLHS, FoundRHS, CtxI);
12000 LHS, FoundLHS, FoundRHS, CtxI);
12002 return isImpliedCondOperands(*
P,
LHS,
RHS, FoundRHS, FoundLHS, CtxI);
12024 assert(P1 != P2 &&
"Handled earlier!");
12028 if (IsSignFlippedPredicate(Pred, FoundPred)) {
12033 return isImpliedCondOperands(Pred,
LHS,
RHS, FoundLHS, FoundRHS, CtxI);
12036 CmpPredicate CanonicalPred = Pred, CanonicalFoundPred = FoundPred;
12037 const SCEV *CanonicalLHS =
LHS, *CanonicalRHS =
RHS,
12038 *CanonicalFoundLHS = FoundLHS, *CanonicalFoundRHS = FoundRHS;
12043 std::swap(CanonicalFoundLHS, CanonicalFoundRHS);
12054 return isImpliedCondOperands(CanonicalFoundPred, CanonicalLHS,
12055 CanonicalRHS, CanonicalFoundLHS,
12056 CanonicalFoundRHS);
12061 return isImpliedCondOperands(CanonicalFoundPred, CanonicalLHS,
12062 CanonicalRHS, CanonicalFoundLHS,
12063 CanonicalFoundRHS);
12070 const SCEVConstant *
C =
nullptr;
12071 const SCEV *
V =
nullptr;
12089 if (Min ==
C->getAPInt()) {
12094 APInt SharperMin = Min + 1;
12097 case ICmpInst::ICMP_SGE:
12098 case ICmpInst::ICMP_UGE:
12101 if (isImpliedCondOperands(Pred, LHS, RHS, V, getConstant(SharperMin),
12106 case ICmpInst::ICMP_SGT:
12107 case ICmpInst::ICMP_UGT:
12117 if (isImpliedCondOperands(Pred, LHS, RHS, V, getConstant(Min), CtxI))
12122 case ICmpInst::ICMP_SLE:
12123 case ICmpInst::ICMP_ULE:
12124 if (isImpliedCondOperands(ICmpInst::getSwappedCmpPredicate(Pred), RHS,
12125 LHS, V, getConstant(SharperMin), CtxI))
12129 case ICmpInst::ICMP_SLT:
12130 case ICmpInst::ICMP_ULT:
12131 if (isImpliedCondOperands(ICmpInst::getSwappedCmpPredicate(Pred), RHS,
12132 LHS, V, getConstant(Min), CtxI))
12146 if (isImpliedCondOperands(Pred,
LHS,
RHS, FoundLHS, FoundRHS, CtxI))
12150 if (isImpliedCondOperands(FoundPred,
LHS,
RHS, FoundLHS, FoundRHS, CtxI))
12153 if (isImpliedCondOperandsViaRanges(Pred,
LHS,
RHS, FoundPred, FoundLHS, FoundRHS))
12160bool ScalarEvolution::splitBinaryAdd(
const SCEV *Expr,
12161 const SCEV *&L,
const SCEV *&R,
12164 if (!AE || AE->getNumOperands() != 2)
12167 L = AE->getOperand(0);
12168 R = AE->getOperand(1);
12169 Flags = AE->getNoWrapFlags();
12173std::optional<APInt>
12180 APInt DiffMul(BW, 1);
12183 for (
unsigned I = 0;
I < 8; ++
I) {
12192 if (LAR->getLoop() != MAR->getLoop())
12193 return std::nullopt;
12197 if (!LAR->isAffine() || !MAR->isAffine())
12198 return std::nullopt;
12200 if (LAR->getStepRecurrence(*
this) != MAR->getStepRecurrence(*
this))
12201 return std::nullopt;
12203 Less = LAR->getStart();
12204 More = MAR->getStart();
12209 auto MatchConstMul =
12210 [](
const SCEV *S) -> std::optional<std::pair<const SCEV *, APInt>> {
12212 if (!M || M->getNumOperands() != 2 ||
12214 return std::nullopt;
12218 if (
auto MatchedMore = MatchConstMul(More)) {
12219 if (
auto MatchedLess = MatchConstMul(
Less)) {
12220 if (MatchedMore->second == MatchedLess->second) {
12221 More = MatchedMore->first;
12222 Less = MatchedLess->first;
12223 DiffMul *= MatchedMore->second;
12234 Diff +=
C->getAPInt() * DiffMul;
12237 Diff -=
C->getAPInt() * DiffMul;
12240 Multiplicity[S] +=
Mul;
12242 auto Decompose = [&](
const SCEV *S,
int Mul) {
12249 Decompose(More, 1);
12250 Decompose(
Less, -1);
12254 const SCEV *NewMore =
nullptr, *NewLess =
nullptr;
12255 for (
const auto &[S,
Mul] : Multiplicity) {
12260 return std::nullopt;
12262 }
else if (
Mul == -1) {
12264 return std::nullopt;
12267 return std::nullopt;
12271 if (NewMore == More || NewLess ==
Less)
12272 return std::nullopt;
12278 if (!More && !
Less)
12282 if (!More || !
Less)
12283 return std::nullopt;
12287 return std::nullopt;
12290bool ScalarEvolution::isImpliedCondOperandsViaAddRecStart(
12314 if (!L->contains(ContextBB) || !DT.
dominates(ContextBB, L->getLoopLatch()))
12325 if (!L->contains(ContextBB) || !DT.
dominates(ContextBB, L->getLoopLatch()))
12335bool ScalarEvolution::isImpliedCondOperandsViaNoOverflow(CmpPredicate Pred,
12338 const SCEV *FoundLHS,
12339 const SCEV *FoundRHS) {
12348 if (!AddRecFoundLHS)
12355 const Loop *
L = AddRecFoundLHS->getLoop();
12356 if (L != AddRecLHS->getLoop())
12395 if (!RDiff || *LDiff != *RDiff)
12398 if (LDiff->isMinValue())
12401 APInt FoundRHSLimit;
12404 FoundRHSLimit = -(*RDiff);
12416bool ScalarEvolution::isImpliedViaMerge(CmpPredicate Pred,
const SCEV *
LHS,
12417 const SCEV *
RHS,
const SCEV *FoundLHS,
12418 const SCEV *FoundRHS,
unsigned Depth) {
12419 const PHINode *LPhi =
nullptr, *RPhi =
nullptr;
12423 bool Erased = PendingMerges.erase(LPhi);
12424 assert(Erased &&
"Failed to erase LPhi!");
12428 bool Erased = PendingMerges.erase(RPhi);
12429 assert(Erased &&
"Failed to erase RPhi!");
12437 if (!PendingMerges.insert(Phi).second)
12451 if (!PendingMerges.insert(Phi).second)
12457 if (!LPhi && !RPhi)
12468 assert(LPhi &&
"LPhi should definitely be a SCEVUnknown Phi!");
12472 auto ProvedEasily = [&](
const SCEV *
S1,
const SCEV *S2) {
12473 return isKnownViaNonRecursiveReasoning(Pred,
S1, S2) ||
12474 isImpliedCondOperandsViaRanges(Pred,
S1, S2, Pred, FoundLHS, FoundRHS) ||
12475 isImpliedViaOperations(Pred,
S1, S2, FoundLHS, FoundRHS,
Depth);
12478 if (RPhi && RPhi->getParent() == LBB) {
12485 const SCEV *
R =
getSCEV(RPhi->getIncomingValueForBlock(IncBB));
12486 if (!ProvedEasily(L, R))
12497 auto *RLoop = RAR->
getLoop();
12498 auto *Predecessor = RLoop->getLoopPredecessor();
12499 assert(Predecessor &&
"Loop with AddRec with no predecessor?");
12501 if (!ProvedEasily(L1, RAR->
getStart()))
12503 auto *Latch = RLoop->getLoopLatch();
12504 assert(Latch &&
"Loop with AddRec with no latch?");
12525 if (
auto *Loop = LI.getLoopFor(LBB))
12528 if (!ProvedEasily(L,
RHS))
12535bool ScalarEvolution::isImpliedCondOperandsViaShift(CmpPredicate Pred,
12538 const SCEV *FoundLHS,
12539 const SCEV *FoundRHS) {
12542 if (
RHS == FoundRHS) {
12547 if (
LHS != FoundLHS)
12554 Value *Shiftee, *ShiftValue;
12556 using namespace PatternMatch;
12557 if (
match(SUFoundRHS->getValue(),
12559 auto *ShifteeS =
getSCEV(Shiftee);
12577bool ScalarEvolution::isImpliedCondOperands(CmpPredicate Pred,
const SCEV *
LHS,
12579 const SCEV *FoundLHS,
12580 const SCEV *FoundRHS,
12581 const Instruction *CtxI) {
12582 return isImpliedCondOperandsViaRanges(Pred,
LHS,
RHS, Pred, FoundLHS,
12584 isImpliedCondOperandsViaNoOverflow(Pred,
LHS,
RHS, FoundLHS,
12586 isImpliedCondOperandsViaShift(Pred,
LHS,
RHS, FoundLHS, FoundRHS) ||
12587 isImpliedCondOperandsViaAddRecStart(Pred,
LHS,
RHS, FoundLHS, FoundRHS,
12589 isImpliedCondOperandsHelper(Pred,
LHS,
RHS, FoundLHS, FoundRHS);
12593template <
typename MinMaxExprType>
12595 const SCEV *Candidate) {
12600 return is_contained(MinMaxExpr->operands(), Candidate);
12613 const SCEV *LStart, *RStart, *Step;
12663bool ScalarEvolution::isImpliedViaOperations(CmpPredicate Pred,
const SCEV *
LHS,
12665 const SCEV *FoundLHS,
12666 const SCEV *FoundRHS,
12670 "LHS and RHS have different sizes?");
12673 "FoundLHS and FoundRHS have different sizes?");
12707 auto GetOpFromSExt = [&](
const SCEV *S) {
12709 return Ext->getOperand();
12716 auto *OrigLHS =
LHS;
12717 auto *OrigFoundLHS = FoundLHS;
12718 LHS = GetOpFromSExt(
LHS);
12719 FoundLHS = GetOpFromSExt(FoundLHS);
12722 auto IsSGTViaContext = [&](
const SCEV *
S1,
const SCEV *S2) {
12725 FoundRHS,
Depth + 1);
12738 if (!LHSAddExpr->hasNoSignedWrap())
12741 auto *LL = LHSAddExpr->getOperand(0);
12742 auto *LR = LHSAddExpr->getOperand(1);
12746 auto IsSumGreaterThanRHS = [&](
const SCEV *
S1,
const SCEV *S2) {
12747 return IsSGTViaContext(
S1, MinusOne) && IsSGTViaContext(S2,
RHS);
12752 if (IsSumGreaterThanRHS(LL, LR) || IsSumGreaterThanRHS(LR, LL))
12758 using namespace llvm::PatternMatch;
12777 if (!Numerator || Numerator->getType() != FoundLHS->
getType())
12785 auto *DTy = Denominator->getType();
12786 auto *FRHSTy = FoundRHS->
getType();
12787 if (DTy->isPointerTy() != FRHSTy->isPointerTy())
12806 IsSGTViaContext(FoundRHSExt, DenomMinusTwo))
12817 auto *NegDenomMinusOne =
getMinusSCEV(MinusOne, DenominatorExt);
12819 IsSGTViaContext(FoundRHSExt, NegDenomMinusOne))
12827 if (isImpliedViaMerge(Pred, OrigLHS,
RHS, OrigFoundLHS, FoundRHS,
Depth + 1))
12860bool ScalarEvolution::isKnownViaNonRecursiveReasoning(CmpPredicate Pred,
12864 isKnownPredicateViaConstantRanges(Pred,
LHS,
RHS) ||
12867 isKnownPredicateViaNoOverflow(Pred,
LHS,
RHS);
12870bool ScalarEvolution::isImpliedCondOperandsHelper(CmpPredicate Pred,
12873 const SCEV *FoundLHS,
12874 const SCEV *FoundRHS) {
12910 if (isImpliedViaOperations(Pred,
LHS,
RHS, FoundLHS, FoundRHS))
12916bool ScalarEvolution::isImpliedCondOperandsViaRanges(
12917 CmpPredicate Pred,
const SCEV *
LHS,
const SCEV *
RHS, CmpPredicate FoundPred,
12918 const SCEV *FoundLHS,
const SCEV *FoundRHS) {
12932 ConstantRange FoundLHSRange =
12936 ConstantRange LHSRange = FoundLHSRange.
add(ConstantRange(*Addend));
12943 return LHSRange.
icmp(Pred, ConstRHS);
12946bool ScalarEvolution::canIVOverflowOnLT(
const SCEV *
RHS,
const SCEV *Stride,
12959 return (std::move(MaxValue) - MaxStrideMinusOne).slt(MaxRHS);
12967 return (std::move(MaxValue) - MaxStrideMinusOne).ult(MaxRHS);
12970bool ScalarEvolution::canIVOverflowOnGT(
const SCEV *
RHS,
const SCEV *Stride,
12982 return (std::move(MinValue) + MaxStrideMinusOne).sgt(MinRHS);
12990 return (std::move(MinValue) + MaxStrideMinusOne).ugt(MinRHS);
13002const SCEV *ScalarEvolution::computeMaxBECountForLT(
const SCEV *Start,
13003 const SCEV *Stride,
13034 APInt Limit = MaxValue - (StrideForMaxBECount - 1);
13045 :
APIntOps::umax(MaxEnd, MinStart);
13052ScalarEvolution::howManyLessThans(
const SCEV *
LHS,
const SCEV *
RHS,
13053 const Loop *L,
bool IsSigned,
13054 bool ControlsOnlyExit,
bool AllowPredicates) {
13058 bool PredicatedIV =
false;
13063 auto canProveNUW = [&]() {
13066 if (!ControlsOnlyExit)
13087 Limit = Limit.
zext(OuterBitWidth);
13099 Type *Ty = ZExt->getType();
13110 if (!
IV && AllowPredicates) {
13115 PredicatedIV =
true;
13119 if (!
IV ||
IV->getLoop() != L || !
IV->isAffine())
13133 bool NoWrap = ControlsOnlyExit &&
IV->getNoWrapFlags(WrapType);
13136 const SCEV *Stride =
IV->getStepRecurrence(*
this);
13141 if (!PositiveStride) {
13193 auto wouldZeroStrideBeUB = [&]() {
13205 if (!wouldZeroStrideBeUB()) {
13209 }
else if (!NoWrap) {
13212 if (canIVOverflowOnLT(
RHS, Stride, IsSigned))
13225 const SCEV *
Start =
IV->getStart();
13231 const SCEV *OrigStart =
Start;
13232 const SCEV *OrigRHS =
RHS;
13233 if (
Start->getType()->isPointerTy()) {
13244 const SCEV *End =
nullptr, *BECount =
nullptr,
13245 *BECountIfBackedgeTaken =
nullptr;
13248 if (PositiveStride && RHSAddRec !=
nullptr && RHSAddRec->getLoop() == L &&
13249 RHSAddRec->getNoWrapFlags()) {
13262 const SCEV *RHSStart = RHSAddRec->getStart();
13263 const SCEV *RHSStride = RHSAddRec->getStepRecurrence(*
this);
13275 const SCEV *Denominator =
getMinusSCEV(Stride, RHSStride);
13284 BECountIfBackedgeTaken =
13289 if (BECount ==
nullptr) {
13294 const SCEV *MaxBECount = computeMaxBECountForLT(
13297 MaxBECount,
false , Predicates);
13304 auto *OrigStartMinusStride =
getMinusSCEV(OrigStart, Stride);
13331 const SCEV *Numerator =
13337 auto canProveRHSGreaterThanEqualStart = [&]() {
13356 auto *StartMinusOne =
13363 if (canProveRHSGreaterThanEqualStart()) {
13378 BECountIfBackedgeTaken =
13394 bool MayAddOverflow = [&] {
13440 if (Start == Stride || Start ==
getMinusSCEV(Stride, One)) {
13454 if (!MayAddOverflow) {
13466 const SCEV *ConstantMaxBECount;
13467 bool MaxOrZero =
false;
13469 ConstantMaxBECount = BECount;
13470 }
else if (BECountIfBackedgeTaken &&
13475 ConstantMaxBECount = BECountIfBackedgeTaken;
13478 ConstantMaxBECount = computeMaxBECountForLT(
13486 const SCEV *SymbolicMaxBECount =
13488 return ExitLimit(BECount, ConstantMaxBECount, SymbolicMaxBECount, MaxOrZero,
13492ScalarEvolution::ExitLimit ScalarEvolution::howManyGreaterThans(
13493 const SCEV *
LHS,
const SCEV *
RHS,
const Loop *L,
bool IsSigned,
13494 bool ControlsOnlyExit,
bool AllowPredicates) {
13501 if (!
IV && AllowPredicates)
13508 if (!
IV ||
IV->getLoop() != L || !
IV->isAffine())
13512 bool NoWrap = ControlsOnlyExit &&
IV->getNoWrapFlags(WrapType);
13525 if (!Stride->
isOne() && !NoWrap)
13526 if (canIVOverflowOnGT(
RHS, Stride, IsSigned))
13529 const SCEV *
Start =
IV->getStart();
13530 const SCEV *End =
RHS;
13541 if (
Start->getType()->isPointerTy()) {
13576 const SCEV *ConstantMaxBECount =
13583 ConstantMaxBECount = BECount;
13584 const SCEV *SymbolicMaxBECount =
13587 return ExitLimit(BECount, ConstantMaxBECount, SymbolicMaxBECount,
false,
13593 if (
Range.isFullSet())
13598 if (!SC->getValue()->isZero()) {
13604 return ShiftedAddRec->getNumIterationsInRange(
13605 Range.subtract(SC->getAPInt()), SE);
13636 APInt ExitVal = (End +
A).udiv(
A);
13649 ConstantInt::get(SE.
getContext(), ExitVal - 1), SE)->getValue()) &&
13650 "Linear scev computation is off in a bad way!");
13681 assert(!
Last->isZero() &&
"Recurrency with zero step?");
13709 Ty = Store->getValueOperand()->getType();
13711 Ty = Load->getType();
13724 assert(SE &&
"SCEVCallbackVH called with a null ScalarEvolution!");
13726 SE->ConstantEvolutionLoopExitValue.erase(PN);
13727 SE->eraseValueFromMap(getValPtr());
13731void ScalarEvolution::SCEVCallbackVH::allUsesReplacedWith(
Value *V) {
13732 assert(SE &&
"SCEVCallbackVH called with a null ScalarEvolution!");
13742 : CallbackVH(
V), SE(se) {}
13751 : F(F), DL(F.
getDataLayout()), TLI(TLI), AC(AC), DT(DT), LI(LI),
13753 LoopDispositions(64), BlockDispositions(64) {
13765 F.getParent(), Intrinsic::experimental_guard);
13766 HasGuards = GuardDecl && !GuardDecl->use_empty();
13770 : F(Arg.F), DL(Arg.DL), HasGuards(Arg.HasGuards), TLI(Arg.TLI), AC(Arg.AC),
13771 DT(Arg.DT), LI(Arg.LI), CouldNotCompute(
std::
move(Arg.CouldNotCompute)),
13772 ValueExprMap(
std::
move(Arg.ValueExprMap)),
13773 PendingLoopPredicates(
std::
move(Arg.PendingLoopPredicates)),
13774 PendingPhiRanges(
std::
move(Arg.PendingPhiRanges)),
13775 PendingMerges(
std::
move(Arg.PendingMerges)),
13776 ConstantMultipleCache(
std::
move(Arg.ConstantMultipleCache)),
13777 BackedgeTakenCounts(
std::
move(Arg.BackedgeTakenCounts)),
13778 PredicatedBackedgeTakenCounts(
13779 std::
move(Arg.PredicatedBackedgeTakenCounts)),
13780 BECountUsers(
std::
move(Arg.BECountUsers)),
13781 ConstantEvolutionLoopExitValue(
13782 std::
move(Arg.ConstantEvolutionLoopExitValue)),
13783 ValuesAtScopes(
std::
move(Arg.ValuesAtScopes)),
13784 ValuesAtScopesUsers(
std::
move(Arg.ValuesAtScopesUsers)),
13785 LoopDispositions(
std::
move(Arg.LoopDispositions)),
13786 LoopPropertiesCache(
std::
move(Arg.LoopPropertiesCache)),
13787 BlockDispositions(
std::
move(Arg.BlockDispositions)),
13788 SCEVUsers(
std::
move(Arg.SCEVUsers)),
13789 UnsignedRanges(
std::
move(Arg.UnsignedRanges)),
13790 SignedRanges(
std::
move(Arg.SignedRanges)),
13791 UniqueSCEVs(
std::
move(Arg.UniqueSCEVs)),
13792 UniquePreds(
std::
move(Arg.UniquePreds)),
13793 SCEVAllocator(
std::
move(Arg.SCEVAllocator)),
13794 LoopUsers(
std::
move(Arg.LoopUsers)),
13795 PredicatedSCEVRewrites(
std::
move(Arg.PredicatedSCEVRewrites)),
13796 FirstUnknown(Arg.FirstUnknown) {
13797 Arg.FirstUnknown =
nullptr;
13806 Tmp->~SCEVUnknown();
13808 FirstUnknown =
nullptr;
13810 ExprValueMap.clear();
13811 ValueExprMap.clear();
13813 BackedgeTakenCounts.clear();
13814 PredicatedBackedgeTakenCounts.clear();
13816 assert(PendingLoopPredicates.empty() &&
"isImpliedCond garbage");
13817 assert(PendingPhiRanges.empty() &&
"getRangeRef garbage");
13818 assert(PendingMerges.empty() &&
"isImpliedViaMerge garbage");
13819 assert(!WalkingBEDominatingConds &&
"isLoopBackedgeGuardedByCond garbage!");
13820 assert(!ProvingSplitPredicate &&
"ProvingSplitPredicate garbage!");
13842 L->getHeader()->printAsOperand(OS,
false);
13846 L->getExitingBlocks(ExitingBlocks);
13847 if (ExitingBlocks.
size() != 1)
13848 OS <<
"<multiple exits> ";
13852 OS <<
"backedge-taken count is ";
13855 OS <<
"Unpredictable backedge-taken count.";
13858 if (ExitingBlocks.
size() > 1)
13859 for (
BasicBlock *ExitingBlock : ExitingBlocks) {
13860 OS <<
" exit count for " << ExitingBlock->
getName() <<
": ";
13868 OS <<
"\n predicated exit count for " << ExitingBlock->
getName()
13871 OS <<
"\n Predicates:\n";
13872 for (
const auto *
P : Predicates)
13880 L->getHeader()->printAsOperand(OS,
false);
13885 OS <<
"constant max backedge-taken count is ";
13888 OS <<
", actual taken count either this or zero.";
13890 OS <<
"Unpredictable constant max backedge-taken count. ";
13895 L->getHeader()->printAsOperand(OS,
false);
13900 OS <<
"symbolic max backedge-taken count is ";
13903 OS <<
", actual taken count either this or zero.";
13905 OS <<
"Unpredictable symbolic max backedge-taken count. ";
13909 if (ExitingBlocks.
size() > 1)
13910 for (
BasicBlock *ExitingBlock : ExitingBlocks) {
13911 OS <<
" symbolic max exit count for " << ExitingBlock->
getName() <<
": ";
13921 OS <<
"\n predicated symbolic max exit count for "
13922 << ExitingBlock->
getName() <<
": ";
13924 OS <<
"\n Predicates:\n";
13925 for (
const auto *
P : Predicates)
13935 assert(!Preds.
empty() &&
"Different predicated BTC, but no predicates");
13937 L->getHeader()->printAsOperand(OS,
false);
13940 OS <<
"Predicated backedge-taken count is ";
13943 OS <<
"Unpredictable predicated backedge-taken count.";
13945 OS <<
" Predicates:\n";
13946 for (
const auto *
P : Preds)
13951 auto *PredConstantMax =
13953 if (PredConstantMax != ConstantBTC) {
13955 "different predicated constant max BTC but no predicates");
13957 L->getHeader()->printAsOperand(OS,
false);
13960 OS <<
"Predicated constant max backedge-taken count is ";
13963 OS <<
"Unpredictable predicated constant max backedge-taken count.";
13965 OS <<
" Predicates:\n";
13966 for (
const auto *
P : Preds)
13971 auto *PredSymbolicMax =
13973 if (SymbolicBTC != PredSymbolicMax) {
13975 "Different predicated symbolic max BTC, but no predicates");
13977 L->getHeader()->printAsOperand(OS,
false);
13980 OS <<
"Predicated symbolic max backedge-taken count is ";
13983 OS <<
"Unpredictable predicated symbolic max backedge-taken count.";
13985 OS <<
" Predicates:\n";
13986 for (
const auto *
P : Preds)
13992 L->getHeader()->printAsOperand(OS,
false);
14008 OS <<
"Computable";
14017 OS <<
"DoesNotDominate";
14023 OS <<
"ProperlyDominates";
14040 OS <<
"Classifying expressions for: ";
14041 F.printAsOperand(OS,
false);
14056 const Loop *L = LI.getLoopFor(
I.getParent());
14071 OS <<
"\t\t" "Exits: ";
14074 OS <<
"<<Unknown>>";
14080 for (
const auto *Iter = L; Iter; Iter = Iter->getParentLoop()) {
14082 OS <<
"\t\t" "LoopDispositions: { ";
14088 Iter->getHeader()->printAsOperand(OS,
false);
14096 OS <<
"\t\t" "LoopDispositions: { ";
14102 InnerL->getHeader()->printAsOperand(OS,
false);
14113 OS <<
"Determining loop execution counts for: ";
14114 F.printAsOperand(OS,
false);
14122 auto &Values = LoopDispositions[S];
14123 for (
auto &V : Values) {
14124 if (V.getPointer() == L)
14129 auto &Values2 = LoopDispositions[S];
14131 if (V.getPointer() == L) {
14140ScalarEvolution::computeLoopDisposition(
const SCEV *S,
const Loop *L) {
14159 assert(!L->contains(AR->
getLoop()) &&
"Containing loop's header does not"
14160 " dominate the contained loop's header?");
14187 bool HasVarying =
false;
14221 auto &Values = BlockDispositions[S];
14222 for (
auto &V : Values) {
14223 if (V.getPointer() == BB)
14228 auto &Values2 = BlockDispositions[S];
14230 if (V.getPointer() == BB) {
14239ScalarEvolution::computeBlockDisposition(
const SCEV *S,
const BasicBlock *BB) {
14268 bool Proper =
true;
14279 if (Instruction *
I =
14281 if (
I->getParent() == BB)
14283 if (DT.properlyDominates(
I->getParent(), BB))
14306void ScalarEvolution::forgetBackedgeTakenCounts(
const Loop *L,
14309 Predicated ? PredicatedBackedgeTakenCounts : BackedgeTakenCounts;
14310 auto It = BECounts.find(L);
14311 if (It != BECounts.end()) {
14312 for (
const ExitNotTakenInfo &ENT : It->second.ExitNotTaken) {
14313 for (
const SCEV *S : {ENT.ExactNotTaken, ENT.SymbolicMaxNotTaken}) {
14315 auto UserIt = BECountUsers.find(S);
14316 assert(UserIt != BECountUsers.end());
14321 BECounts.erase(It);
14329 while (!Worklist.
empty()) {
14331 auto Users = SCEVUsers.find(Curr);
14332 if (
Users != SCEVUsers.end())
14333 for (
const auto *User :
Users->second)
14334 if (ToForget.
insert(User).second)
14338 for (
const auto *S : ToForget)
14339 forgetMemoizedResultsImpl(S);
14341 for (
auto I = PredicatedSCEVRewrites.begin();
14342 I != PredicatedSCEVRewrites.end();) {
14343 std::pair<const SCEV *, const Loop *>
Entry =
I->first;
14344 if (ToForget.count(
Entry.first))
14345 PredicatedSCEVRewrites.erase(
I++);
14351void ScalarEvolution::forgetMemoizedResultsImpl(
const SCEV *S) {
14352 LoopDispositions.erase(S);
14353 BlockDispositions.erase(S);
14354 UnsignedRanges.erase(S);
14355 SignedRanges.erase(S);
14356 HasRecMap.erase(S);
14357 ConstantMultipleCache.erase(S);
14360 UnsignedWrapViaInductionTried.erase(AR);
14361 SignedWrapViaInductionTried.erase(AR);
14364 auto ExprIt = ExprValueMap.find(S);
14365 if (ExprIt != ExprValueMap.end()) {
14366 for (
Value *V : ExprIt->second) {
14367 auto ValueIt = ValueExprMap.find_as(V);
14368 if (ValueIt != ValueExprMap.end())
14369 ValueExprMap.erase(ValueIt);
14371 ExprValueMap.erase(ExprIt);
14374 auto ScopeIt = ValuesAtScopes.find(S);
14375 if (ScopeIt != ValuesAtScopes.end()) {
14376 for (
const auto &Pair : ScopeIt->second)
14379 std::make_pair(Pair.first, S));
14380 ValuesAtScopes.erase(ScopeIt);
14383 auto ScopeUserIt = ValuesAtScopesUsers.find(S);
14384 if (ScopeUserIt != ValuesAtScopesUsers.end()) {
14385 for (
const auto &Pair : ScopeUserIt->second)
14386 llvm::erase(ValuesAtScopes[Pair.second], std::make_pair(Pair.first, S));
14387 ValuesAtScopesUsers.erase(ScopeUserIt);
14390 auto BEUsersIt = BECountUsers.find(S);
14391 if (BEUsersIt != BECountUsers.end()) {
14393 auto Copy = BEUsersIt->second;
14394 for (
const auto &Pair : Copy)
14395 forgetBackedgeTakenCounts(Pair.getPointer(), Pair.getInt());
14396 BECountUsers.erase(BEUsersIt);
14399 auto FoldUser = FoldCacheUser.find(S);
14400 if (FoldUser != FoldCacheUser.end())
14401 for (
auto &KV : FoldUser->second)
14402 FoldCache.erase(KV);
14403 FoldCacheUser.erase(S);
14407ScalarEvolution::getUsedLoops(
const SCEV *S,
14409 struct FindUsedLoops {
14410 FindUsedLoops(SmallPtrSetImpl<const Loop *> &LoopsUsed)
14411 : LoopsUsed(LoopsUsed) {}
14412 SmallPtrSetImpl<const Loop *> &LoopsUsed;
14413 bool follow(
const SCEV *S) {
14419 bool isDone()
const {
return false; }
14422 FindUsedLoops
F(LoopsUsed);
14423 SCEVTraversal<FindUsedLoops>(F).visitAll(S);
14426void ScalarEvolution::getReachableBlocks(
14429 Worklist.
push_back(&F.getEntryBlock());
14430 while (!Worklist.
empty()) {
14432 if (!Reachable.
insert(BB).second)
14440 Worklist.
push_back(
C->isOne() ? TrueBB : FalseBB);
14447 if (isKnownPredicateViaConstantRanges(
Cmp->getCmpPredicate(), L, R)) {
14451 if (isKnownPredicateViaConstantRanges(
Cmp->getInverseCmpPredicate(), L,
14486 SCEVMapper SCM(SE2);
14488 SE2.getReachableBlocks(ReachableBlocks, F);
14490 auto GetDelta = [&](
const SCEV *Old,
const SCEV *New) ->
const SCEV * {
14508 while (!LoopStack.
empty()) {
14514 if (!ReachableBlocks.
contains(L->getHeader()))
14519 auto It = BackedgeTakenCounts.find(L);
14520 if (It == BackedgeTakenCounts.end())
14524 SCM.visit(It->second.getExact(L,
const_cast<ScalarEvolution *
>(
this)));
14544 const SCEV *Delta = GetDelta(CurBECount, NewBECount);
14545 if (Delta && !Delta->
isZero()) {
14546 dbgs() <<
"Trip Count for " << *L <<
" Changed!\n";
14547 dbgs() <<
"Old: " << *CurBECount <<
"\n";
14548 dbgs() <<
"New: " << *NewBECount <<
"\n";
14549 dbgs() <<
"Delta: " << *Delta <<
"\n";
14557 while (!Worklist.
empty()) {
14559 if (ValidLoops.
insert(L).second)
14560 Worklist.
append(L->begin(), L->end());
14562 for (
const auto &KV : ValueExprMap) {
14567 "AddRec references invalid loop");
14572 auto It = ExprValueMap.find(KV.second);
14573 if (It == ExprValueMap.end() || !It->second.contains(KV.first)) {
14574 dbgs() <<
"Value " << *KV.first
14575 <<
" is in ValueExprMap but not in ExprValueMap\n";
14580 if (!ReachableBlocks.
contains(
I->getParent()))
14582 const SCEV *OldSCEV = SCM.visit(KV.second);
14584 const SCEV *Delta = GetDelta(OldSCEV, NewSCEV);
14585 if (Delta && !Delta->
isZero()) {
14586 dbgs() <<
"SCEV for value " << *
I <<
" changed!\n"
14587 <<
"Old: " << *OldSCEV <<
"\n"
14588 <<
"New: " << *NewSCEV <<
"\n"
14589 <<
"Delta: " << *Delta <<
"\n";
14595 for (
const auto &KV : ExprValueMap) {
14596 for (
Value *V : KV.second) {
14597 const SCEV *S = ValueExprMap.lookup(V);
14599 dbgs() <<
"Value " << *V
14600 <<
" is in ExprValueMap but not in ValueExprMap\n";
14603 if (S != KV.first) {
14604 dbgs() <<
"Value " << *V <<
" mapped to " << *S <<
" rather than "
14605 << *KV.first <<
"\n";
14612 for (
const auto &S : UniqueSCEVs) {
14617 auto It = SCEVUsers.find(
Op);
14618 if (It != SCEVUsers.end() && It->second.count(&S))
14620 dbgs() <<
"Use of operand " << *
Op <<
" by user " << S
14621 <<
" is not being tracked!\n";
14627 for (
const auto &ValueAndVec : ValuesAtScopes) {
14629 for (
const auto &LoopAndValueAtScope : ValueAndVec.second) {
14630 const Loop *L = LoopAndValueAtScope.first;
14631 const SCEV *ValueAtScope = LoopAndValueAtScope.second;
14633 auto It = ValuesAtScopesUsers.find(ValueAtScope);
14634 if (It != ValuesAtScopesUsers.end() &&
14637 dbgs() <<
"Value: " << *
Value <<
", Loop: " << *L <<
", ValueAtScope: "
14638 << *ValueAtScope <<
" missing in ValuesAtScopesUsers\n";
14644 for (
const auto &ValueAtScopeAndVec : ValuesAtScopesUsers) {
14645 const SCEV *ValueAtScope = ValueAtScopeAndVec.first;
14646 for (
const auto &LoopAndValue : ValueAtScopeAndVec.second) {
14647 const Loop *L = LoopAndValue.first;
14648 const SCEV *
Value = LoopAndValue.second;
14650 auto It = ValuesAtScopes.find(
Value);
14651 if (It != ValuesAtScopes.end() &&
14652 is_contained(It->second, std::make_pair(L, ValueAtScope)))
14654 dbgs() <<
"Value: " << *
Value <<
", Loop: " << *L <<
", ValueAtScope: "
14655 << *ValueAtScope <<
" missing in ValuesAtScopes\n";
14661 auto VerifyBECountUsers = [&](
bool Predicated) {
14663 Predicated ? PredicatedBackedgeTakenCounts : BackedgeTakenCounts;
14664 for (
const auto &LoopAndBEInfo : BECounts) {
14665 for (
const ExitNotTakenInfo &ENT : LoopAndBEInfo.second.ExitNotTaken) {
14666 for (
const SCEV *S : {ENT.ExactNotTaken, ENT.SymbolicMaxNotTaken}) {
14668 auto UserIt = BECountUsers.find(S);
14669 if (UserIt != BECountUsers.end() &&
14670 UserIt->second.contains({ LoopAndBEInfo.first, Predicated }))
14672 dbgs() <<
"Value " << *S <<
" for loop " << *LoopAndBEInfo.first
14673 <<
" missing from BECountUsers\n";
14680 VerifyBECountUsers(
false);
14681 VerifyBECountUsers(
true);
14684 for (
auto &[S, Values] : LoopDispositions) {
14685 for (
auto [
Loop, CachedDisposition] : Values) {
14687 if (CachedDisposition != RecomputedDisposition) {
14688 dbgs() <<
"Cached disposition of " << *S <<
" for loop " << *
Loop
14689 <<
" is incorrect: cached " << CachedDisposition <<
", actual "
14690 << RecomputedDisposition <<
"\n";
14697 for (
auto &[S, Values] : BlockDispositions) {
14698 for (
auto [BB, CachedDisposition] : Values) {
14700 if (CachedDisposition != RecomputedDisposition) {
14701 dbgs() <<
"Cached disposition of " << *S <<
" for block %"
14702 << BB->
getName() <<
" is incorrect: cached " << CachedDisposition
14703 <<
", actual " << RecomputedDisposition <<
"\n";
14710 for (
auto [
FoldID, Expr] : FoldCache) {
14711 auto I = FoldCacheUser.find(Expr);
14712 if (
I == FoldCacheUser.end()) {
14713 dbgs() <<
"Missing entry in FoldCacheUser for cached expression " << *Expr
14718 dbgs() <<
"Missing FoldID in cached users of " << *Expr <<
"!\n";
14722 for (
auto [Expr, IDs] : FoldCacheUser) {
14723 for (
auto &
FoldID : IDs) {
14726 dbgs() <<
"Missing entry in FoldCache for expression " << *Expr
14731 dbgs() <<
"Entry in FoldCache doesn't match FoldCacheUser: " << *S
14732 <<
" != " << *Expr <<
"!\n";
14743 for (
auto [S, Multiple] : ConstantMultipleCache) {
14745 if ((Multiple != 0 && RecomputedMultiple != 0 &&
14746 Multiple.
urem(RecomputedMultiple) != 0 &&
14747 RecomputedMultiple.
urem(Multiple) != 0)) {
14748 dbgs() <<
"Incorrect cached computation in ConstantMultipleCache for "
14749 << *S <<
" : Computed " << RecomputedMultiple
14750 <<
" but cache contains " << Multiple <<
"!\n";
14758 FunctionAnalysisManager::Invalidator &Inv) {
14790 OS <<
"Printing analysis 'Scalar Evolution Analysis' for function '"
14791 <<
F.getName() <<
"':\n";
14797 "Scalar Evolution Analysis",
false,
true)
14846 const SCEV *LHS,
const SCEV *RHS) {
14848 assert(LHS->getType() == RHS->getType() &&
14849 "Type mismatch between LHS and RHS");
14852 ID.AddInteger(Pred);
14853 ID.AddPointer(LHS);
14854 ID.AddPointer(RHS);
14855 void *IP =
nullptr;
14856 if (
const auto *S = UniquePreds.FindNodeOrInsertPos(
ID, IP))
14860 UniquePreds.InsertNode(Eq, IP);
14871 ID.AddInteger(AddedFlags);
14872 void *IP =
nullptr;
14873 if (
const auto *S = UniquePreds.FindNodeOrInsertPos(
ID, IP))
14875 auto *OF =
new (SCEVAllocator)
14877 UniquePreds.InsertNode(OF, IP);
14897 SCEVPredicateRewriter
Rewriter(L, SE, NewPreds, Pred);
14898 return Rewriter.visit(S);
14904 for (
const auto *Pred : U->getPredicates())
14906 if (IPred->getLHS() == Expr &&
14908 return IPred->getRHS();
14910 if (IPred->getLHS() == Expr &&
14911 IPred->getPredicate() == ICmpInst::ICMP_EQ)
14912 return IPred->getRHS();
14915 return convertToAddRecWithPreds(Expr);
14918 const SCEV *visitZeroExtendExpr(
const SCEVZeroExtendExpr *Expr) {
14934 const SCEV *visitSignExtendExpr(
const SCEVSignExtendExpr *Expr) {
14951 explicit SCEVPredicateRewriter(
14952 const Loop *L, ScalarEvolution &SE,
14953 SmallVectorImpl<const SCEVPredicate *> *NewPreds,
14954 const SCEVPredicate *Pred)
14955 : SCEVRewriteVisitor(SE), NewPreds(NewPreds), Pred(Pred),
L(
L) {}
14957 bool addOverflowAssumption(
const SCEVPredicate *
P) {
14960 return Pred && Pred->
implies(
P, SE);
14966 bool addOverflowAssumption(
const SCEVAddRecExpr *AR,
14969 return addOverflowAssumption(
A);
14978 const SCEV *convertToAddRecWithPreds(
const SCEVUnknown *Expr) {
14982 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
14984 if (!PredicatedRewrite)
14986 for (
const auto *
P : PredicatedRewrite->second){
14989 if (L != WP->getExpr()->getLoop())
14992 if (!addOverflowAssumption(
P))
14995 return PredicatedRewrite->first;
14998 SmallVectorImpl<const SCEVPredicate *> *NewPreds;
14999 const SCEVPredicate *Pred;
15008 return SCEVPredicateRewriter::rewrite(S, L, *
this,
nullptr, &Preds);
15015 S = SCEVPredicateRewriter::rewrite(S, L, *
this, &TransformPreds,
nullptr);
15035 if (!Step->
isOne())
15060 assert(LHS->getType() == RHS->getType() &&
"LHS and RHS types don't match");
15061 assert(LHS != RHS &&
"LHS and RHS are the same SCEV");
15074 return Op->LHS == LHS &&
Op->RHS == RHS;
15081 OS.
indent(
Depth) <<
"Equal predicate: " << *LHS <<
" == " << *RHS <<
"\n";
15083 OS.
indent(
Depth) <<
"Compare predicate: " << *LHS <<
" " << Pred <<
") "
15108 const SCEV *Start = AR->getStart();
15109 const SCEV *OpStart =
Op->AR->getStart();
15114 if (Start->getType()->isPointerTy() && Start->getType() != OpStart->
getType())
15117 const SCEV *Step = AR->getStepRecurrence(SE);
15118 const SCEV *OpStep =
Op->AR->getStepRecurrence(SE);
15171 if (Step->getValue()->getValue().isNonNegative())
15175 return ImpliedFlags;
15182 for (
const auto *
P : Preds)
15195 return this->implies(I, SE);
15203 for (
const auto *Pred : Preds)
15204 Pred->print(OS,
Depth);
15209 for (
const auto *Pred : Set->Preds)
15217 bool CheckImplies = Preds.
size() < 16;
15220 if (CheckImplies &&
implies(
N, SE))
15226 for (
auto *
P : Preds) {
15227 if (CheckImplies &&
N->implies(
P, SE))
15231 Preds = std::move(PrunedPreds);
15232 Preds.push_back(
N);
15239 Preds = std::make_unique<SCEVUnionPredicate>(
Empty, SE);
15244 for (
const auto *
Op :
Ops)
15249 SCEVUsers[
Op].insert(
User);
15253 const SCEV *Expr = SE.getSCEV(V);
15254 RewriteEntry &Entry = RewriteMap[Expr];
15257 if (Entry.second && Generation == Entry.first)
15258 return Entry.second;
15263 Expr = Entry.second;
15265 const SCEV *NewSCEV = SE.rewriteUsingPredicate(Expr, &L, *Preds);
15266 Entry = {Generation, NewSCEV};
15272 if (!BackedgeCount) {
15274 BackedgeCount = SE.getPredicatedBackedgeTakenCount(&L, Preds);
15275 for (
const auto *
P : Preds)
15278 return BackedgeCount;
15282 if (!SymbolicMaxBackedgeCount) {
15284 SymbolicMaxBackedgeCount =
15285 SE.getPredicatedSymbolicMaxBackedgeTakenCount(&L, Preds);
15286 for (
const auto *
P : Preds)
15289 return SymbolicMaxBackedgeCount;
15293 if (!SmallConstantMaxTripCount) {
15295 SmallConstantMaxTripCount = SE.getSmallConstantMaxTripCount(&L, &Preds);
15296 for (
const auto *
P : Preds)
15299 return *SmallConstantMaxTripCount;
15303 if (Preds->implies(&Pred, SE))
15308 Preds = std::make_unique<SCEVUnionPredicate>(NewPreds, SE);
15309 updateGeneration();
15316void PredicatedScalarEvolution::updateGeneration() {
15318 if (++Generation == 0) {
15319 for (
auto &
II : RewriteMap) {
15320 const SCEV *Rewritten =
II.second.second;
15337 auto II = FlagsMap.insert({V, Flags});
15350 auto II = FlagsMap.find(V);
15352 if (
II != FlagsMap.end())
15361 auto *New = SE.convertSCEVToAddRecWithPredicates(Expr, &L, NewPreds);
15366 for (
const auto *
P : NewPreds)
15369 RewriteMap[SE.getSCEV(V)] = {Generation, New};
15375 : RewriteMap(
Init.RewriteMap), SE(
Init.SE), L(
Init.L),
15378 Generation(
Init.Generation), BackedgeCount(
Init.BackedgeCount) {
15379 for (
auto I :
Init.FlagsMap)
15380 FlagsMap.insert(
I);
15385 for (
auto *BB : L.getBlocks())
15386 for (
auto &
I : *BB) {
15387 if (!SE.isSCEVable(
I.getType()))
15390 auto *Expr = SE.getSCEV(&
I);
15391 auto II = RewriteMap.find(Expr);
15393 if (
II == RewriteMap.end())
15397 if (
II->second.second == Expr)
15402 OS.
indent(
Depth + 2) <<
"--> " << *
II->second.second <<
"\n";
15411bool ScalarEvolution::matchURem(
const SCEV *Expr,
const SCEV *&LHS,
15412 const SCEV *&RHS) {
15421 LHS = Trunc->getOperand();
15427 if (LHS->getType() != Expr->
getType())
15434 if (
Add ==
nullptr ||
Add->getNumOperands() != 2)
15437 const SCEV *
A =
Add->getOperand(1);
15440 if (
Mul ==
nullptr)
15443 const auto MatchURemWithDivisor = [&](
const SCEV *
B) {
15455 return MatchURemWithDivisor(
Mul->getOperand(1)) ||
15456 MatchURemWithDivisor(
Mul->getOperand(2));
15459 if (
Mul->getNumOperands() == 2)
15460 return MatchURemWithDivisor(
Mul->getOperand(1)) ||
15461 MatchURemWithDivisor(
Mul->getOperand(0)) ||
15471 LoopGuards Guards(SE);
15475 collectFromBlock(SE, Guards, Header, Pred, VisitedBlocks);
15479void ScalarEvolution::LoopGuards::collectFromPHI(
15487 using MinMaxPattern = std::pair<const SCEVConstant *, SCEVTypes>;
15488 auto GetMinMaxConst = [&](
unsigned IncomingIdx) -> MinMaxPattern {
15500 collectFromBlock(SE,
G->second, Phi.getParent(),
InBlock, VisitedBlocks,
15502 auto &RewriteMap =
G->second.RewriteMap;
15503 if (RewriteMap.empty())
15505 auto S = RewriteMap.find(SE.
getSCEV(
Phi.getIncomingValue(IncomingIdx)));
15506 if (S == RewriteMap.end())
15512 return {C0, SM->getSCEVType()};
15515 auto MergeMinMaxConst = [](MinMaxPattern
P1,
15516 MinMaxPattern P2) -> MinMaxPattern {
15517 auto [C1,
T1] =
P1;
15518 auto [C2, T2] = P2;
15519 if (!C1 || !C2 ||
T1 != T2)
15523 return {C1->getAPInt().
ult(C2->getAPInt()) ? C1 : C2,
T1};
15525 return {C1->getAPInt().
slt(C2->getAPInt()) ? C1 : C2,
T1};
15527 return {C1->getAPInt().
ugt(C2->getAPInt()) ? C1 : C2,
T1};
15529 return {C1->getAPInt().
sgt(C2->getAPInt()) ? C1 : C2,
T1};
15534 auto P = GetMinMaxConst(0);
15535 for (
unsigned int In = 1;
In <
Phi.getNumIncomingValues();
In++) {
15538 P = MergeMinMaxConst(
P, GetMinMaxConst(In));
15541 const SCEV *
LHS = SE.
getSCEV(
const_cast<PHINode *
>(&Phi));
15544 Guards.RewriteMap.insert({
LHS,
RHS});
15548void ScalarEvolution::LoopGuards::collectFromBlock(
15550 const BasicBlock *
Block,
const BasicBlock *Pred,
15551 SmallPtrSetImpl<const BasicBlock *> &VisitedBlocks,
unsigned Depth) {
15558 DenseMap<const SCEV *, const SCEV *>
15575 &ExprsToRewrite]() {
15576 const SCEVConstant *C1;
15589 if (ExactRegion.isWrappedSet() || ExactRegion.isFullSet())
15591 auto [
I,
Inserted] = RewriteMap.try_emplace(LHSUnknown);
15592 const SCEV *RewrittenLHS =
Inserted ? LHSUnknown :
I->second;
15600 if (MatchRangeCheckIdiom())
15606 auto IsMinMaxSCEVWithNonNegativeConstant =
15607 [&](
const SCEV *Expr,
SCEVTypes &SCTy,
const SCEV *&
LHS,
15608 const SCEV *&
RHS) {
15610 if (MinMax->getNumOperands() != 2)
15613 if (
C->getAPInt().isNegative())
15615 SCTy = MinMax->getSCEVType();
15616 LHS = MinMax->getOperand(0);
15617 RHS = MinMax->getOperand(1);
15626 auto GetNonNegExprAndPosDivisor = [&](
const SCEV *Expr,
const SCEV *Divisor,
15627 APInt &ExprVal, APInt &DivisorVal) {
15630 if (!ConstExpr || !ConstDivisor)
15632 ExprVal = ConstExpr->getAPInt();
15633 DivisorVal = ConstDivisor->getAPInt();
15634 return ExprVal.isNonNegative() && !DivisorVal.isNonPositive();
15640 auto GetNextSCEVDividesByDivisor = [&](
const SCEV *Expr,
15641 const SCEV *Divisor) {
15644 if (!GetNonNegExprAndPosDivisor(Expr, Divisor, ExprVal, DivisorVal))
15646 APInt Rem = ExprVal.
urem(DivisorVal);
15649 return SE.
getConstant(ExprVal + DivisorVal - Rem);
15656 auto GetPreviousSCEVDividesByDivisor = [&](
const SCEV *Expr,
15657 const SCEV *Divisor) {
15660 if (!GetNonNegExprAndPosDivisor(Expr, Divisor, ExprVal, DivisorVal))
15662 APInt Rem = ExprVal.
urem(DivisorVal);
15670 std::function<
const SCEV *(
const SCEV *,
const SCEV *)>
15671 ApplyDivisibiltyOnMinMaxExpr = [&](
const SCEV *MinMaxExpr,
15672 const SCEV *Divisor) {
15673 const SCEV *MinMaxLHS =
nullptr, *MinMaxRHS =
nullptr;
15675 if (!IsMinMaxSCEVWithNonNegativeConstant(MinMaxExpr, SCTy, MinMaxLHS,
15681 "Expected non-negative operand!");
15682 auto *DivisibleExpr =
15683 IsMin ? GetPreviousSCEVDividesByDivisor(MinMaxLHS, Divisor)
15684 : GetNextSCEVDividesByDivisor(MinMaxLHS, Divisor);
15686 ApplyDivisibiltyOnMinMaxExpr(MinMaxRHS, Divisor), DivisibleExpr};
15695 const SCEV *URemLHS =
nullptr;
15696 const SCEV *URemRHS =
nullptr;
15697 if (SE.matchURem(
LHS, URemLHS, URemRHS)) {
15699 auto I = RewriteMap.find(LHSUnknown);
15700 const SCEV *RewrittenLHS =
15701 I != RewriteMap.end() ?
I->second : LHSUnknown;
15702 RewrittenLHS = ApplyDivisibiltyOnMinMaxExpr(RewrittenLHS, URemRHS);
15703 const auto *Multiple =
15705 RewriteMap[LHSUnknown] = Multiple;
15726 auto AddRewrite = [&](
const SCEV *From,
const SCEV *FromRewritten,
15728 if (From == FromRewritten)
15730 RewriteMap[From] = To;
15736 auto GetMaybeRewritten = [&](
const SCEV *S) {
15737 return RewriteMap.lookup_or(S, S);
15747 std::function<bool(
const SCEV *,
const SCEV *&)> HasDivisibiltyInfo =
15748 [&](
const SCEV *Expr,
const SCEV *&DividesBy) {
15750 if (
Mul->getNumOperands() != 2)
15752 auto *MulLHS =
Mul->getOperand(0);
15753 auto *MulRHS =
Mul->getOperand(1);
15757 if (Div->getOperand(1) == MulRHS) {
15758 DividesBy = MulRHS;
15763 return HasDivisibiltyInfo(MinMax->getOperand(0), DividesBy) ||
15764 HasDivisibiltyInfo(MinMax->getOperand(1), DividesBy);
15769 std::function<bool(
const SCEV *,
const SCEV *&)> IsKnownToDivideBy =
15770 [&](
const SCEV *Expr,
const SCEV *DividesBy) {
15774 return IsKnownToDivideBy(MinMax->getOperand(0), DividesBy) &&
15775 IsKnownToDivideBy(MinMax->getOperand(1), DividesBy);
15779 const SCEV *RewrittenLHS = GetMaybeRewritten(
LHS);
15780 const SCEV *DividesBy =
nullptr;
15781 if (HasDivisibiltyInfo(RewrittenLHS, DividesBy))
15784 IsKnownToDivideBy(RewrittenLHS, DividesBy) ? DividesBy :
nullptr;
15798 switch (Predicate) {
15806 RHS = DividesBy ? GetPreviousSCEVDividesByDivisor(
RHS, DividesBy) :
RHS;
15812 RHS = DividesBy ? GetNextSCEVDividesByDivisor(
RHS, DividesBy) :
RHS;
15816 RHS = DividesBy ? GetPreviousSCEVDividesByDivisor(
RHS, DividesBy) :
RHS;
15820 RHS = DividesBy ? GetNextSCEVDividesByDivisor(
RHS, DividesBy) :
RHS;
15827 SmallPtrSet<const SCEV *, 16> Visited;
15829 auto EnqueueOperands = [&Worklist](
const SCEVNAryExpr *S) {
15833 while (!Worklist.
empty()) {
15837 if (!Visited.
insert(From).second)
15839 const SCEV *FromRewritten = GetMaybeRewritten(From);
15840 const SCEV *To =
nullptr;
15842 switch (Predicate) {
15847 EnqueueOperands(
UMax);
15853 EnqueueOperands(
SMax);
15859 EnqueueOperands(
UMin);
15865 EnqueueOperands(
SMin);
15873 const SCEV *OneAlignedUp =
15874 DividesBy ? GetNextSCEVDividesByDivisor(One, DividesBy) : One;
15875 To = SE.
getUMaxExpr(FromRewritten, OneAlignedUp);
15883 AddRewrite(From, FromRewritten, To);
15900 SE.F.
getParent(), Intrinsic::experimental_guard);
15902 for (
const auto *GU : GuardDecl->users())
15904 if (Guard->getFunction() ==
Block->getParent() &&
15913 unsigned NumCollectedConditions = 0;
15915 std::pair<const BasicBlock *, const BasicBlock *> Pair(Pred,
Block);
15917 Pair = SE.getPredecessorWithUniqueSuccessorForBB(Pair.first)) {
15918 VisitedBlocks.
insert(Pair.second);
15919 const BranchInst *LoopEntryPredicate =
15926 NumCollectedConditions++;
15930 if (
Depth > 0 && NumCollectedConditions == 2)
15938 if (Pair.second->hasNPredecessorsOrMore(2) &&
15940 SmallDenseMap<const BasicBlock *, LoopGuards> IncomingGuards;
15941 for (
auto &Phi : Pair.second->phis())
15942 collectFromPHI(SE, Guards, Phi, VisitedBlocks, IncomingGuards,
Depth);
15949 for (
auto [Term, EnterIfTrue] :
reverse(Terms)) {
15950 SmallVector<Value *, 8> Worklist;
15951 SmallPtrSet<Value *, 8> Visited;
15953 while (!Worklist.
empty()) {
15960 EnterIfTrue ?
Cmp->getPredicate() :
Cmp->getInversePredicate();
15963 CollectCondition(Predicate,
LHS,
RHS, Guards.RewriteMap);
15979 Guards.PreserveNUW =
true;
15980 Guards.PreserveNSW =
true;
15981 for (
const SCEV *Expr : ExprsToRewrite) {
15982 const SCEV *RewriteTo = Guards.RewriteMap[Expr];
15983 Guards.PreserveNUW &=
15985 Guards.PreserveNSW &=
15992 if (ExprsToRewrite.size() > 1) {
15993 for (
const SCEV *Expr : ExprsToRewrite) {
15994 const SCEV *RewriteTo = Guards.RewriteMap[Expr];
15995 Guards.RewriteMap.erase(Expr);
15996 Guards.RewriteMap.insert({Expr, Guards.
rewrite(RewriteTo)});
16005 class SCEVLoopGuardRewriter
16015 if (Guards.PreserveNUW)
16017 if (Guards.PreserveNSW)
16024 return Map.lookup_or(Expr, Expr);
16028 if (
const SCEV *S = Map.lookup(Expr))
16035 unsigned Bitwidth = Ty->getScalarSizeInBits() / 2;
16036 while (Bitwidth % 8 == 0 && Bitwidth >= 8 &&
16037 Bitwidth >
Op->getType()->getScalarSizeInBits()) {
16039 auto *NarrowExt = SE.getZeroExtendExpr(
Op, NarrowTy);
16040 if (
const SCEV *S = Map.lookup(NarrowExt))
16041 return SE.getZeroExtendExpr(S, Ty);
16042 Bitwidth = Bitwidth / 2;
16050 if (
const SCEV *S = Map.lookup(Expr))
16057 if (
const SCEV *S = Map.lookup(Expr))
16063 if (
const SCEV *S = Map.lookup(Expr))
16075 if (
const SCEV *S = Map.lookup(
16077 return SE.getAddExpr(Expr->
getOperand(0), S);
16111 if (RewriteMap.empty())
16114 SCEVLoopGuardRewriter
Rewriter(SE, *
this);
16115 return Rewriter.visit(Expr);
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Expand Atomic instructions
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file defines the DenseMap class.
This file builds on the ADT/GraphTraits.h file to build generic depth first graph iterator.
static bool isSigned(unsigned int Opcode)
This file defines a hash set that can be used to remove duplication of nodes in a graph.
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
This defines the Use class.
iv Induction Variable Users
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
mir Rename Register Operands
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
PowerPC Reduce CR logical Operation
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
const SmallVectorImpl< MachineOperand > & Cond
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
SI optimize exec mask operations pre RA
void visit(MachineFunction &MF, MachineBasicBlock &Start, std::function< void(MachineBasicBlock *)> op)
This file provides utility classes that use RAII to save and restore values.
bool SCEVMinMaxExprContains(const SCEV *Root, const SCEV *OperandToFind, SCEVTypes RootKind)
static cl::opt< unsigned > MaxAddRecSize("scalar-evolution-max-add-rec-size", cl::Hidden, cl::desc("Max coefficients in AddRec during evolving"), cl::init(8))
static cl::opt< unsigned > RangeIterThreshold("scev-range-iter-threshold", cl::Hidden, cl::desc("Threshold for switching to iteratively computing SCEV ranges"), cl::init(32))
static const Loop * isIntegerLoopHeaderPHI(const PHINode *PN, LoopInfo &LI)
static unsigned getConstantTripCount(const SCEVConstant *ExitCount)
static int CompareValueComplexity(const LoopInfo *const LI, Value *LV, Value *RV, unsigned Depth)
Compare the two values LV and RV in terms of their "complexity" where "complexity" is a partial (and ...
static void PushLoopPHIs(const Loop *L, SmallVectorImpl< Instruction * > &Worklist, SmallPtrSetImpl< Instruction * > &Visited)
Push PHI nodes in the header of the given loop onto the given Worklist.
static void insertFoldCacheEntry(const ScalarEvolution::FoldID &ID, const SCEV *S, DenseMap< ScalarEvolution::FoldID, const SCEV * > &FoldCache, DenseMap< const SCEV *, SmallVector< ScalarEvolution::FoldID, 2 > > &FoldCacheUser)
static cl::opt< bool > ClassifyExpressions("scalar-evolution-classify-expressions", cl::Hidden, cl::init(true), cl::desc("When printing analysis, include information on every instruction"))
static bool CanConstantFold(const Instruction *I)
Return true if we can constant fold an instruction of the specified type, assuming that all operands ...
static cl::opt< unsigned > AddOpsInlineThreshold("scev-addops-inline-threshold", cl::Hidden, cl::desc("Threshold for inlining addition operands into a SCEV"), cl::init(500))
static cl::opt< unsigned > MaxLoopGuardCollectionDepth("scalar-evolution-max-loop-guard-collection-depth", cl::Hidden, cl::desc("Maximum depth for recursive loop guard collection"), cl::init(1))
static cl::opt< bool > VerifyIR("scev-verify-ir", cl::Hidden, cl::desc("Verify IR correctness when making sensitive SCEV queries (slow)"), cl::init(false))
static bool BrPHIToSelect(DominatorTree &DT, BranchInst *BI, PHINode *Merge, Value *&C, Value *&LHS, Value *&RHS)
static const SCEV * getPreStartForExtend(const SCEVAddRecExpr *AR, Type *Ty, ScalarEvolution *SE, unsigned Depth)
static std::optional< APInt > MinOptional(std::optional< APInt > X, std::optional< APInt > Y)
Helper function to compare optional APInts: (a) if X and Y both exist, return min(X,...
static cl::opt< unsigned > MulOpsInlineThreshold("scev-mulops-inline-threshold", cl::Hidden, cl::desc("Threshold for inlining multiplication operands into a SCEV"), cl::init(32))
static void GroupByComplexity(SmallVectorImpl< const SCEV * > &Ops, LoopInfo *LI, DominatorTree &DT)
Given a list of SCEV objects, order them by their complexity, and group objects of the same complexit...
static const SCEV * constantFoldAndGroupOps(ScalarEvolution &SE, LoopInfo &LI, DominatorTree &DT, SmallVectorImpl< const SCEV * > &Ops, FoldT Fold, IsIdentityT IsIdentity, IsAbsorberT IsAbsorber)
Performs a number of common optimizations on the passed Ops.
static std::optional< const SCEV * > createNodeForSelectViaUMinSeq(ScalarEvolution *SE, const SCEV *CondExpr, const SCEV *TrueExpr, const SCEV *FalseExpr)
static Constant * BuildConstantFromSCEV(const SCEV *V)
This builds up a Constant using the ConstantExpr interface.
static ConstantInt * EvaluateConstantChrecAtConstant(const SCEVAddRecExpr *AddRec, ConstantInt *C, ScalarEvolution &SE)
static const SCEV * BinomialCoefficient(const SCEV *It, unsigned K, ScalarEvolution &SE, Type *ResultTy)
Compute BC(It, K). The result has width W. Assume, K > 0.
static cl::opt< unsigned > MaxCastDepth("scalar-evolution-max-cast-depth", cl::Hidden, cl::desc("Maximum depth of recursive SExt/ZExt/Trunc"), cl::init(8))
static bool IsMinMaxConsistingOf(const SCEV *MaybeMinMaxExpr, const SCEV *Candidate)
Is MaybeMinMaxExpr an (U|S)(Min|Max) of Candidate and some other values?
static PHINode * getConstantEvolvingPHI(Value *V, const Loop *L)
getConstantEvolvingPHI - Given an LLVM value and a loop, return a PHI node in the loop that V is deri...
static cl::opt< unsigned > MaxBruteForceIterations("scalar-evolution-max-iterations", cl::ReallyHidden, cl::desc("Maximum number of iterations SCEV will " "symbolically execute a constant " "derived loop"), cl::init(100))
static bool MatchBinarySub(const SCEV *S, const SCEV *&LHS, const SCEV *&RHS)
static uint64_t umul_ov(uint64_t i, uint64_t j, bool &Overflow)
static void PrintSCEVWithTypeHint(raw_ostream &OS, const SCEV *S)
When printing a top-level SCEV for trip counts, it's helpful to include a type for constants which ar...
static void PrintLoopInfo(raw_ostream &OS, ScalarEvolution *SE, const Loop *L)
static bool containsConstantInAddMulChain(const SCEV *StartExpr)
Determine if any of the operands in this SCEV are a constant or if any of the add or multiply express...
static const SCEV * getExtendAddRecStart(const SCEVAddRecExpr *AR, Type *Ty, ScalarEvolution *SE, unsigned Depth)
static bool hasHugeExpression(ArrayRef< const SCEV * > Ops)
Returns true if Ops contains a huge SCEV (the subtree of S contains at least HugeExprThreshold nodes)...
static cl::opt< unsigned > MaxPhiSCCAnalysisSize("scalar-evolution-max-scc-analysis-depth", cl::Hidden, cl::desc("Maximum amount of nodes to process while searching SCEVUnknown " "Phi strongly connected components"), cl::init(8))
static bool IsKnownPredicateViaAddRecStart(ScalarEvolution &SE, CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
static cl::opt< unsigned > MaxSCEVOperationsImplicationDepth("scalar-evolution-max-scev-operations-implication-depth", cl::Hidden, cl::desc("Maximum depth of recursive SCEV operations implication analysis"), cl::init(2))
static void PushDefUseChildren(Instruction *I, SmallVectorImpl< Instruction * > &Worklist, SmallPtrSetImpl< Instruction * > &Visited)
Push users of the given Instruction onto the given Worklist.
static std::optional< APInt > SolveQuadraticAddRecRange(const SCEVAddRecExpr *AddRec, const ConstantRange &Range, ScalarEvolution &SE)
Let c(n) be the value of the quadratic chrec {0,+,M,+,N} after n iterations.
static cl::opt< bool > UseContextForNoWrapFlagInference("scalar-evolution-use-context-for-no-wrap-flag-strenghening", cl::Hidden, cl::desc("Infer nuw/nsw flags using context where suitable"), cl::init(true))
static cl::opt< bool > EnableFiniteLoopControl("scalar-evolution-finite-loop", cl::Hidden, cl::desc("Handle <= and >= in finite loops"), cl::init(true))
static std::optional< std::tuple< APInt, APInt, APInt, APInt, unsigned > > GetQuadraticEquation(const SCEVAddRecExpr *AddRec)
For a given quadratic addrec, generate coefficients of the corresponding quadratic equation,...
static bool isKnownPredicateExtendIdiom(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
static std::optional< BinaryOp > MatchBinaryOp(Value *V, const DataLayout &DL, AssumptionCache &AC, const DominatorTree &DT, const Instruction *CxtI)
Try to map V into a BinaryOp, and return std::nullopt on failure.
static std::optional< APInt > SolveQuadraticAddRecExact(const SCEVAddRecExpr *AddRec, ScalarEvolution &SE)
Let c(n) be the value of the quadratic chrec {L,+,M,+,N} after n iterations.
static std::optional< APInt > TruncIfPossible(std::optional< APInt > X, unsigned BitWidth)
Helper function to truncate an optional APInt to a given BitWidth.
static cl::opt< unsigned > MaxSCEVCompareDepth("scalar-evolution-max-scev-compare-depth", cl::Hidden, cl::desc("Maximum depth of recursive SCEV complexity comparisons"), cl::init(32))
static APInt extractConstantWithoutWrapping(ScalarEvolution &SE, const SCEVConstant *ConstantTerm, const SCEVAddExpr *WholeAddExpr)
static cl::opt< unsigned > MaxConstantEvolvingDepth("scalar-evolution-max-constant-evolving-depth", cl::Hidden, cl::desc("Maximum depth of recursive constant evolving"), cl::init(32))
static ConstantRange getRangeForAffineARHelper(APInt Step, const ConstantRange &StartRange, const APInt &MaxBECount, bool Signed)
static std::optional< ConstantRange > GetRangeFromMetadata(Value *V)
Helper method to assign a range to V from metadata present in the IR.
static const SCEV * SolveLinEquationWithOverflow(const APInt &A, const SCEV *B, SmallVectorImpl< const SCEVPredicate * > *Predicates, ScalarEvolution &SE)
Finds the minimum unsigned root of the following equation:
static cl::opt< unsigned > HugeExprThreshold("scalar-evolution-huge-expr-threshold", cl::Hidden, cl::desc("Size of the expression which is considered huge"), cl::init(4096))
static Type * isSimpleCastedPHI(const SCEV *Op, const SCEVUnknown *SymbolicPHI, bool &Signed, ScalarEvolution &SE)
Helper function to createAddRecFromPHIWithCasts.
static Constant * EvaluateExpression(Value *V, const Loop *L, DenseMap< Instruction *, Constant * > &Vals, const DataLayout &DL, const TargetLibraryInfo *TLI)
EvaluateExpression - Given an expression that passes the getConstantEvolvingPHI predicate,...
static const SCEV * MatchNotExpr(const SCEV *Expr)
If Expr computes ~A, return A else return nullptr.
static cl::opt< unsigned > MaxValueCompareDepth("scalar-evolution-max-value-compare-depth", cl::Hidden, cl::desc("Maximum depth of recursive value complexity comparisons"), cl::init(2))
static cl::opt< bool, true > VerifySCEVOpt("verify-scev", cl::Hidden, cl::location(VerifySCEV), cl::desc("Verify ScalarEvolution's backedge taken counts (slow)"))
static const SCEV * getSignedOverflowLimitForStep(const SCEV *Step, ICmpInst::Predicate *Pred, ScalarEvolution *SE)
static SCEV::NoWrapFlags StrengthenNoWrapFlags(ScalarEvolution *SE, SCEVTypes Type, const ArrayRef< const SCEV * > Ops, SCEV::NoWrapFlags Flags)
static cl::opt< unsigned > MaxArithDepth("scalar-evolution-max-arith-depth", cl::Hidden, cl::desc("Maximum depth of recursive arithmetics"), cl::init(32))
static bool HasSameValue(const SCEV *A, const SCEV *B)
SCEV structural equivalence is usually sufficient for testing whether two expressions are equal,...
static uint64_t Choose(uint64_t n, uint64_t k, bool &Overflow)
Compute the result of "n choose k", the binomial coefficient.
static std::optional< int > CompareSCEVComplexity(const LoopInfo *const LI, const SCEV *LHS, const SCEV *RHS, DominatorTree &DT, unsigned Depth=0)
static bool CollectAddOperandsWithScales(SmallDenseMap< const SCEV *, APInt, 16 > &M, SmallVectorImpl< const SCEV * > &NewOps, APInt &AccumulatedConstant, ArrayRef< const SCEV * > Ops, const APInt &Scale, ScalarEvolution &SE)
Process the given Ops list, which is a list of operands to be added under the given scale,...
static bool canConstantEvolve(Instruction *I, const Loop *L)
Determine whether this instruction can constant evolve within this loop assuming its operands can all...
static PHINode * getConstantEvolvingPHIOperands(Instruction *UseInst, const Loop *L, DenseMap< Instruction *, PHINode * > &PHIMap, unsigned Depth)
getConstantEvolvingPHIOperands - Implement getConstantEvolvingPHI by recursing through each instructi...
static bool scevUnconditionallyPropagatesPoisonFromOperands(SCEVTypes Kind)
static cl::opt< bool > VerifySCEVStrict("verify-scev-strict", cl::Hidden, cl::desc("Enable stricter verification with -verify-scev is passed"))
static Constant * getOtherIncomingValue(PHINode *PN, BasicBlock *BB)
static cl::opt< bool > UseExpensiveRangeSharpening("scalar-evolution-use-expensive-range-sharpening", cl::Hidden, cl::init(false), cl::desc("Use more powerful methods of sharpening expression ranges. May " "be costly in terms of compile time"))
static const SCEV * getUnsignedOverflowLimitForStep(const SCEV *Step, ICmpInst::Predicate *Pred, ScalarEvolution *SE)
static bool IsKnownPredicateViaMinOrMax(ScalarEvolution &SE, CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
Is LHS Pred RHS true on the virtue of LHS or RHS being a Min or Max expression?
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
static bool InBlock(const Value *V, const BasicBlock *BB)
Provides some synthesis utilities to produce sequences of values.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")
static SymbolRef::Type getType(const Symbol *Sym)
LocallyHashedType DenseMapInfo< LocallyHashedType >::Empty
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
static std::optional< bool > isImpliedCondOperands(CmpInst::Predicate Pred, const Value *ALHS, const Value *ARHS, const Value *BLHS, const Value *BRHS)
Return true if "icmp Pred BLHS BRHS" is true whenever "icmp PredALHS ARHS" is true.
Virtual Register Rewriter
static const uint32_t IV[8]
Class for arbitrary precision integers.
LLVM_ABI APInt umul_ov(const APInt &RHS, bool &Overflow) const
LLVM_ABI APInt zext(unsigned width) const
Zero extend to a new width.
bool isMinSignedValue() const
Determine if this is the smallest signed value.
uint64_t getZExtValue() const
Get zero extended value.
void setHighBits(unsigned hiBits)
Set the top hiBits bits.
LLVM_ABI APInt getHiBits(unsigned numBits) const
Compute an APInt containing numBits highbits from this APInt.
unsigned getActiveBits() const
Compute the number of active bits in the value.
LLVM_ABI APInt trunc(unsigned width) const
Truncate to new width.
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
APInt abs() const
Get the absolute value.
bool sgt(const APInt &RHS) const
Signed greater than comparison.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
bool isSignMask() const
Check if the APInt's value is returned by getSignMask.
LLVM_ABI APInt urem(const APInt &RHS) const
Unsigned remainder operation.
unsigned getBitWidth() const
Return the number of bits in the APInt.
bool ult(const APInt &RHS) const
Unsigned less than comparison.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
static APInt getMinValue(unsigned numBits)
Gets minimum unsigned value of APInt for a specific bit width.
bool isNegative() const
Determine sign of this APInt.
bool sle(const APInt &RHS) const
Signed less or equal comparison.
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
unsigned countTrailingZeros() const
bool isStrictlyPositive() const
Determine if this APInt Value is positive.
unsigned logBase2() const
APInt ashr(unsigned ShiftAmt) const
Arithmetic right-shift function.
LLVM_ABI APInt multiplicativeInverse() const
bool ule(const APInt &RHS) const
Unsigned less or equal comparison.
LLVM_ABI APInt sext(unsigned width) const
Sign extend to a new width.
APInt shl(unsigned shiftAmt) const
Left-shift function.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
bool isSignBitSet() const
Determine if sign bit of this APInt is set.
bool slt(const APInt &RHS) const
Signed less than comparison.
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
bool isIntN(unsigned N) const
Check if this APInt has an N-bits unsigned integer value.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
This templated class represents "all analyses that operate over <aparticular IR unit>" (e....
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Represent the analysis usage information of a pass.
void setPreservesAll()
Set by analyses that do not transform their input at all.
AnalysisUsage & addRequiredTransitive()
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
A function analysis which provides an AssumptionCache.
An immutable pass that tracks lazily created AssumptionCache objects.
A cache of @llvm.assume calls within a function.
MutableArrayRef< WeakVH > assumptions()
Access the list of assumption handles currently tracked for this function.
LLVM_ABI bool isSingleEdge() const
Check if this is the only edge between Start and End.
LLVM Basic Block Representation.
iterator begin()
Instruction iterator methods.
const Function * getParent() const
Return the enclosing method, or null if none.
LLVM_ABI const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
const Instruction & front() const
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
LLVM_ABI unsigned getNoWrapKind() const
Returns one of OBO::NoSignedWrap or OBO::NoUnsignedWrap.
LLVM_ABI Instruction::BinaryOps getBinaryOp() const
Returns the binary operation underlying the intrinsic.
BinaryOps getOpcode() const
Conditional or Unconditional Branch instruction.
bool isConditional() const
BasicBlock * getSuccessor(unsigned i) const
bool isUnconditional() const
Value * getCondition() const
LLVM_ATTRIBUTE_RETURNS_NONNULL void * Allocate(size_t Size, Align Alignment)
Allocate space at the specified alignment.
This class represents a function call, abstracting a target machine's calling convention.
virtual void deleted()
Callback for Value destruction.
bool isFalseWhenEqual() const
This is just a convenience.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_SLT
signed less than
@ ICMP_SLE
signed less or equal
@ ICMP_UGE
unsigned greater or equal
@ ICMP_UGT
unsigned greater than
@ ICMP_SGT
signed greater than
@ ICMP_ULT
unsigned less than
@ ICMP_SGE
signed greater or equal
@ ICMP_ULE
unsigned less or equal
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
bool isTrueWhenEqual() const
This is just a convenience.
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
bool isRelational() const
Return true if the predicate is relational (not EQ or NE).
An abstraction over a floating-point predicate, and a pack of an integer predicate with samesign info...
static LLVM_ABI std::optional< CmpPredicate > getMatching(CmpPredicate A, CmpPredicate B)
Compares two CmpPredicates taking samesign into account and returns the canonicalized CmpPredicate if...
LLVM_ABI CmpInst::Predicate getPreferredSignedPredicate() const
Attempts to return a signed CmpInst::Predicate from the CmpPredicate.
CmpInst::Predicate dropSameSign() const
Drops samesign information.
static LLVM_ABI Constant * getNot(Constant *C)
static LLVM_ABI Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static Constant * getGetElementPtr(Type *Ty, Constant *C, ArrayRef< Constant * > IdxList, GEPNoWrapFlags NW=GEPNoWrapFlags::none(), std::optional< ConstantRange > InRange=std::nullopt, Type *OnlyIfReducedTy=nullptr)
Getelementptr form.
static LLVM_ABI Constant * getAdd(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
static LLVM_ABI Constant * getNeg(Constant *C, bool HasNSW=false)
static LLVM_ABI Constant * getTrunc(Constant *C, Type *Ty, bool OnlyIfReduced=false)
This is the shared class of boolean and integer constants.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
const APInt & getValue() const
Return the constant as an APInt value reference.
static LLVM_ABI ConstantInt * getBool(LLVMContext &Context, bool V)
This class represents a range of values.
LLVM_ABI ConstantRange add(const ConstantRange &Other) const
Return a new range representing the possible values resulting from an addition of a value in this ran...
LLVM_ABI ConstantRange zextOrTrunc(uint32_t BitWidth) const
Make this range have the bit width given by BitWidth.
PreferredRangeType
If represented precisely, the result of some range operations may consist of multiple disjoint ranges...
LLVM_ABI bool getEquivalentICmp(CmpInst::Predicate &Pred, APInt &RHS) const
Set up Pred and RHS such that ConstantRange::makeExactICmpRegion(Pred, RHS) == *this.
const APInt & getLower() const
Return the lower value for this range.
LLVM_ABI bool isFullSet() const
Return true if this set contains all of the elements possible for this data-type.
LLVM_ABI bool icmp(CmpInst::Predicate Pred, const ConstantRange &Other) const
Does the predicate Pred hold between ranges this and Other?
LLVM_ABI bool isEmptySet() const
Return true if this set contains no members.
LLVM_ABI ConstantRange zeroExtend(uint32_t BitWidth) const
Return a new range in the specified integer type, which must be strictly larger than the current type...
LLVM_ABI bool isSignWrappedSet() const
Return true if this set wraps around the signed domain.
LLVM_ABI APInt getSignedMin() const
Return the smallest signed value contained in the ConstantRange.
LLVM_ABI bool isWrappedSet() const
Return true if this set wraps around the unsigned domain.
LLVM_ABI void print(raw_ostream &OS) const
Print out the bounds to a stream.
LLVM_ABI ConstantRange truncate(uint32_t BitWidth, unsigned NoWrapKind=0) const
Return a new range in the specified integer type, which must be strictly smaller than the current typ...
LLVM_ABI ConstantRange signExtend(uint32_t BitWidth) const
Return a new range in the specified integer type, which must be strictly larger than the current type...
const APInt & getUpper() const
Return the upper value for this range.
LLVM_ABI ConstantRange unionWith(const ConstantRange &CR, PreferredRangeType Type=Smallest) const
Return the range that results from the union of this range with another range.
static LLVM_ABI ConstantRange makeExactICmpRegion(CmpInst::Predicate Pred, const APInt &Other)
Produce the exact range such that all values in the returned range satisfy the given predicate with a...
LLVM_ABI bool contains(const APInt &Val) const
Return true if the specified value is in the set.
LLVM_ABI APInt getUnsignedMax() const
Return the largest unsigned value contained in the ConstantRange.
LLVM_ABI ConstantRange intersectWith(const ConstantRange &CR, PreferredRangeType Type=Smallest) const
Return the range that results from the intersection of this range with another range.
LLVM_ABI APInt getSignedMax() const
Return the largest signed value contained in the ConstantRange.
static ConstantRange getNonEmpty(APInt Lower, APInt Upper)
Create non-empty constant range with the given bounds.
static LLVM_ABI ConstantRange makeGuaranteedNoWrapRegion(Instruction::BinaryOps BinOp, const ConstantRange &Other, unsigned NoWrapKind)
Produce the largest range containing all X such that "X BinOp Y" is guaranteed not to wrap (overflow)...
LLVM_ABI unsigned getMinSignedBits() const
Compute the maximal number of bits needed to represent every value in this signed range.
uint32_t getBitWidth() const
Get the bit width of this ConstantRange.
LLVM_ABI ConstantRange sub(const ConstantRange &Other) const
Return a new range representing the possible values resulting from a subtraction of a value in this r...
LLVM_ABI ConstantRange sextOrTrunc(uint32_t BitWidth) const
Make this range have the bit width given by BitWidth.
static LLVM_ABI ConstantRange makeExactNoWrapRegion(Instruction::BinaryOps BinOp, const APInt &Other, unsigned NoWrapKind)
Produce the range that contains X if and only if "X BinOp Other" does not wrap.
This is an important base class in LLVM.
A parsed version of the target data layout string in and methods for querying it.
LLVM_ABI const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
LLVM_ABI IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space.
LLVM_ABI unsigned getIndexTypeSizeInBits(Type *Ty) const
The size in bits of the index used in GEP calculation for this type.
LLVM_ABI IntegerType * getIndexType(LLVMContext &C, unsigned AddressSpace) const
Returns the type of a GEP index in AddressSpace.
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)
DenseMapIterator< KeyT, ValueT, KeyInfoT, BucketT > iterator
iterator find_as(const LookupKeyT &Val)
Alternate version of find() which allows a different, and possibly less expensive,...
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Analysis pass which computes a DominatorTree.
Legacy analysis pass which computes a DominatorTree.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
LLVM_ABI bool isReachableFromEntry(const Use &U) const
Provide an overload for a Use.
LLVM_ABI bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
FoldingSetNodeIDRef - This class describes a reference to an interned FoldingSetNodeID,...
FoldingSetNodeID - This class is used to gather all the unique data bits of a node.
Represents flags for the getelementptr instruction/expression.
bool hasNoUnsignedSignedWrap() const
bool hasNoUnsignedWrap() const
static GEPNoWrapFlags none()
static LLVM_ABI Type * getTypeAtIndex(Type *Ty, Value *Idx)
Return the type of the element at the given index of an indexable type.
Module * getParent()
Get the module that this global value is contained inside of...
static bool isPrivateLinkage(LinkageTypes Linkage)
static bool isInternalLinkage(LinkageTypes Linkage)
This instruction compares its operands according to the predicate given to the constructor.
CmpPredicate getCmpPredicate() const
static bool isGE(Predicate P)
Return true if the predicate is SGE or UGE.
CmpPredicate getSwappedCmpPredicate() const
static LLVM_ABI bool compare(const APInt &LHS, const APInt &RHS, ICmpInst::Predicate Pred)
Return result of LHS Pred RHS comparison.
static bool isLT(Predicate P)
Return true if the predicate is SLT or ULT.
CmpPredicate getInverseCmpPredicate() const
Predicate getNonStrictCmpPredicate() const
For example, SGT -> SGE, SLT -> SLE, ULT -> ULE, UGT -> UGE.
static bool isGT(Predicate P)
Return true if the predicate is SGT or UGT.
Predicate getFlippedSignednessPredicate() const
For example, SLT->ULT, ULT->SLT, SLE->ULE, ULE->SLE, EQ->EQ.
static CmpPredicate getInverseCmpPredicate(CmpPredicate Pred)
static bool isEquality(Predicate P)
Return true if this predicate is either EQ or NE.
bool isRelational() const
Return true if the predicate is relational (not EQ or NE).
static bool isLE(Predicate P)
Return true if the predicate is SLE or ULE.
LLVM_ABI bool hasNoUnsignedWrap() const LLVM_READONLY
Determine whether the no unsigned wrap flag is set.
LLVM_ABI bool hasNoSignedWrap() const LLVM_READONLY
Determine whether the no signed wrap flag is set.
LLVM_ABI bool isIdenticalToWhenDefined(const Instruction *I, bool IntersectAttrs=false) const LLVM_READONLY
This is like isIdenticalTo, except that it ignores the SubclassOptionalData flags,...
Class to represent integer types.
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
An instruction for reading from memory.
Analysis pass that exposes the LoopInfo for a function.
bool contains(const LoopT *L) const
Return true if the specified loop is contained within in this loop.
BlockT * getHeader() const
unsigned getLoopDepth() const
Return the nesting level of this loop.
BlockT * getLoopPredecessor() const
If the given loop's header has exactly one unique predecessor outside the loop, return it.
LoopT * getParentLoop() const
Return the parent loop if it exists or nullptr for top level loops.
unsigned getLoopDepth(const BlockT *BB) const
Return the loop nesting level of the specified block.
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
The legacy pass manager's analysis pass to compute loop information.
Represents a single loop in the control flow graph.
bool isLoopInvariant(const Value *V) const
Return true if the specified value is loop invariant.
A Module instance is used to store all the information related to an LLVM module.
unsigned getOpcode() const
Return the opcode for this Instruction or ConstantExpr.
Utility class for integer operators which may exhibit overflow - Add, Sub, Mul, and Shl.
bool hasNoSignedWrap() const
Test whether this operation is known to never undergo signed overflow, aka the nsw property.
bool hasNoUnsignedWrap() const
Test whether this operation is known to never undergo unsigned overflow, aka the nuw property.
iterator_range< const_block_iterator > blocks() const
op_range incoming_values()
Value * getIncomingValueForBlock(const BasicBlock *BB) const
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
AnalysisType & getAnalysis() const
getAnalysis<AnalysisType>() - This function is used by subclasses to get to the analysis information ...
PointerIntPair - This class implements a pair of a pointer and small integer.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
LLVM_ABI void addPredicate(const SCEVPredicate &Pred)
Adds a new predicate.
LLVM_ABI const SCEVPredicate & getPredicate() const
LLVM_ABI bool hasNoOverflow(Value *V, SCEVWrapPredicate::IncrementWrapFlags Flags)
Returns true if we've proved that V doesn't wrap by means of a SCEV predicate.
LLVM_ABI void setNoOverflow(Value *V, SCEVWrapPredicate::IncrementWrapFlags Flags)
Proves that V doesn't overflow by adding SCEV predicate.
LLVM_ABI void print(raw_ostream &OS, unsigned Depth) const
Print the SCEV mappings done by the Predicated Scalar Evolution.
LLVM_ABI bool areAddRecsEqualWithPreds(const SCEVAddRecExpr *AR1, const SCEVAddRecExpr *AR2) const
Check if AR1 and AR2 are equal, while taking into account Equal predicates in Preds.
LLVM_ABI PredicatedScalarEvolution(ScalarEvolution &SE, Loop &L)
LLVM_ABI const SCEVAddRecExpr * getAsAddRec(Value *V)
Attempts to produce an AddRecExpr for V by adding additional SCEV predicates.
LLVM_ABI unsigned getSmallConstantMaxTripCount()
Returns the upper bound of the loop trip count as a normal unsigned value, or 0 if the trip count is ...
LLVM_ABI const SCEV * getBackedgeTakenCount()
Get the (predicated) backedge count for the analyzed loop.
LLVM_ABI const SCEV * getSymbolicMaxBackedgeTakenCount()
Get the (predicated) symbolic max backedge count for the analyzed loop.
LLVM_ABI const SCEV * getSCEV(Value *V)
Returns the SCEV expression of V, in the context of the current SCEV predicate.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
PreservedAnalysisChecker getChecker() const
Build a checker for this PreservedAnalyses and the specified analysis type.
constexpr bool isValid() const
This node represents an addition of some number of SCEVs.
This node represents a polynomial recurrence on the trip count of the specified loop.
friend class ScalarEvolution
const SCEV * getStart() const
LLVM_ABI const SCEV * evaluateAtIteration(const SCEV *It, ScalarEvolution &SE) const
Return the value of this chain of recurrences at the specified iteration number.
const SCEV * getStepRecurrence(ScalarEvolution &SE) const
Constructs and returns the recurrence indicating how much this expression steps by.
void setNoWrapFlags(NoWrapFlags Flags)
Set flags for a recurrence without clearing any previously set flags.
bool isAffine() const
Return true if this represents an expression A + B*x where A and B are loop invariant values.
bool isQuadratic() const
Return true if this represents an expression A + B*x + C*x^2 where A, B and C are loop invariant valu...
LLVM_ABI const SCEV * getNumIterationsInRange(const ConstantRange &Range, ScalarEvolution &SE) const
Return the number of iterations of this loop that produce values in the specified constant range.
LLVM_ABI const SCEVAddRecExpr * getPostIncExpr(ScalarEvolution &SE) const
Return an expression representing the value of this expression one iteration of the loop ahead.
const Loop * getLoop() const
This is the base class for unary cast operator classes.
const SCEV * getOperand() const
LLVM_ABI SCEVCastExpr(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy, const SCEV *op, Type *ty)
void setNoWrapFlags(NoWrapFlags Flags)
Set flags for a non-recurrence without clearing previously set flags.
This class represents an assumption that the expression LHS Pred RHS evaluates to true,...
SCEVComparePredicate(const FoldingSetNodeIDRef ID, const ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS)
bool isAlwaysTrue() const override
Returns true if the predicate is always true.
void print(raw_ostream &OS, unsigned Depth=0) const override
Prints a textual representation of this predicate with an indentation of Depth.
bool implies(const SCEVPredicate *N, ScalarEvolution &SE) const override
Implementation of the SCEVPredicate interface.
This class represents a constant integer value.
ConstantInt * getValue() const
const APInt & getAPInt() const
This is the base class for unary integral cast operator classes.
LLVM_ABI SCEVIntegralCastExpr(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy, const SCEV *op, Type *ty)
This node is the base class min/max selections.
static enum SCEVTypes negate(enum SCEVTypes T)
This node represents multiplication of some number of SCEVs.
This node is a base class providing common functionality for n'ary operators.
bool hasNoUnsignedWrap() const
bool hasNoSelfWrap() const
size_t getNumOperands() const
bool hasNoSignedWrap() const
NoWrapFlags getNoWrapFlags(NoWrapFlags Mask=NoWrapMask) const
const SCEV * getOperand(unsigned i) const
const SCEV *const * Operands
ArrayRef< const SCEV * > operands() const
This class represents an assumption made using SCEV expressions which can be checked at run-time.
SCEVPredicate(const SCEVPredicate &)=default
virtual bool implies(const SCEVPredicate *N, ScalarEvolution &SE) const =0
Returns true if this predicate implies N.
This class represents a cast from a pointer to a pointer-sized integer value.
This visitor recursively visits a SCEV expression and re-writes it.
const SCEV * visitSignExtendExpr(const SCEVSignExtendExpr *Expr)
const SCEV * visit(const SCEV *S)
const SCEV * visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr)
const SCEV * visitSMinExpr(const SCEVSMinExpr *Expr)
const SCEV * visitUMinExpr(const SCEVUMinExpr *Expr)
This class represents a signed minimum selection.
This node is the base class for sequential/in-order min/max selections.
static SCEVTypes getEquivalentNonSequentialSCEVType(SCEVTypes Ty)
This class represents a sign extension of a small integer value to a larger integer value.
Visit all nodes in the expression tree using worklist traversal.
This class represents a truncation of an integer value to a smaller integer value.
This class represents a binary unsigned division operation.
const SCEV * getLHS() const
const SCEV * getRHS() const
This class represents an unsigned minimum selection.
This class represents a composition of other SCEV predicates, and is the class that most clients will...
void print(raw_ostream &OS, unsigned Depth) const override
Prints a textual representation of this predicate with an indentation of Depth.
bool implies(const SCEVPredicate *N, ScalarEvolution &SE) const override
Returns true if this predicate implies N.
SCEVUnionPredicate(ArrayRef< const SCEVPredicate * > Preds, ScalarEvolution &SE)
Union predicates don't get cached so create a dummy set ID for it.
bool isAlwaysTrue() const override
Implementation of the SCEVPredicate interface.
This means that we are dealing with an entirely unknown SCEV value, and only represent it as its LLVM...
This class represents the value of vscale, as used when defining the length of a scalable vector or r...
This class represents an assumption made on an AddRec expression.
IncrementWrapFlags
Similar to SCEV::NoWrapFlags, but with slightly different semantics for FlagNUSW.
SCEVWrapPredicate(const FoldingSetNodeIDRef ID, const SCEVAddRecExpr *AR, IncrementWrapFlags Flags)
bool implies(const SCEVPredicate *N, ScalarEvolution &SE) const override
Returns true if this predicate implies N.
static SCEVWrapPredicate::IncrementWrapFlags setFlags(SCEVWrapPredicate::IncrementWrapFlags Flags, SCEVWrapPredicate::IncrementWrapFlags OnFlags)
void print(raw_ostream &OS, unsigned Depth=0) const override
Prints a textual representation of this predicate with an indentation of Depth.
bool isAlwaysTrue() const override
Returns true if the predicate is always true.
const SCEVAddRecExpr * getExpr() const
Implementation of the SCEVPredicate interface.
static SCEVWrapPredicate::IncrementWrapFlags clearFlags(SCEVWrapPredicate::IncrementWrapFlags Flags, SCEVWrapPredicate::IncrementWrapFlags OffFlags)
Convenient IncrementWrapFlags manipulation methods.
static SCEVWrapPredicate::IncrementWrapFlags getImpliedFlags(const SCEVAddRecExpr *AR, ScalarEvolution &SE)
Returns the set of SCEVWrapPredicate no wrap flags implied by a SCEVAddRecExpr.
IncrementWrapFlags getFlags() const
Returns the set assumed no overflow flags.
This class represents a zero extension of a small integer value to a larger integer value.
This class represents an analyzed expression in the program.
LLVM_ABI ArrayRef< const SCEV * > operands() const
Return operands of this SCEV expression.
unsigned short getExpressionSize() const
LLVM_ABI bool isOne() const
Return true if the expression is a constant one.
LLVM_ABI bool isZero() const
Return true if the expression is a constant zero.
LLVM_ABI void dump() const
This method is used for debugging.
LLVM_ABI bool isAllOnesValue() const
Return true if the expression is a constant all-ones value.
LLVM_ABI bool isNonConstantNegative() const
Return true if the specified scev is negated, but not a constant.
LLVM_ABI void print(raw_ostream &OS) const
Print out the internal representation of this scalar to the specified stream.
SCEV(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy, unsigned short ExpressionSize)
SCEVTypes getSCEVType() const
LLVM_ABI Type * getType() const
Return the LLVM type of this SCEV expression.
NoWrapFlags
NoWrapFlags are bitfield indices into SubclassData.
Analysis pass that exposes the ScalarEvolution for a function.
LLVM_ABI ScalarEvolution run(Function &F, FunctionAnalysisManager &AM)
LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
void print(raw_ostream &OS, const Module *=nullptr) const override
print - Print out the internal state of the pass.
bool runOnFunction(Function &F) override
runOnFunction - Virtual method overriden by subclasses to do the per-function processing of the pass.
void releaseMemory() override
releaseMemory() - This member can be implemented by a pass if it wants to be able to release its memo...
void verifyAnalysis() const override
verifyAnalysis() - This member can be implemented by a analysis pass to check state of analysis infor...
ScalarEvolutionWrapperPass()
static LLVM_ABI LoopGuards collect(const Loop *L, ScalarEvolution &SE)
Collect rewrite map for loop guards for loop L, together with flags indicating if NUW and NSW can be ...
LLVM_ABI const SCEV * rewrite(const SCEV *Expr) const
Try to apply the collected loop guards to Expr.
The main scalar evolution driver.
const SCEV * getConstantMaxBackedgeTakenCount(const Loop *L)
When successful, this returns a SCEVConstant that is greater than or equal to (i.e.
static bool hasFlags(SCEV::NoWrapFlags Flags, SCEV::NoWrapFlags TestFlags)
const DataLayout & getDataLayout() const
Return the DataLayout associated with the module this SCEV instance is operating on.
LLVM_ABI bool isKnownNonNegative(const SCEV *S)
Test if the given expression is known to be non-negative.
LLVM_ABI bool isKnownOnEveryIteration(CmpPredicate Pred, const SCEVAddRecExpr *LHS, const SCEV *RHS)
Test if the condition described by Pred, LHS, RHS is known to be true on every iteration of the loop ...
LLVM_ABI const SCEV * getNegativeSCEV(const SCEV *V, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
Return the SCEV object corresponding to -V.
LLVM_ABI std::optional< LoopInvariantPredicate > getLoopInvariantExitCondDuringFirstIterationsImpl(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L, const Instruction *CtxI, const SCEV *MaxIter)
LLVM_ABI const SCEV * getSMaxExpr(const SCEV *LHS, const SCEV *RHS)
LLVM_ABI const SCEV * getUDivCeilSCEV(const SCEV *N, const SCEV *D)
Compute ceil(N / D).
LLVM_ABI const SCEV * getGEPExpr(GEPOperator *GEP, const SmallVectorImpl< const SCEV * > &IndexExprs)
Returns an expression for a GEP.
LLVM_ABI std::optional< LoopInvariantPredicate > getLoopInvariantExitCondDuringFirstIterations(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L, const Instruction *CtxI, const SCEV *MaxIter)
If the result of the predicate LHS Pred RHS is loop invariant with respect to L at given Context duri...
LLVM_ABI Type * getWiderType(Type *Ty1, Type *Ty2) const
LLVM_ABI const SCEV * getAbsExpr(const SCEV *Op, bool IsNSW)
LLVM_ABI bool isKnownNonPositive(const SCEV *S)
Test if the given expression is known to be non-positive.
LLVM_ABI const SCEV * getURemExpr(const SCEV *LHS, const SCEV *RHS)
Represents an unsigned remainder expression based on unsigned division.
LLVM_ABI APInt getConstantMultiple(const SCEV *S)
Returns the max constant multiple of S.
LLVM_ABI bool isKnownNegative(const SCEV *S)
Test if the given expression is known to be negative.
LLVM_ABI const SCEV * getPredicatedConstantMaxBackedgeTakenCount(const Loop *L, SmallVectorImpl< const SCEVPredicate * > &Predicates)
Similar to getConstantMaxBackedgeTakenCount, except it will add a set of SCEV predicates to Predicate...
LLVM_ABI const SCEV * removePointerBase(const SCEV *S)
Compute an expression equivalent to S - getPointerBase(S).
LLVM_ABI bool isLoopEntryGuardedByCond(const Loop *L, CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
Test whether entry to the loop is protected by a conditional between LHS and RHS.
LLVM_ABI bool isKnownNonZero(const SCEV *S)
Test if the given expression is known to be non-zero.
LLVM_ABI const SCEV * getSCEVAtScope(const SCEV *S, const Loop *L)
Return a SCEV expression for the specified value at the specified scope in the program.
LLVM_ABI const SCEV * getSMinExpr(const SCEV *LHS, const SCEV *RHS)
LLVM_ABI const SCEV * getBackedgeTakenCount(const Loop *L, ExitCountKind Kind=Exact)
If the specified loop has a predictable backedge-taken count, return it, otherwise return a SCEVCould...
LLVM_ABI const SCEV * getUMaxExpr(const SCEV *LHS, const SCEV *RHS)
LLVM_ABI void setNoWrapFlags(SCEVAddRecExpr *AddRec, SCEV::NoWrapFlags Flags)
Update no-wrap flags of an AddRec.
LLVM_ABI const SCEV * getUMaxFromMismatchedTypes(const SCEV *LHS, const SCEV *RHS)
Promote the operands to the wider of the types using zero-extension, and then perform a umax operatio...
const SCEV * getZero(Type *Ty)
Return a SCEV for the constant 0 of a specific type.
LLVM_ABI bool willNotOverflow(Instruction::BinaryOps BinOp, bool Signed, const SCEV *LHS, const SCEV *RHS, const Instruction *CtxI=nullptr)
Is operation BinOp between LHS and RHS provably does not have a signed/unsigned overflow (Signed)?
LLVM_ABI ExitLimit computeExitLimitFromCond(const Loop *L, Value *ExitCond, bool ExitIfTrue, bool ControlsOnlyExit, bool AllowPredicates=false)
Compute the number of times the backedge of the specified loop will execute if its exit condition wer...
LLVM_ABI const SCEV * getZeroExtendExprImpl(const SCEV *Op, Type *Ty, unsigned Depth=0)
LLVM_ABI const SCEVPredicate * getEqualPredicate(const SCEV *LHS, const SCEV *RHS)
LLVM_ABI unsigned getSmallConstantTripMultiple(const Loop *L, const SCEV *ExitCount)
Returns the largest constant divisor of the trip count as a normal unsigned value,...
LLVM_ABI uint64_t getTypeSizeInBits(Type *Ty) const
Return the size in bits of the specified type, for which isSCEVable must return true.
LLVM_ABI const SCEV * getConstant(ConstantInt *V)
LLVM_ABI const SCEV * getPredicatedBackedgeTakenCount(const Loop *L, SmallVectorImpl< const SCEVPredicate * > &Predicates)
Similar to getBackedgeTakenCount, except it will add a set of SCEV predicates to Predicates that are ...
LLVM_ABI const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
ConstantRange getSignedRange(const SCEV *S)
Determine the signed range for a particular SCEV.
LLVM_ABI const SCEV * getNoopOrSignExtend(const SCEV *V, Type *Ty)
Return a SCEV corresponding to a conversion of the input value to the specified type.
bool loopHasNoAbnormalExits(const Loop *L)
Return true if the loop has no abnormal exits.
LLVM_ABI const SCEV * getTripCountFromExitCount(const SCEV *ExitCount)
A version of getTripCountFromExitCount below which always picks an evaluation type which can not resu...
LLVM_ABI ScalarEvolution(Function &F, TargetLibraryInfo &TLI, AssumptionCache &AC, DominatorTree &DT, LoopInfo &LI)
const SCEV * getOne(Type *Ty)
Return a SCEV for the constant 1 of a specific type.
LLVM_ABI const SCEV * getTruncateOrNoop(const SCEV *V, Type *Ty)
Return a SCEV corresponding to a conversion of the input value to the specified type.
LLVM_ABI const SCEV * getCastExpr(SCEVTypes Kind, const SCEV *Op, Type *Ty)
LLVM_ABI const SCEV * getSequentialMinMaxExpr(SCEVTypes Kind, SmallVectorImpl< const SCEV * > &Operands)
LLVM_ABI const SCEV * getLosslessPtrToIntExpr(const SCEV *Op, unsigned Depth=0)
LLVM_ABI std::optional< bool > evaluatePredicateAt(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Instruction *CtxI)
Check whether the condition described by Pred, LHS, and RHS is true or false in the given Context.
LLVM_ABI unsigned getSmallConstantMaxTripCount(const Loop *L, SmallVectorImpl< const SCEVPredicate * > *Predicates=nullptr)
Returns the upper bound of the loop trip count as a normal unsigned value.
LLVM_ABI const SCEV * getPtrToIntExpr(const SCEV *Op, Type *Ty)
LLVM_ABI bool isBackedgeTakenCountMaxOrZero(const Loop *L)
Return true if the backedge taken count is either the value returned by getConstantMaxBackedgeTakenCo...
LLVM_ABI void forgetLoop(const Loop *L)
This method should be called by the client when it has changed a loop in a way that may effect Scalar...
LLVM_ABI bool isLoopInvariant(const SCEV *S, const Loop *L)
Return true if the value of the given SCEV is unchanging in the specified loop.
LLVM_ABI bool isKnownPositive(const SCEV *S)
Test if the given expression is known to be positive.
APInt getUnsignedRangeMin(const SCEV *S)
Determine the min of the unsigned range for a particular SCEV.
LLVM_ABI bool SimplifyICmpOperands(CmpPredicate &Pred, const SCEV *&LHS, const SCEV *&RHS, unsigned Depth=0)
Simplify LHS and RHS in a comparison with predicate Pred.
LLVM_ABI const SCEV * getOffsetOfExpr(Type *IntTy, StructType *STy, unsigned FieldNo)
Return an expression for offsetof on the given field with type IntTy.
LLVM_ABI LoopDisposition getLoopDisposition(const SCEV *S, const Loop *L)
Return the "disposition" of the given SCEV with respect to the given loop.
LLVM_ABI bool containsAddRecurrence(const SCEV *S)
Return true if the SCEV is a scAddRecExpr or it contains scAddRecExpr.
LLVM_ABI const SCEV * getSignExtendExprImpl(const SCEV *Op, Type *Ty, unsigned Depth=0)
LLVM_ABI const SCEV * getAddRecExpr(const SCEV *Start, const SCEV *Step, const Loop *L, SCEV::NoWrapFlags Flags)
Get an add recurrence expression for the specified loop.
LLVM_ABI bool hasOperand(const SCEV *S, const SCEV *Op) const
Test whether the given SCEV has Op as a direct or indirect operand.
LLVM_ABI const SCEV * getUDivExpr(const SCEV *LHS, const SCEV *RHS)
Get a canonical unsigned division expression, or something simpler if possible.
LLVM_ABI const SCEV * getZeroExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth=0)
LLVM_ABI bool isSCEVable(Type *Ty) const
Test if values of the given type are analyzable within the SCEV framework.
LLVM_ABI Type * getEffectiveSCEVType(Type *Ty) const
Return a type with the same bitwidth as the given type and which represents how SCEV will treat the g...
LLVM_ABI const SCEVPredicate * getComparePredicate(ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS)
LLVM_ABI const SCEV * getNotSCEV(const SCEV *V)
Return the SCEV object corresponding to ~V.
LLVM_ABI const SCEV * getElementCount(Type *Ty, ElementCount EC, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
LLVM_ABI bool instructionCouldExistWithOperands(const SCEV *A, const SCEV *B)
Return true if there exists a point in the program at which both A and B could be operands to the sam...
ConstantRange getUnsignedRange(const SCEV *S)
Determine the unsigned range for a particular SCEV.
LLVM_ABI uint32_t getMinTrailingZeros(const SCEV *S)
Determine the minimum number of zero bits that S is guaranteed to end in (at every loop iteration).
LLVM_ABI void print(raw_ostream &OS) const
LLVM_ABI const SCEV * getUMinExpr(const SCEV *LHS, const SCEV *RHS, bool Sequential=false)
LLVM_ABI const SCEV * getPredicatedExitCount(const Loop *L, const BasicBlock *ExitingBlock, SmallVectorImpl< const SCEVPredicate * > *Predicates, ExitCountKind Kind=Exact)
Same as above except this uses the predicated backedge taken info and may require predicates.
static SCEV::NoWrapFlags clearFlags(SCEV::NoWrapFlags Flags, SCEV::NoWrapFlags OffFlags)
LLVM_ABI void forgetTopmostLoop(const Loop *L)
LLVM_ABI void forgetValue(Value *V)
This method should be called by the client when it has changed a value in a way that may effect its v...
APInt getSignedRangeMin(const SCEV *S)
Determine the min of the signed range for a particular SCEV.
LLVM_ABI const SCEV * getNoopOrAnyExtend(const SCEV *V, Type *Ty)
Return a SCEV corresponding to a conversion of the input value to the specified type.
LLVM_ABI void forgetBlockAndLoopDispositions(Value *V=nullptr)
Called when the client has changed the disposition of values in a loop or block.
LLVM_ABI const SCEV * getTruncateExpr(const SCEV *Op, Type *Ty, unsigned Depth=0)
@ MonotonicallyDecreasing
@ MonotonicallyIncreasing
LLVM_ABI std::optional< LoopInvariantPredicate > getLoopInvariantPredicate(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L, const Instruction *CtxI=nullptr)
If the result of the predicate LHS Pred RHS is loop invariant with respect to L, return a LoopInvaria...
LLVM_ABI const SCEV * getStoreSizeOfExpr(Type *IntTy, Type *StoreTy)
Return an expression for the store size of StoreTy that is type IntTy.
LLVM_ABI const SCEVPredicate * getWrapPredicate(const SCEVAddRecExpr *AR, SCEVWrapPredicate::IncrementWrapFlags AddedFlags)
LLVM_ABI bool isLoopBackedgeGuardedByCond(const Loop *L, CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
Test whether the backedge of the loop is protected by a conditional between LHS and RHS.
LLVM_ABI const SCEV * getMinusSCEV(const SCEV *LHS, const SCEV *RHS, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap, unsigned Depth=0)
Return LHS-RHS.
LLVM_ABI APInt getNonZeroConstantMultiple(const SCEV *S)
const SCEV * getMinusOne(Type *Ty)
Return a SCEV for the constant -1 of a specific type.
static SCEV::NoWrapFlags setFlags(SCEV::NoWrapFlags Flags, SCEV::NoWrapFlags OnFlags)
LLVM_ABI bool hasLoopInvariantBackedgeTakenCount(const Loop *L)
Return true if the specified loop has an analyzable loop-invariant backedge-taken count.
LLVM_ABI BlockDisposition getBlockDisposition(const SCEV *S, const BasicBlock *BB)
Return the "disposition" of the given SCEV with respect to the given block.
LLVM_ABI const SCEV * getNoopOrZeroExtend(const SCEV *V, Type *Ty)
Return a SCEV corresponding to a conversion of the input value to the specified type.
LLVM_ABI bool invalidate(Function &F, const PreservedAnalyses &PA, FunctionAnalysisManager::Invalidator &Inv)
LLVM_ABI const SCEV * getUMinFromMismatchedTypes(const SCEV *LHS, const SCEV *RHS, bool Sequential=false)
Promote the operands to the wider of the types using zero-extension, and then perform a umin operatio...
LLVM_ABI bool loopIsFiniteByAssumption(const Loop *L)
Return true if this loop is finite by assumption.
LLVM_ABI const SCEV * getExistingSCEV(Value *V)
Return an existing SCEV for V if there is one, otherwise return nullptr.
LoopDisposition
An enum describing the relationship between a SCEV and a loop.
@ LoopComputable
The SCEV varies predictably with the loop.
@ LoopVariant
The SCEV is loop-variant (unknown).
@ LoopInvariant
The SCEV is loop-invariant.
friend class SCEVCallbackVH
LLVM_ABI bool isKnownMultipleOf(const SCEV *S, uint64_t M, SmallVectorImpl< const SCEVPredicate * > &Assumptions)
Check that S is a multiple of M.
LLVM_ABI const SCEV * getAnyExtendExpr(const SCEV *Op, Type *Ty)
getAnyExtendExpr - Return a SCEV for the given operand extended with unspecified bits out to the give...
LLVM_ABI bool isKnownToBeAPowerOfTwo(const SCEV *S, bool OrZero=false, bool OrNegative=false)
Test if the given expression is known to be a power of 2.
LLVM_ABI std::optional< SCEV::NoWrapFlags > getStrengthenedNoWrapFlagsFromBinOp(const OverflowingBinaryOperator *OBO)
Parse NSW/NUW flags from add/sub/mul IR binary operation Op into SCEV no-wrap flags,...
LLVM_ABI void forgetLcssaPhiWithNewPredecessor(Loop *L, PHINode *V)
Forget LCSSA phi node V of loop L to which a new predecessor was added, such that it may no longer be...
LLVM_ABI bool containsUndefs(const SCEV *S) const
Return true if the SCEV expression contains an undef value.
LLVM_ABI std::optional< MonotonicPredicateType > getMonotonicPredicateType(const SCEVAddRecExpr *LHS, ICmpInst::Predicate Pred)
If, for all loop invariant X, the predicate "LHS `Pred` X" is monotonically increasing or decreasing,...
LLVM_ABI const SCEV * getCouldNotCompute()
LLVM_ABI bool isAvailableAtLoopEntry(const SCEV *S, const Loop *L)
Determine if the SCEV can be evaluated at loop's entry.
BlockDisposition
An enum describing the relationship between a SCEV and a basic block.
@ DominatesBlock
The SCEV dominates the block.
@ ProperlyDominatesBlock
The SCEV properly dominates the block.
@ DoesNotDominateBlock
The SCEV does not dominate the block.
LLVM_ABI const SCEV * getExitCount(const Loop *L, const BasicBlock *ExitingBlock, ExitCountKind Kind=Exact)
Return the number of times the backedge executes before the given exit would be taken; if not exactly...
LLVM_ABI const SCEV * getSignExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth=0)
LLVM_ABI void getPoisonGeneratingValues(SmallPtrSetImpl< const Value * > &Result, const SCEV *S)
Return the set of Values that, if poison, will definitively result in S being poison as well.
LLVM_ABI void forgetLoopDispositions()
Called when the client has changed the disposition of values in this loop.
LLVM_ABI const SCEV * getVScale(Type *Ty)
LLVM_ABI unsigned getSmallConstantTripCount(const Loop *L)
Returns the exact trip count of the loop if we can compute it, and the result is a small constant.
LLVM_ABI bool hasComputableLoopEvolution(const SCEV *S, const Loop *L)
Return true if the given SCEV changes value in a known way in the specified loop.
LLVM_ABI const SCEV * getPointerBase(const SCEV *V)
Transitively follow the chain of pointer-type operands until reaching a SCEV that does not have a sin...
LLVM_ABI const SCEV * getMinMaxExpr(SCEVTypes Kind, SmallVectorImpl< const SCEV * > &Operands)
LLVM_ABI void forgetAllLoops()
LLVM_ABI bool dominates(const SCEV *S, const BasicBlock *BB)
Return true if elements that makes up the given SCEV dominate the specified basic block.
APInt getUnsignedRangeMax(const SCEV *S)
Determine the max of the unsigned range for a particular SCEV.
ExitCountKind
The terms "backedge taken count" and "exit count" are used interchangeably to refer to the number of ...
@ SymbolicMaximum
An expression which provides an upper bound on the exact trip count.
@ ConstantMaximum
A constant which provides an upper bound on the exact trip count.
@ Exact
An expression exactly describing the number of times the backedge has executed when a loop is exited.
LLVM_ABI const SCEV * applyLoopGuards(const SCEV *Expr, const Loop *L)
Try to apply information from loop guards for L to Expr.
LLVM_ABI const SCEV * getMulExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap, unsigned Depth=0)
Get a canonical multiply expression, or something simpler if possible.
LLVM_ABI const SCEVAddRecExpr * convertSCEVToAddRecWithPredicates(const SCEV *S, const Loop *L, SmallVectorImpl< const SCEVPredicate * > &Preds)
Tries to convert the S expression to an AddRec expression, adding additional predicates to Preds as r...
LLVM_ABI const SCEV * getElementSize(Instruction *Inst)
Return the size of an element read or written by Inst.
LLVM_ABI const SCEV * getSizeOfExpr(Type *IntTy, TypeSize Size)
Return an expression for a TypeSize.
LLVM_ABI std::optional< bool > evaluatePredicate(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
Check whether the condition described by Pred, LHS, and RHS is true or false.
LLVM_ABI const SCEV * getUnknown(Value *V)
LLVM_ABI std::optional< std::pair< const SCEV *, SmallVector< const SCEVPredicate *, 3 > > > createAddRecFromPHIWithCasts(const SCEVUnknown *SymbolicPHI)
Checks if SymbolicPHI can be rewritten as an AddRecExpr under some Predicates.
LLVM_ABI const SCEV * getTruncateOrZeroExtend(const SCEV *V, Type *Ty, unsigned Depth=0)
Return a SCEV corresponding to a conversion of the input value to the specified type.
static SCEV::NoWrapFlags maskFlags(SCEV::NoWrapFlags Flags, int Mask)
Convenient NoWrapFlags manipulation that hides enum casts and is visible in the ScalarEvolution name ...
LLVM_ABI std::optional< APInt > computeConstantDifference(const SCEV *LHS, const SCEV *RHS)
Compute LHS - RHS and returns the result as an APInt if it is a constant, and std::nullopt if it isn'...
LLVM_ABI bool properlyDominates(const SCEV *S, const BasicBlock *BB)
Return true if elements that makes up the given SCEV properly dominate the specified basic block.
LLVM_ABI const SCEV * rewriteUsingPredicate(const SCEV *S, const Loop *L, const SCEVPredicate &A)
Re-writes the SCEV according to the Predicates in A.
LLVM_ABI std::pair< const SCEV *, const SCEV * > SplitIntoInitAndPostInc(const Loop *L, const SCEV *S)
Splits SCEV expression S into two SCEVs.
LLVM_ABI bool canReuseInstruction(const SCEV *S, Instruction *I, SmallVectorImpl< Instruction * > &DropPoisonGeneratingInsts)
Check whether it is poison-safe to represent the expression S using the instruction I.
LLVM_ABI bool isKnownPredicateAt(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Instruction *CtxI)
Test if the given expression is known to satisfy the condition described by Pred, LHS,...
LLVM_ABI const SCEV * getPredicatedSymbolicMaxBackedgeTakenCount(const Loop *L, SmallVectorImpl< const SCEVPredicate * > &Predicates)
Similar to getSymbolicMaxBackedgeTakenCount, except it will add a set of SCEV predicates to Predicate...
LLVM_ABI ~ScalarEvolution()
LLVM_ABI const SCEV * getUDivExactExpr(const SCEV *LHS, const SCEV *RHS)
Get a canonical unsigned division expression, or something simpler if possible.
LLVM_ABI void registerUser(const SCEV *User, ArrayRef< const SCEV * > Ops)
Notify this ScalarEvolution that User directly uses SCEVs in Ops.
LLVM_ABI const SCEV * getAddExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap, unsigned Depth=0)
Get a canonical add expression, or something simpler if possible.
LLVM_ABI bool isBasicBlockEntryGuardedByCond(const BasicBlock *BB, CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
Test whether entry to the basic block is protected by a conditional between LHS and RHS.
LLVM_ABI const SCEV * getTruncateOrSignExtend(const SCEV *V, Type *Ty, unsigned Depth=0)
Return a SCEV corresponding to a conversion of the input value to the specified type.
LLVM_ABI bool containsErasedValue(const SCEV *S) const
Return true if the SCEV expression contains a Value that has been optimised out and is now a nullptr.
LLVM_ABI bool isKnownPredicate(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
Test if the given expression is known to satisfy the condition described by Pred, LHS,...
LLVM_ABI bool isKnownViaInduction(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
We'd like to check the predicate on every iteration of the most dominated loop between loops used in ...
const SCEV * getSymbolicMaxBackedgeTakenCount(const Loop *L)
When successful, this returns a SCEV that is greater than or equal to (i.e.
APInt getSignedRangeMax(const SCEV *S)
Determine the max of the signed range for a particular SCEV.
LLVM_ABI void verify() const
LLVMContext & getContext() const
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
iterator erase(const_iterator CI)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
Used to lazily calculate structure layout information for a target machine, based on the DataLayout s...
TypeSize getElementOffset(unsigned Idx) const
TypeSize getSizeInBits() const
Class to represent struct types.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
The instances of the Type class are immutable: once they are created, they are never changed.
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
bool isPointerTy() const
True if this is an instance of PointerType.
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
static LLVM_ABI IntegerType * getInt1Ty(LLVMContext &C)
bool isIntOrPtrTy() const
Return true if this is an integer type or a pointer type.
bool isIntegerTy() const
True if this is an instance of IntegerType.
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
A Use represents the edge between a Value definition and its users.
Value * getOperand(unsigned i) const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
unsigned getValueID() const
Return an ID for the concrete type of this object.
LLVM_ABI void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
LLVM_ABI LLVMContext & getContext() const
All values hold a context through their type.
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
const ParentTy * getParent() const
This class implements an extremely fast bulk output stream that can only output to a stream.
raw_ostream & indent(unsigned NumSpaces)
indent - Insert 'NumSpaces' spaces.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
const APInt & smin(const APInt &A, const APInt &B)
Determine the smaller of two APInts considered to be signed.
const APInt & smax(const APInt &A, const APInt &B)
Determine the larger of two APInts considered to be signed.
const APInt & umin(const APInt &A, const APInt &B)
Determine the smaller of two APInts considered to be unsigned.
LLVM_ABI std::optional< APInt > SolveQuadraticEquationWrap(APInt A, APInt B, APInt C, unsigned RangeWidth)
Let q(n) = An^2 + Bn + C, and BW = bit width of the value range (e.g.
const APInt & umax(const APInt &A, const APInt &B)
Determine the larger of two APInts considered to be unsigned.
LLVM_ABI APInt GreatestCommonDivisor(APInt A, APInt B)
Compute GCD of two unsigned APInt values.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
int getMinValue(MCInstrInfo const &MCII, MCInst const &MCI)
Return the minimum value of an extendable operand.
@ BasicBlock
Various leaf nodes.
LLVM_ABI Function * getDeclarationIfExists(const Module *M, ID id)
Look up the Function declaration of the intrinsic id in the Module M and return it if it exists.
Predicate
Predicate - These are "(BI << 5) | BO" for various predicates.
BinaryOp_match< LHS, RHS, Instruction::AShr > m_AShr(const LHS &L, const RHS &R)
ap_match< APInt > m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
bool match(Val *V, const Pattern &P)
class_match< ConstantInt > m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
IntrinsicID_match m_Intrinsic()
Match intrinsic calls like this: m_Intrinsic<Intrinsic::fabs>(m_Value(X))
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
ExtractValue_match< Ind, Val_t > m_ExtractValue(const Val_t &V)
Match a single index ExtractValue instruction.
bind_ty< WithOverflowInst > m_WithOverflowInst(WithOverflowInst *&I)
Match a with overflow intrinsic, capturing it if we match.
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
brc_match< Cond_t, bind_ty< BasicBlock >, bind_ty< BasicBlock > > m_Br(const Cond_t &C, BasicBlock *&T, BasicBlock *&F)
BinaryOp_match< LHS, RHS, Instruction::SDiv > m_SDiv(const LHS &L, const RHS &R)
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
BinaryOp_match< LHS, RHS, Instruction::LShr > m_LShr(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Shl > m_Shl(const LHS &L, const RHS &R)
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
class_match< BasicBlock > m_BasicBlock()
Match an arbitrary basic block value and ignore it.
match_combine_or< LTy, RTy > m_CombineOr(const LTy &L, const RTy &R)
Combine two pattern matchers matching L || R.
class_match< const SCEVVScale > m_SCEVVScale()
bind_cst_ty m_scev_APInt(const APInt *&C)
Match an SCEV constant and bind it to an APInt.
cst_pred_ty< is_all_ones > m_scev_AllOnes()
Match an integer with all bits set.
SCEVUnaryExpr_match< SCEVZeroExtendExpr, Op0_t > m_scev_ZExt(const Op0_t &Op0)
class_match< const SCEVConstant > m_SCEVConstant()
cst_pred_ty< is_one > m_scev_One()
Match an integer 1.
specificloop_ty m_SpecificLoop(const Loop *L)
SCEVAffineAddRec_match< Op0_t, Op1_t, class_match< const Loop > > m_scev_AffineAddRec(const Op0_t &Op0, const Op1_t &Op1)
SCEVUnaryExpr_match< SCEVSignExtendExpr, Op0_t > m_scev_SExt(const Op0_t &Op0)
cst_pred_ty< is_zero > m_scev_Zero()
Match an integer 0.
bool match(const SCEV *S, const Pattern &P)
SCEVBinaryExpr_match< SCEVUDivExpr, Op0_t, Op1_t > m_scev_UDiv(const Op0_t &Op0, const Op1_t &Op1)
specificscev_ty m_scev_Specific(const SCEV *S)
Match if we have a specific specified SCEV.
SCEVBinaryExpr_match< SCEVMulExpr, Op0_t, Op1_t, SCEV::FlagNUW, true > m_scev_c_NUWMul(const Op0_t &Op0, const Op1_t &Op1)
class_match< const Loop > m_Loop()
bind_ty< const SCEVAddExpr > m_scev_Add(const SCEVAddExpr *&V)
bind_ty< const SCEVUnknown > m_SCEVUnknown(const SCEVUnknown *&V)
SCEVBinaryExpr_match< SCEVMulExpr, Op0_t, Op1_t, SCEV::FlagAnyWrap, true > m_scev_c_Mul(const Op0_t &Op0, const Op1_t &Op1)
class_match< const SCEV > m_SCEV()
initializer< Ty > init(const Ty &Val)
LocationClass< Ty > location(Ty &L)
@ Switch
The "resume-switch" lowering, where there are separate resume and destroy functions that are shared b...
NodeAddr< PhiNode * > Phi
friend class Instruction
Iterator for Instructions in a `BasicBlock.
This is an optimization pass for GlobalISel generic memory operations.
void visitAll(const SCEV *Root, SV &Visitor)
Use SCEVTraversal to visit all nodes in the given expression tree.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
FunctionAddr VTableAddr Value
LLVM_ATTRIBUTE_ALWAYS_INLINE DynamicAPInt gcd(const DynamicAPInt &A, const DynamicAPInt &B)
void stable_sort(R &&Range)
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
SaveAndRestore(T &) -> SaveAndRestore< T >
Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST=nullptr, unsigned DynamicVGPRBlockSize=0)
LLVM_ABI bool canCreatePoison(const Operator *Op, bool ConsiderFlagsAndMetadata=true)
LLVM_ABI bool mustTriggerUB(const Instruction *I, const SmallPtrSetImpl< const Value * > &KnownPoison)
Return true if the given instruction must trigger undefined behavior when I is executed with any oper...
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
LLVM_ABI bool canConstantFoldCallTo(const CallBase *Call, const Function *F)
canConstantFoldCallTo - Return true if its even possible to fold a call to the specified function.
InterleavedRange< Range > interleaved(const Range &R, StringRef Separator=", ", StringRef Prefix="", StringRef Suffix="")
Output range R as a sequence of interleaved elements.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
LLVM_ABI bool verifyFunction(const Function &F, raw_ostream *OS=nullptr)
Check a function for errors, useful for use when debugging a pass.
auto successors(const MachineBasicBlock *BB)
constexpr from_range_t from_range
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
bool set_is_subset(const S1Ty &S1, const S2Ty &S2)
set_is_subset(A, B) - Return true iff A in B
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
constexpr bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
LLVM_ABI Constant * ConstantFoldCompareInstOperands(unsigned Predicate, Constant *LHS, Constant *RHS, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, const Instruction *I=nullptr)
Attempt to constant fold a compare instruction (icmp/fcmp) with the specified operands.
unsigned short computeExpressionSize(ArrayRef< const SCEV * > Args)
auto uninitialized_copy(R &&Src, IterTy Dst)
bool isa_and_nonnull(const Y &Val)
LLVM_ABI ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD)
Parse out a conservative ConstantRange from !range metadata.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
LLVM_ABI Value * simplifyInstruction(Instruction *I, const SimplifyQuery &Q)
See if we can compute a simplified version of this instruction.
LLVM_ABI bool isOverflowIntrinsicNoWrap(const WithOverflowInst *WO, const DominatorTree &DT)
Returns true if the arithmetic part of the WO 's result is used only along the paths control dependen...
DomTreeNodeBase< BasicBlock > DomTreeNode
LLVM_ABI bool matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO, Value *&Start, Value *&Step)
Attempt to match a simple first order recurrence cycle of the form: iv = phi Ty [Start,...
auto dyn_cast_or_null(const Y &Val)
void erase(Container &C, ValueType V)
Wrapper function to remove a value from a container:
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
iterator_range< pointee_iterator< WrappedIteratorT > > make_pointee_range(RangeT &&Range)
auto reverse(ContainerTy &&C)
LLVM_ABI bool isMustProgress(const Loop *L)
Return true if this loop can be assumed to make progress.
LLVM_ABI bool impliesPoison(const Value *ValAssumedPoison, const Value *V)
Return true if V is poison given that ValAssumedPoison is already poison.
LLVM_ABI bool isFinite(const Loop *L)
Return true if this loop can be assumed to run for a finite number of iterations.
LLVM_ABI void computeKnownBits(const Value *V, KnownBits &Known, const DataLayout &DL, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true, unsigned Depth=0)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
LLVM_ABI bool programUndefinedIfPoison(const Instruction *Inst)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool isPointerTy(const Type *T)
FunctionAddr VTableAddr Count
LLVM_ABI ConstantRange getVScaleRange(const Function *F, unsigned BitWidth)
Determine the possible constant range of vscale with the given bit width, based on the vscale_range f...
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
LLVM_ABI bool isKnownNonZero(const Value *V, const SimplifyQuery &Q, unsigned Depth=0)
Return true if the given value is known to be non-zero when defined.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
LLVM_ABI bool propagatesPoison(const Use &PoisonOp)
Return true if PoisonOp's user yields poison or raises UB if its operand PoisonOp is poison.
@ UMin
Unsigned integer min implemented in terms of select(cmp()).
@ Mul
Product of integers.
@ SMax
Signed integer max implemented in terms of select(cmp()).
@ SMin
Signed integer min implemented in terms of select(cmp()).
@ UMax
Unsigned integer max implemented in terms of select(cmp()).
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
DWARFExpression::Operation Op
auto max_element(R &&Range)
Provide wrappers to std::max_element which take ranges instead of having to pass begin/end explicitly...
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
ArrayRef(const T &OneElt) -> ArrayRef< T >
LLVM_ABI unsigned ComputeNumSignBits(const Value *Op, const DataLayout &DL, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true, unsigned Depth=0)
Return the number of times the sign bit of the register is replicated into the other bits.
constexpr unsigned BitWidth
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
LLVM_ABI bool isGuaranteedToTransferExecutionToSuccessor(const Instruction *I)
Return true if this function can prove that the instruction I will always transfer execution to one o...
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
constexpr bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
auto predecessors(const MachineBasicBlock *BB)
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
iterator_range< df_iterator< T > > depth_first(const T &G)
auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
LLVM_ABI bool isGuaranteedNotToBePoison(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Returns true if V cannot be poison, but may be undef.
LLVM_ABI Constant * ConstantFoldInstOperands(const Instruction *I, ArrayRef< Constant * > Ops, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, bool AllowNonDeterministic=true)
ConstantFoldInstOperands - Attempt to constant fold an instruction with the specified operands.
bool SCEVExprContains(const SCEV *Root, PredTy Pred)
Return true if any node in Root satisfies the predicate Pred.
Implement std::hash so that hash_code can be used in STL containers.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
A special type used by analysis passes to provide an address that identifies that particular analysis...
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
bool isNonNegative() const
Returns true if this value is known to be non-negative.
static LLVM_ABI KnownBits ashr(const KnownBits &LHS, const KnownBits &RHS, bool ShAmtNonZero=false, bool Exact=false)
Compute known bits for ashr(LHS, RHS).
unsigned getBitWidth() const
Get the bit width of this value.
static LLVM_ABI KnownBits lshr(const KnownBits &LHS, const KnownBits &RHS, bool ShAmtNonZero=false, bool Exact=false)
Compute known bits for lshr(LHS, RHS).
KnownBits zextOrTrunc(unsigned BitWidth) const
Return known bits for a zero extension or truncation of the value we're tracking.
APInt getMaxValue() const
Return the maximal unsigned value possible given these KnownBits.
APInt getMinValue() const
Return the minimal unsigned value possible given these KnownBits.
bool isNegative() const
Returns true if this value is known to be negative.
static LLVM_ABI KnownBits shl(const KnownBits &LHS, const KnownBits &RHS, bool NUW=false, bool NSW=false, bool ShAmtNonZero=false)
Compute known bits for shl(LHS, RHS).
An object of this class is returned by queries that could not be answered.
LLVM_ABI SCEVCouldNotCompute()
static LLVM_ABI bool classof(const SCEV *S)
Methods for support type inquiry through isa, cast, and dyn_cast:
This class defines a simple visitor class that may be used for various SCEV analysis purposes.
A utility class that uses RAII to save and restore the value of a variable.
Information about the number of loop iterations for which a loop exit's branch condition evaluates to...
LLVM_ABI ExitLimit(const SCEV *E)
Construct either an exact exit limit from a constant, or an unknown one from a SCEVCouldNotCompute.
const SCEV * ExactNotTaken
const SCEV * SymbolicMaxNotTaken
SmallVector< const SCEVPredicate *, 4 > Predicates
A vector of predicate guards for this ExitLimit.
const SCEV * ConstantMaxNotTaken