44#ifndef LLVM_ADT_GENERICUNIFORMITYIMPL_H
45#define LLVM_ADT_GENERICUNIFORMITYIMPL_H
56#define DEBUG_TYPE "uniformity"
91 using BlockT =
typename ContextT::BlockT;
96 using CycleT =
typename CycleInfoT::CycleT;
101 bool empty()
const {
return m_order.empty(); }
102 size_t size()
const {
return m_order.size(); }
111 POIndex[&BB] = m_order.size();
112 m_order.push_back(&BB);
114 <<
"): " << Context.print(&BB) <<
"\n");
116 ReducibleCycleHeaders.insert(&BB);
120 assert(POIndex.count(BB));
121 return POIndex.lookup(BB);
125 return ReducibleCycleHeaders.contains(BB);
132 const ContextT &Context;
142template <
typename>
class DivergencePropagator;
264 using BlockT =
typename ContextT::BlockT;
271 using CycleT =
typename CycleInfoT::CycleT;
321 CachedControlDivDescs;
332 using BlockT =
typename ContextT::BlockT;
336 using UseT =
typename ContextT::UseT;
341 using CycleT =
typename CycleInfoT::CycleT;
345 typename SyncDependenceAnalysisT::DivergenceDescriptor;
349 std::tuple<ConstValueRefT, InstructionT *, const CycleT *>;
388 if (
I.isTerminator()) {
447 void taintAndPushAllDefs(
const BlockT &JoinBlock);
451 void taintAndPushPhiNodes(
const BlockT &JoinBlock);
456 void propagateCycleExitDivergence(
const BlockT &DivExit,
460 void analyzeCycleExitDivergence(
const CycleT &DefCycle);
473 bool isTemporalDivergent(
const BlockT &ObservingBlock,
477template <
typename ImplT>
485 using BlockT =
typename ContextT::BlockT;
491 using CycleT =
typename CycleInfoT::CycleT;
496 typename SyncDependenceAnalysisT::DivergenceDescriptor;
512 std::unique_ptr<DivergenceDescriptorT>
DivDesc;
522 Out <<
"Propagator::BlockLabels {\n";
523 for (
int BlockIdx = (
int)
CyclePOT.size() - 1; BlockIdx >= 0; --BlockIdx) {
526 Out <<
Context.print(
Block) <<
"(" << BlockIdx <<
") : ";
530 Out <<
Context.print(Label) <<
"\n";
542 <<
"\tpushed label: " <<
Context.print(&PushedLabel)
544 <<
"\told label: " <<
Context.print(OldLabel) <<
"\n");
547 if (OldLabel == &PushedLabel)
550 if (OldLabel != &SuccBlock) {
551 auto SuccIdx =
CyclePOT.getIndex(&SuccBlock);
580 DivDesc->CycleDivBlocks.insert(&ExitBlock);
592 DivDesc->JoinDivBlocks.insert(&SuccBlock);
615 if (
C->isReducible())
617 while (
const CycleT *
P =
C->getParentCycle()) {
618 if (
P->isReducible())
628 if (DivTermCycle && !DivTermCycle->contains(SuccBlock)) {
632 DivDesc->CycleDivBlocks.insert(SuccBlock);
634 <<
Context.print(SuccBlock) <<
"\n");
653 (!IrreducibleAncestor || !IrreducibleAncestor->contains(
Block)))
659 if (BlockIdx == DivTermIdx) {
665 << BlockIdx <<
"\n");
688 const auto *BlockCycle =
CI.getCycle(
Block);
694 if (
const auto *BlockCycle = getReducibleParent(
Block)) {
696 BlockCycle->getExitBlocks(BlockCycleExits);
697 for (
auto *BlockCycleExit : BlockCycleExits)
712 if (
Cycle->isReducible()) {
718 Cycle->getExitBlocks(Exits);
719 auto *Header =
Cycle->getHeader();
721 for (
const auto *Exit : Exits) {
724 DivDesc->CycleDivBlocks.insert(Exit);
735template <
typename ContextT>
738 : CyclePO(Context), DT(DT), CI(CI) {
742template <
typename ContextT>
747 return EmptyDivergenceDesc;
751 auto ItCached = CachedControlDivDescs.find(DivTermBlock);
752 if (ItCached != CachedControlDivDescs.end())
753 return *ItCached->second;
763 for (
const auto *BB : Blocks) {
764 Out << LS << CI.getSSAContext().print(BB);
771 dbgs() <<
"\nResult (" << CI.getSSAContext().print(DivTermBlock)
772 <<
"):\n JoinDivBlocks: " << printBlockSet(DivDesc->JoinDivBlocks)
773 <<
" CycleDivBlocks: " << printBlockSet(DivDesc->CycleDivBlocks)
778 CachedControlDivDescs.try_emplace(DivTermBlock, std::move(DivDesc));
779 assert(ItInserted.second);
780 return *ItInserted.first->second;
783template <
typename ContextT>
789 if (
I.isTerminator()) {
793 <<
Context.print(
I.getParent()) <<
"\n");
803template <
typename ContextT>
813template <
typename ContextT>
816 UniformOverrides.insert(&Instr);
832template <
typename ContextT>
833void GenericUniformityAnalysisImpl<ContextT>::analyzeCycleExitDivergence(
834 const CycleT &DefCycle) {
836 DefCycle.getExitBlocks(Exits);
837 for (
auto *Exit : Exits) {
838 for (
auto &Phi : Exit->phis()) {
839 if (usesValueFromCycle(Phi, DefCycle)) {
845 for (
auto *BB : DefCycle.blocks()) {
847 [&](BlockT *Exit) {
return DT.dominates(BB, Exit); }))
849 for (
auto &
II : *BB) {
850 propagateTemporalDivergence(
II, DefCycle);
855template <
typename ContextT>
856void GenericUniformityAnalysisImpl<ContextT>::propagateCycleExitDivergence(
857 const BlockT &DivExit,
const CycleT &InnerDivCycle) {
860 auto *DivCycle = &InnerDivCycle;
861 auto *OuterDivCycle = DivCycle;
862 auto *ExitLevelCycle = CI.getCycle(&DivExit);
863 const unsigned CycleExitDepth =
864 ExitLevelCycle ? ExitLevelCycle->getDepth() : 0;
867 while (DivCycle && DivCycle->getDepth() > CycleExitDepth) {
869 <<
Context.print(DivCycle->getHeader()) <<
"\n");
870 OuterDivCycle = DivCycle;
871 DivCycle = DivCycle->getParentCycle();
874 <<
Context.print(OuterDivCycle->getHeader()) <<
"\n");
876 if (!DivergentExitCycles.insert(OuterDivCycle).second)
881 for (
const auto *
C : AssumedDivergent) {
882 if (
C->contains(OuterDivCycle))
886 analyzeCycleExitDivergence(*OuterDivCycle);
889template <
typename ContextT>
890void GenericUniformityAnalysisImpl<ContextT>::taintAndPushAllDefs(
893 for (
const auto &
I :
instrs(BB)) {
897 if (
I.isTerminator())
905template <
typename ContextT>
906void GenericUniformityAnalysisImpl<ContextT>::taintAndPushPhiNodes(
907 const BlockT &JoinBlock) {
910 for (
const auto &Phi : JoinBlock.phis()) {
918 if (ContextT::isConstantOrUndefValuePhi(Phi))
927template <
typename CycleT>
931 [Candidate](CycleT *
C) {
return C->contains(Candidate); }))
933 Cycles.push_back(Candidate);
942template <
typename CycleT,
typename BlockT>
944 const BlockT *DivTermBlock,
945 const BlockT *JoinBlock) {
949 if (
Cycle->contains(DivTermBlock))
952 const auto *OriginalCycle =
Cycle;
953 const auto *Parent =
Cycle->getParentCycle();
954 while (Parent && !Parent->contains(DivTermBlock)) {
956 Parent =
Cycle->getParentCycle();
965 if (
Cycle->isReducible()) {
970 LLVM_DEBUG(
dbgs() <<
"cycle made divergent by external branch\n");
978template <
typename ContextT,
typename CycleT,
typename BlockT,
979 typename DominatorTreeT>
982 const BlockT *JoinBlock,
const DominatorTreeT &DT,
985 <<
" for internal branch " << Context.print(DivTermBlock)
987 if (DT.properlyDominates(DivTermBlock, JoinBlock))
992 while (
Cycle && !
Cycle->contains(DivTermBlock)) {
998 if (DT.properlyDominates(
Cycle->getHeader(), JoinBlock))
1002 <<
" does not dominate join\n");
1004 const auto *Parent =
Cycle->getParentCycle();
1005 while (Parent && !DT.properlyDominates(Parent->getHeader(), JoinBlock)) {
1006 LLVM_DEBUG(
dbgs() <<
" header " << Context.print(Parent->getHeader())
1007 <<
" does not dominate join\n");
1009 Parent = Parent->getParentCycle();
1012 LLVM_DEBUG(
dbgs() <<
" cycle made divergent by internal branch\n");
1016template <
typename ContextT,
typename CycleT,
typename BlockT,
1017 typename DominatorTreeT>
1018static const CycleT *
1020 const BlockT *JoinBlock,
const DominatorTreeT &DT,
1021 ContextT &Context) {
1037template <
typename ContextT>
1038bool GenericUniformityAnalysisImpl<ContextT>::isTemporalDivergent(
1039 const BlockT &ObservingBlock,
const InstructionT &Def)
const {
1040 const BlockT *DefBlock = Def.getParent();
1041 for (
const CycleT *
Cycle = CI.getCycle(DefBlock);
1044 if (DivergentExitCycles.contains(
Cycle)) {
1051template <
typename ContextT>
1054 const auto *DivTermBlock = Term.getParent();
1060 if (!DT.isReachableFromEntry(DivTermBlock))
1063 const auto &DivDesc = SDA.getJoinBlocks(DivTermBlock);
1067 for (
const auto *JoinBlock : DivDesc.JoinDivBlocks) {
1068 const auto *
Cycle =
CI.getCycle(JoinBlock);
1077 taintAndPushPhiNodes(*JoinBlock);
1083 return A->getDepth() >
B->getDepth();
1091 for (
auto *
C : DivCycles) {
1095 for (
const BlockT *BB :
C->blocks()) {
1096 taintAndPushAllDefs(*BB);
1100 const auto *BranchCycle =
CI.getCycle(DivTermBlock);
1101 assert(DivDesc.CycleDivBlocks.empty() || BranchCycle);
1102 for (
const auto *DivExitBlock : DivDesc.CycleDivBlocks) {
1103 propagateCycleExitDivergence(*DivExitBlock, *BranchCycle);
1107template <
typename ContextT>
1111 for (
const auto DivVal : DivValuesCopy) {
1124 if (
I->isTerminator()) {
1135template <
typename ContextT>
1142template <
typename ContextT>
1145 return UniformOverrides.contains(&Instr);
1148template <
typename ContextT>
1152 DA.reset(
new ImplT{DT, CI,
TTI});
1155template <
typename ContextT>
1157 bool haveDivergentArgs =
false;
1162 constexpr bool IsMIR = std::is_same<InstructionT, MachineInstr>::value;
1163 std::string NewLine = IsMIR ?
"" :
"\n";
1169 DivergentExitCycles.empty()) {
1170 OS <<
"ALL VALUES UNIFORM\n";
1177 if (!haveDivergentArgs) {
1178 OS <<
"DIVERGENT ARGUMENTS:\n";
1179 haveDivergentArgs =
true;
1181 OS <<
" DIVERGENT: " <<
Context.print(entry) <<
'\n';
1185 if (!AssumedDivergent.empty()) {
1186 OS <<
"CYCLES ASSUMED DIVERGENT:\n";
1187 for (
const CycleT *cycle : AssumedDivergent) {
1188 OS <<
" " << cycle->print(
Context) <<
'\n';
1192 if (!DivergentExitCycles.empty()) {
1193 OS <<
"CYCLES WITH DIVERGENT EXIT:\n";
1194 for (
const CycleT *cycle : DivergentExitCycles) {
1195 OS <<
" " << cycle->print(
Context) <<
'\n';
1200 OS <<
"\nTEMPORAL DIVERGENCE LIST:\n";
1203 OS <<
"Value :" <<
Context.print(Val) << NewLine
1204 <<
"Used by :" <<
Context.print(UseInst) << NewLine
1205 <<
"Outside cycle :" <<
Cycle->print(
Context) <<
"\n\n";
1212 OS <<
"DEFINITIONS\n";
1215 for (
auto value : defs) {
1217 OS <<
" DIVERGENT: ";
1220 OS <<
Context.print(value) << NewLine;
1223 OS <<
"TERMINATORS\n";
1227 for (
auto *
T : terms) {
1228 if (divergentTerminators)
1229 OS <<
" DIVERGENT: ";
1235 OS <<
"END BLOCK\n";
1239template <
typename ContextT>
1243 return make_range(DA->TemporalDivergenceList.begin(),
1244 DA->TemporalDivergenceList.end());
1247template <
typename ContextT>
1249 return DA->hasDivergence();
1252template <
typename ContextT>
1253const typename ContextT::FunctionT &
1255 return DA->getFunction();
1259template <
typename ContextT>
1261 return DA->isDivergent(V);
1264template <
typename ContextT>
1266 return DA->isDivergent(*
I);
1269template <
typename ContextT>
1271 return DA->isDivergentUse(U);
1274template <
typename ContextT>
1276 return DA->hasDivergentTerminator(
B);
1280template <
typename ContextT>
1285template <
typename ContextT>
1286void llvm::ModifiedPostOrder<ContextT>::computeStackPO(
1290 while (!Stack.empty()) {
1291 auto *NextBB = Stack.back();
1292 if (Finalized.
count(NextBB)) {
1296 LLVM_DEBUG(
dbgs() <<
" visiting " << CI.getSSAContext().print(NextBB)
1298 auto *NestedCycle = CI.getCycle(NextBB);
1301 while (NestedCycle->getParentCycle() !=
Cycle)
1302 NestedCycle = NestedCycle->getParentCycle();
1305 NestedCycle->getExitBlocks(NestedExits);
1306 bool PushedNodes =
false;
1307 for (
auto *NestedExitBB : NestedExits) {
1309 << CI.getSSAContext().print(NestedExitBB) <<
"\n");
1312 if (Finalized.
count(NestedExitBB))
1315 Stack.push_back(NestedExitBB);
1317 << CI.getSSAContext().print(NestedExitBB) <<
"\n");
1322 computeCyclePO(CI, NestedCycle, Finalized);
1329 bool PushedNodes =
false;
1332 << CI.getSSAContext().print(SuccBB) <<
"\n");
1335 if (Finalized.
count(SuccBB))
1338 Stack.push_back(SuccBB);
1339 LLVM_DEBUG(
dbgs() <<
" pushed succ: " << CI.getSSAContext().print(SuccBB)
1345 << CI.getSSAContext().print(NextBB) <<
"\n");
1347 Finalized.
insert(NextBB);
1348 appendBlock(*NextBB);
1354template <
typename ContextT>
1355void ModifiedPostOrder<ContextT>::computeCyclePO(
1356 const CycleInfoT &CI,
const CycleT *
Cycle,
1360 auto *CycleHeader =
Cycle->getHeader();
1363 << CI.getSSAContext().print(CycleHeader) <<
"\n");
1364 assert(!Finalized.count(CycleHeader));
1365 Finalized.insert(CycleHeader);
1369 << CI.getSSAContext().print(CycleHeader) <<
"\n");
1370 appendBlock(*CycleHeader,
Cycle->isReducible());
1374 LLVM_DEBUG(
dbgs() <<
" examine succ: " << CI.getSSAContext().print(BB)
1376 if (!
Cycle->contains(BB))
1378 if (BB == CycleHeader)
1380 if (!Finalized.count(BB)) {
1381 LLVM_DEBUG(
dbgs() <<
" pushed succ: " << CI.getSSAContext().print(BB)
1383 Stack.push_back(BB);
1388 computeStackPO(Stack, CI,
Cycle, Finalized);
1394template <
typename ContextT>
1398 auto *
F = CI.getFunction();
1400 Stack.push_back(&
F->front());
1401 computeStackPO(Stack, CI,
nullptr, Finalized);
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
This file defines the DenseSet and SmallDenseSet classes.
uint64_t IntrinsicInst * II
This file defines the SmallPtrSet class.
This file defines the SparseBitVector class.
unify loop Fixup each natural loop to have a single exit block
Implements a dense probed hash-table based set.
Compute divergence starting with a divergent branch.
typename SyncDependenceAnalysisT::BlockLabelMap BlockLabelMapT
const ModifiedPO & CyclePOT
GenericSyncDependenceAnalysis< ContextT > SyncDependenceAnalysisT
typename ContextT::DominatorTreeT DominatorTreeT
bool computeJoin(const BlockT &SuccBlock, const BlockT &PushedLabel)
const BlockT & DivTermBlock
std::unique_ptr< DivergenceDescriptorT > DivDesc
void printDefs(raw_ostream &Out)
typename ContextT::FunctionT FunctionT
GenericCycleInfo< ContextT > CycleInfoT
const DominatorTreeT & DT
ModifiedPostOrder< ContextT > ModifiedPO
std::unique_ptr< DivergenceDescriptorT > computeJoinPoints()
BlockLabelMapT & BlockLabels
SparseBitVector FreshLabels
bool visitCycleExitEdge(const BlockT &ExitBlock, const BlockT &Label)
typename ContextT::ValueRefT ValueRefT
typename ContextT::BlockT BlockT
typename CycleInfoT::CycleT CycleT
DivergencePropagator(const ModifiedPO &CyclePOT, const DominatorTreeT &DT, const CycleInfoT &CI, const BlockT &DivTermBlock)
bool visitEdge(const BlockT &SuccBlock, const BlockT &Label)
typename SyncDependenceAnalysisT::DivergenceDescriptor DivergenceDescriptorT
Cycle information for a function.
bool contains(const BlockT *Block) const
Return whether Block is contained in the cycle.
const GenericCycle * getParentCycle() const
Locate join blocks for disjoint paths starting at a divergent branch.
GenericSyncDependenceAnalysis(const ContextT &Context, const DominatorTreeT &DT, const CycleInfoT &CI)
ModifiedPostOrder< ContextT > ModifiedPO
DivergencePropagator< ContextT > DivergencePropagatorT
DenseMap< const BlockT *, const BlockT * > BlockLabelMap
SmallPtrSet< const BlockT *, 4 > ConstBlockSet
typename ContextT::DominatorTreeT DominatorTreeT
GenericCycleInfo< ContextT > CycleInfoT
typename ContextT::FunctionT FunctionT
typename ContextT::InstructionT InstructionT
typename ContextT::BlockT BlockT
typename ContextT::ValueRefT ValueRefT
typename CycleInfoT::CycleT CycleT
const DivergenceDescriptor & getJoinBlocks(const BlockT *DivTermBlock)
Computes divergent join points and cycle exits caused by branch divergence in Term.
A helper class to return the specified delimiter string after the first invocation of operator String...
Construct a specially modified post-order traversal of cycles.
typename ContextT::FunctionT FunctionT
const BlockT * operator[](size_t idx) const
typename CycleInfoT::CycleT CycleT
bool isReducibleCycleHeader(const BlockT *BB) const
ModifiedPostOrder(const ContextT &C)
unsigned count(BlockT *BB) const
void compute(const CycleInfoT &CI)
Generically compute the modified post order.
GenericCycleInfo< ContextT > CycleInfoT
void appendBlock(const BlockT &BB, bool isReducibleCycleHeader=false)
unsigned getIndex(const BlockT *BB) const
typename std::vector< BlockT * >::const_iterator const_iterator
typename ContextT::DominatorTreeT DominatorTreeT
typename ContextT::BlockT BlockT
Simple wrapper around std::function<void(raw_ostream&)>.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
A range adaptor for a pair of iterators.
This class implements an extremely fast bulk output stream that can only output to a stream.
@ C
The default llvm calling convention, compatible with C.
This is an optimization pass for GlobalISel generic memory operations.
static const CycleT * getIntDivCycle(const CycleT *Cycle, const BlockT *DivTermBlock, const BlockT *JoinBlock, const DominatorTreeT &DT, ContextT &Context)
Return the outermost cycle made divergent by branch inside it.
static const CycleT * getExtDivCycle(const CycleT *Cycle, const BlockT *DivTermBlock, const BlockT *JoinBlock)
Return the outermost cycle made divergent by branch outside it.
auto successors(const MachineBasicBlock *BB)
static bool insertIfNotContained(SmallVector< CycleT * > &Cycles, CycleT *Candidate)
Add Candidate to Cycles if it is not already contained in Cycles.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
void sort(IteratorTy Start, IteratorTy End)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
auto succ_size(const MachineBasicBlock *BB)
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
auto instrs(const MachineBasicBlock &BB)
static const CycleT * getOutermostDivergentCycle(const CycleT *Cycle, const BlockT *DivTermBlock, const BlockT *JoinBlock, const DominatorTreeT &DT, ContextT &Context)
Information discovered by the sync dependence analysis for each divergent branch.
ConstBlockSet CycleDivBlocks
ConstBlockSet JoinDivBlocks
BlockLabelMap BlockLabels