Thanks to visit codestin.com
Credit goes to llvm.org

LLVM 22.0.0git
GenericUniformityImpl.h
Go to the documentation of this file.
1//===- GenericUniformityImpl.h -----------------------*- C++ -*------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This template implementation resides in a separate file so that it
10// does not get injected into every .cpp file that includes the
11// generic header.
12//
13// DO NOT INCLUDE THIS FILE WHEN MERELY USING UNIFORMITYINFO.
14//
15// This file should only be included by files that implement a
16// specialization of the relvant templates. Currently these are:
17// - UniformityAnalysis.cpp
18//
19// Note: The DEBUG_TYPE macro should be defined before using this
20// file so that any use of LLVM_DEBUG is associated with the
21// including file rather than this file.
22//
23//===----------------------------------------------------------------------===//
24///
25/// \file
26/// \brief Implementation of uniformity analysis.
27///
28/// The algorithm is a fixed point iteration that starts with the assumption
29/// that all control flow and all values are uniform. Starting from sources of
30/// divergence (whose discovery must be implemented by a CFG- or even
31/// target-specific derived class), divergence of values is propagated from
32/// definition to uses in a straight-forward way. The main complexity lies in
33/// the propagation of the impact of divergent control flow on the divergence of
34/// values (sync dependencies).
35///
36/// NOTE: In general, no interface exists for a transform to update
37/// (Machine)UniformityInfo. Additionally, (Machine)CycleAnalysis is a
38/// transitive dependence, but it also does not provide an interface for
39/// updating itself. Given that, transforms should not preserve uniformity in
40/// their getAnalysisUsage() callback.
41///
42//===----------------------------------------------------------------------===//
43
44#ifndef LLVM_ADT_GENERICUNIFORMITYIMPL_H
45#define LLVM_ADT_GENERICUNIFORMITYIMPL_H
46
48
49#include "llvm/ADT/DenseSet.h"
50#include "llvm/ADT/STLExtras.h"
55
56#define DEBUG_TYPE "uniformity"
57
58namespace llvm {
59
60// Forward decl from llvm/CodeGen/MachineInstr.h
61class MachineInstr;
62
63/// Construct a specially modified post-order traversal of cycles.
64///
65/// The ModifiedPO is contructed using a virtually modified CFG as follows:
66///
67/// 1. The successors of pre-entry nodes (predecessors of an cycle
68/// entry that are outside the cycle) are replaced by the
69/// successors of the successors of the header.
70/// 2. Successors of the cycle header are replaced by the exit blocks
71/// of the cycle.
72///
73/// Effectively, we produce a depth-first numbering with the following
74/// properties:
75///
76/// 1. Nodes after a cycle are numbered earlier than the cycle header.
77/// 2. The header is numbered earlier than the nodes in the cycle.
78/// 3. The numbering of the nodes within the cycle forms an interval
79/// starting with the header.
80///
81/// Effectively, the virtual modification arranges the nodes in a
82/// cycle as a DAG with the header as the sole leaf, and successors of
83/// the header as the roots. A reverse traversal of this numbering has
84/// the following invariant on the unmodified original CFG:
85///
86/// Each node is visited after all its predecessors, except if that
87/// predecessor is the cycle header.
88///
89template <typename ContextT> class ModifiedPostOrder {
90public:
91 using BlockT = typename ContextT::BlockT;
92 using FunctionT = typename ContextT::FunctionT;
93 using DominatorTreeT = typename ContextT::DominatorTreeT;
94
96 using CycleT = typename CycleInfoT::CycleT;
97 using const_iterator = typename std::vector<BlockT *>::const_iterator;
98
99 ModifiedPostOrder(const ContextT &C) : Context(C) {}
100
101 bool empty() const { return m_order.empty(); }
102 size_t size() const { return m_order.size(); }
103
104 void clear() { m_order.clear(); }
105 void compute(const CycleInfoT &CI);
106
107 unsigned count(BlockT *BB) const { return POIndex.count(BB); }
108 const BlockT *operator[](size_t idx) const { return m_order[idx]; }
109
110 void appendBlock(const BlockT &BB, bool isReducibleCycleHeader = false) {
111 POIndex[&BB] = m_order.size();
112 m_order.push_back(&BB);
113 LLVM_DEBUG(dbgs() << "ModifiedPO(" << POIndex[&BB]
114 << "): " << Context.print(&BB) << "\n");
116 ReducibleCycleHeaders.insert(&BB);
117 }
118
119 unsigned getIndex(const BlockT *BB) const {
120 assert(POIndex.count(BB));
121 return POIndex.lookup(BB);
122 }
123
124 bool isReducibleCycleHeader(const BlockT *BB) const {
125 return ReducibleCycleHeaders.contains(BB);
126 }
127
128private:
131 SmallPtrSet<const BlockT *, 32> ReducibleCycleHeaders;
132 const ContextT &Context;
133
134 void computeCyclePO(const CycleInfoT &CI, const CycleT *Cycle,
136
137 void computeStackPO(SmallVectorImpl<const BlockT *> &Stack,
138 const CycleInfoT &CI, const CycleT *Cycle,
140};
141
142template <typename> class DivergencePropagator;
143
144/// \class GenericSyncDependenceAnalysis
145///
146/// \brief Locate join blocks for disjoint paths starting at a divergent branch.
147///
148/// An analysis per divergent branch that returns the set of basic
149/// blocks whose phi nodes become divergent due to divergent control.
150/// These are the blocks that are reachable by two disjoint paths from
151/// the branch, or cycle exits reachable along a path that is disjoint
152/// from a path to the cycle latch.
153
154// --- Above line is not a doxygen comment; intentionally left blank ---
155//
156// Originally implemented in SyncDependenceAnalysis.cpp for DivergenceAnalysis.
157//
158// The SyncDependenceAnalysis is used in the UniformityAnalysis to model
159// control-induced divergence in phi nodes.
160//
161// -- Reference --
162// The algorithm is an extension of Section 5 of
163//
164// An abstract interpretation for SPMD divergence
165// on reducible control flow graphs.
166// Julian Rosemann, Simon Moll and Sebastian Hack
167// POPL '21
168//
169//
170// -- Sync dependence --
171// Sync dependence characterizes the control flow aspect of the
172// propagation of branch divergence. For example,
173//
174// %cond = icmp slt i32 %tid, 10
175// br i1 %cond, label %then, label %else
176// then:
177// br label %merge
178// else:
179// br label %merge
180// merge:
181// %a = phi i32 [ 0, %then ], [ 1, %else ]
182//
183// Suppose %tid holds the thread ID. Although %a is not data dependent on %tid
184// because %tid is not on its use-def chains, %a is sync dependent on %tid
185// because the branch "br i1 %cond" depends on %tid and affects which value %a
186// is assigned to.
187//
188//
189// -- Reduction to SSA construction --
190// There are two disjoint paths from A to X, if a certain variant of SSA
191// construction places a phi node in X under the following set-up scheme.
192//
193// This variant of SSA construction ignores incoming undef values.
194// That is paths from the entry without a definition do not result in
195// phi nodes.
196//
197// entry
198// / \
199// A \
200// / \ Y
201// B C /
202// \ / \ /
203// D E
204// \ /
205// F
206//
207// Assume that A contains a divergent branch. We are interested
208// in the set of all blocks where each block is reachable from A
209// via two disjoint paths. This would be the set {D, F} in this
210// case.
211// To generally reduce this query to SSA construction we introduce
212// a virtual variable x and assign to x different values in each
213// successor block of A.
214//
215// entry
216// / \
217// A \
218// / \ Y
219// x = 0 x = 1 /
220// \ / \ /
221// D E
222// \ /
223// F
224//
225// Our flavor of SSA construction for x will construct the following
226//
227// entry
228// / \
229// A \
230// / \ Y
231// x0 = 0 x1 = 1 /
232// \ / \ /
233// x2 = phi E
234// \ /
235// x3 = phi
236//
237// The blocks D and F contain phi nodes and are thus each reachable
238// by two disjoins paths from A.
239//
240// -- Remarks --
241// * In case of cycle exits we need to check for temporal divergence.
242// To this end, we check whether the definition of x differs between the
243// cycle exit and the cycle header (_after_ SSA construction).
244//
245// * In the presence of irreducible control flow, the fixed point is
246// reached only after multiple iterations. This is because labels
247// reaching the header of a cycle must be repropagated through the
248// cycle. This is true even in a reducible cycle, since the labels
249// may have been produced by a nested irreducible cycle.
250//
251// * Note that SyncDependenceAnalysis is not concerned with the points
252// of convergence in an irreducible cycle. It's only purpose is to
253// identify join blocks. The "diverged entry" criterion is
254// separately applied on join blocks to determine if an entire
255// irreducible cycle is assumed to be divergent.
256//
257// * Relevant related work:
258// A simple algorithm for global data flow analysis problems.
259// Matthew S. Hecht and Jeffrey D. Ullman.
260// SIAM Journal on Computing, 4(4):519–532, December 1975.
261//
262template <typename ContextT> class GenericSyncDependenceAnalysis {
263public:
264 using BlockT = typename ContextT::BlockT;
265 using DominatorTreeT = typename ContextT::DominatorTreeT;
266 using FunctionT = typename ContextT::FunctionT;
267 using ValueRefT = typename ContextT::ValueRefT;
268 using InstructionT = typename ContextT::InstructionT;
269
271 using CycleT = typename CycleInfoT::CycleT;
272
275
276 // * if BlockLabels[B] == C then C is the dominating definition at
277 // block B
278 // * if BlockLabels[B] == nullptr then we haven't seen B yet
279 // * if BlockLabels[B] == B then:
280 // - B is a join point of disjoint paths from X, or,
281 // - B is an immediate successor of X (initial value), or,
282 // - B is X
284
285 /// Information discovered by the sync dependence analysis for each
286 /// divergent branch.
288 // Join points of diverged paths.
290 // Divergent cycle exits
292 // Labels assigned to blocks on diverged paths.
294 };
295
297
298 GenericSyncDependenceAnalysis(const ContextT &Context,
299 const DominatorTreeT &DT, const CycleInfoT &CI);
300
301 /// \brief Computes divergent join points and cycle exits caused by branch
302 /// divergence in \p Term.
303 ///
304 /// This returns a pair of sets:
305 /// * The set of blocks which are reachable by disjoint paths from
306 /// \p Term.
307 /// * The set also contains cycle exits if there two disjoint paths:
308 /// one from \p Term to the cycle exit and another from \p Term to
309 /// the cycle header.
310 const DivergenceDescriptor &getJoinBlocks(const BlockT *DivTermBlock);
311
312private:
313 static inline DivergenceDescriptor EmptyDivergenceDesc;
314
315 ModifiedPO CyclePO;
316
317 const DominatorTreeT &DT;
318 const CycleInfoT &CI;
319
321 CachedControlDivDescs;
322};
323
324/// \brief Analysis that identifies uniform values in a data-parallel
325/// execution.
326///
327/// This analysis propagates divergence in a data-parallel context
328/// from sources of divergence to all users. It can be instantiated
329/// for an IR that provides a suitable SSAContext.
330template <typename ContextT> class GenericUniformityAnalysisImpl {
331public:
332 using BlockT = typename ContextT::BlockT;
333 using FunctionT = typename ContextT::FunctionT;
334 using ValueRefT = typename ContextT::ValueRefT;
335 using ConstValueRefT = typename ContextT::ConstValueRefT;
336 using UseT = typename ContextT::UseT;
337 using InstructionT = typename ContextT::InstructionT;
338 using DominatorTreeT = typename ContextT::DominatorTreeT;
339
341 using CycleT = typename CycleInfoT::CycleT;
342
345 typename SyncDependenceAnalysisT::DivergenceDescriptor;
346 using BlockLabelMapT = typename SyncDependenceAnalysisT::BlockLabelMap;
347
349 std::tuple<ConstValueRefT, InstructionT *, const CycleT *>;
350
353 : Context(CI.getSSAContext()), F(*Context.getFunction()), CI(CI),
354 TTI(TTI), DT(DT), SDA(Context, DT, CI) {}
355
357
358 const FunctionT &getFunction() const { return F; }
359
360 /// \brief Mark \p UniVal as a value that is always uniform.
361 void addUniformOverride(const InstructionT &Instr);
362
363 /// \brief Examine \p I for divergent outputs and add to the worklist.
364 void markDivergent(const InstructionT &I);
365
366 /// \brief Mark \p DivVal as a divergent value.
367 /// \returns Whether the tracked divergence state of \p DivVal changed.
368 bool markDivergent(ConstValueRefT DivVal);
369
370 /// \brief Mark outputs of \p Instr as divergent.
371 /// \returns Whether the tracked divergence state of any output has changed.
372 bool markDefsDivergent(const InstructionT &Instr);
373
374 /// \brief Propagate divergence to all instructions in the region.
375 /// Divergence is seeded by calls to \p markDivergent.
376 void compute();
377
378 /// \brief Whether any value was marked or analyzed to be divergent.
379 bool hasDivergence() const { return !DivergentValues.empty(); }
380
381 /// \brief Whether \p Val will always return a uniform value regardless of its
382 /// operands
383 bool isAlwaysUniform(const InstructionT &Instr) const;
384
385 bool hasDivergentDefs(const InstructionT &I) const;
386
387 bool isDivergent(const InstructionT &I) const {
388 if (I.isTerminator()) {
389 return DivergentTermBlocks.contains(I.getParent());
390 }
391 return hasDivergentDefs(I);
392 };
393
394 /// \brief Whether \p Val is divergent at its definition.
395 bool isDivergent(ConstValueRefT V) const { return DivergentValues.count(V); }
396
397 bool isDivergentUse(const UseT &U) const;
398
399 bool hasDivergentTerminator(const BlockT &B) const {
400 return DivergentTermBlocks.contains(&B);
401 }
402
403 void print(raw_ostream &out) const;
404
406
408 const CycleT *);
409
410protected:
411 const ContextT &Context;
412 const FunctionT &F;
414 const TargetTransformInfo *TTI = nullptr;
415
416 // Detected/marked divergent values.
419
420 // Internal worklist for divergence propagation.
421 std::vector<const InstructionT *> Worklist;
422
423 /// \brief Mark \p Term as divergent and push all Instructions that become
424 /// divergent as a result on the worklist.
425 void analyzeControlDivergence(const InstructionT &Term);
426
427private:
428 const DominatorTreeT &DT;
429
430 // Recognized cycles with divergent exits.
431 SmallPtrSet<const CycleT *, 16> DivergentExitCycles;
432
433 // Cycles assumed to be divergent.
434 //
435 // We don't use a set here because every insertion needs an explicit
436 // traversal of all existing members.
437 SmallVector<const CycleT *> AssumedDivergent;
438
439 // The SDA links divergent branches to divergent control-flow joins.
441
442 // Set of known-uniform values.
444
445 /// \brief Mark all nodes in \p JoinBlock as divergent and push them on
446 /// the worklist.
447 void taintAndPushAllDefs(const BlockT &JoinBlock);
448
449 /// \brief Mark all phi nodes in \p JoinBlock as divergent and push them on
450 /// the worklist.
451 void taintAndPushPhiNodes(const BlockT &JoinBlock);
452
453 /// \brief Identify all Instructions that become divergent because \p DivExit
454 /// is a divergent cycle exit of \p DivCycle. Mark those instructions as
455 /// divergent and push them on the worklist.
456 void propagateCycleExitDivergence(const BlockT &DivExit,
457 const CycleT &DivCycle);
458
459 /// Mark as divergent all external uses of values defined in \p DefCycle.
460 void analyzeCycleExitDivergence(const CycleT &DefCycle);
461
462 /// \brief Mark as divergent all uses of \p I that are outside \p DefCycle.
463 void propagateTemporalDivergence(const InstructionT &I,
464 const CycleT &DefCycle);
465
466 /// \brief Push all users of \p Val (in the region) to the worklist.
467 void pushUsers(const InstructionT &I);
468 void pushUsers(ConstValueRefT V);
469
470 bool usesValueFromCycle(const InstructionT &I, const CycleT &DefCycle) const;
471
472 /// \brief Whether \p Def is divergent when read in \p ObservingBlock.
473 bool isTemporalDivergent(const BlockT &ObservingBlock,
474 const InstructionT &Def) const;
475};
476
477template <typename ImplT>
479 delete Impl;
480}
481
482/// Compute divergence starting with a divergent branch.
483template <typename ContextT> class DivergencePropagator {
484public:
485 using BlockT = typename ContextT::BlockT;
486 using DominatorTreeT = typename ContextT::DominatorTreeT;
487 using FunctionT = typename ContextT::FunctionT;
488 using ValueRefT = typename ContextT::ValueRefT;
489
491 using CycleT = typename CycleInfoT::CycleT;
492
496 typename SyncDependenceAnalysisT::DivergenceDescriptor;
497 using BlockLabelMapT = typename SyncDependenceAnalysisT::BlockLabelMap;
498
503 const ContextT &Context;
504
505 // Track blocks that receive a new label. Every time we relabel a
506 // cycle header, we another pass over the modified post-order in
507 // order to propagate the header label. The bit vector also allows
508 // us to skip labels that have not changed.
510
511 // divergent join and cycle exit descriptor.
512 std::unique_ptr<DivergenceDescriptorT> DivDesc;
514
520
522 Out << "Propagator::BlockLabels {\n";
523 for (int BlockIdx = (int)CyclePOT.size() - 1; BlockIdx >= 0; --BlockIdx) {
524 const auto *Block = CyclePOT[BlockIdx];
525 const auto *Label = BlockLabels[Block];
526 Out << Context.print(Block) << "(" << BlockIdx << ") : ";
527 if (!Label) {
528 Out << "<null>\n";
529 } else {
530 Out << Context.print(Label) << "\n";
531 }
532 }
533 Out << "}\n";
534 }
535
536 // Push a definition (\p PushedLabel) to \p SuccBlock and return whether this
537 // causes a divergent join.
538 bool computeJoin(const BlockT &SuccBlock, const BlockT &PushedLabel) {
539 const auto *OldLabel = BlockLabels[&SuccBlock];
540
541 LLVM_DEBUG(dbgs() << "labeling " << Context.print(&SuccBlock) << ":\n"
542 << "\tpushed label: " << Context.print(&PushedLabel)
543 << "\n"
544 << "\told label: " << Context.print(OldLabel) << "\n");
545
546 // Early exit if there is no change in the label.
547 if (OldLabel == &PushedLabel)
548 return false;
549
550 if (OldLabel != &SuccBlock) {
551 auto SuccIdx = CyclePOT.getIndex(&SuccBlock);
552 // Assigning a new label, mark this in FreshLabels.
553 LLVM_DEBUG(dbgs() << "\tfresh label: " << SuccIdx << "\n");
554 FreshLabels.set(SuccIdx);
555 }
556
557 // This is not a join if the succ was previously unlabeled.
558 if (!OldLabel) {
559 LLVM_DEBUG(dbgs() << "\tnew label: " << Context.print(&PushedLabel)
560 << "\n");
561 BlockLabels[&SuccBlock] = &PushedLabel;
562 return false;
563 }
564
565 // This is a new join. Label the join block as itself, and not as
566 // the pushed label.
567 LLVM_DEBUG(dbgs() << "\tnew label: " << Context.print(&SuccBlock) << "\n");
568 BlockLabels[&SuccBlock] = &SuccBlock;
569
570 return true;
571 }
572
573 // visiting a virtual cycle exit edge from the cycle header --> temporal
574 // divergence on join
575 bool visitCycleExitEdge(const BlockT &ExitBlock, const BlockT &Label) {
576 if (!computeJoin(ExitBlock, Label))
577 return false;
578
579 // Identified a divergent cycle exit
580 DivDesc->CycleDivBlocks.insert(&ExitBlock);
581 LLVM_DEBUG(dbgs() << "\tDivergent cycle exit: " << Context.print(&ExitBlock)
582 << "\n");
583 return true;
584 }
585
586 // process \p SuccBlock with reaching definition \p Label
587 bool visitEdge(const BlockT &SuccBlock, const BlockT &Label) {
588 if (!computeJoin(SuccBlock, Label))
589 return false;
590
591 // Divergent, disjoint paths join.
592 DivDesc->JoinDivBlocks.insert(&SuccBlock);
593 LLVM_DEBUG(dbgs() << "\tDivergent join: " << Context.print(&SuccBlock)
594 << "\n");
595 return true;
596 }
597
598 std::unique_ptr<DivergenceDescriptorT> computeJoinPoints() {
600
601 LLVM_DEBUG(dbgs() << "SDA:computeJoinPoints: "
602 << Context.print(&DivTermBlock) << "\n");
603
604 int DivTermIdx = CyclePOT.getIndex(&DivTermBlock);
605
606 // Bootstrap with branch targets
607 auto const *DivTermCycle = CI.getCycle(&DivTermBlock);
608
609 // Locate the largest ancestor cycle that is not reducible and does not
610 // contain a reducible ancestor. This is done with a lambda that is defined
611 // and invoked in the same statement.
612 const CycleT *IrreducibleAncestor = [](const CycleT *C) -> const CycleT * {
613 if (!C)
614 return nullptr;
615 if (C->isReducible())
616 return nullptr;
617 while (const CycleT *P = C->getParentCycle()) {
618 if (P->isReducible())
619 return C;
620 C = P;
621 }
622 assert(!C->getParentCycle());
623 assert(!C->isReducible());
624 return C;
625 }(DivTermCycle);
626
627 for (const auto *SuccBlock : successors(&DivTermBlock)) {
628 if (DivTermCycle && !DivTermCycle->contains(SuccBlock)) {
629 // If DivTerm exits the cycle immediately, computeJoin() might
630 // not reach SuccBlock with a different label. We need to
631 // check for this exit now.
632 DivDesc->CycleDivBlocks.insert(SuccBlock);
633 LLVM_DEBUG(dbgs() << "\tImmediate divergent cycle exit: "
634 << Context.print(SuccBlock) << "\n");
635 }
636 visitEdge(*SuccBlock, *SuccBlock);
637 }
638
639 // Technically propagation can continue until it reaches the last node.
640 //
641 // For efficiency, propagation can stop if FreshLabels.count()==1. But
642 // For irreducible cycles, let propagation continue until it reaches
643 // out of irreducible cycles (see code for details.)
644 while (true) {
645 auto BlockIdx = FreshLabels.find_last();
646 if (BlockIdx == -1)
647 break;
648
649 const auto *Block = CyclePOT[BlockIdx];
650 // If no irreducible cycle, stop if freshLable.count() = 1 and Block
651 // is the IPD. If it is in any irreducible cycle, continue propagation.
652 if (FreshLabels.count() == 1 &&
653 (!IrreducibleAncestor || !IrreducibleAncestor->contains(Block)))
654 break;
655
656 LLVM_DEBUG(dbgs() << "Current labels:\n"; printDefs(dbgs()));
657
658 FreshLabels.reset(BlockIdx);
659 if (BlockIdx == DivTermIdx) {
660 LLVM_DEBUG(dbgs() << "Skipping DivTermBlock\n");
661 continue;
662 }
663
664 LLVM_DEBUG(dbgs() << "visiting " << Context.print(Block) << " at index "
665 << BlockIdx << "\n");
666
667 const auto *Label = BlockLabels[Block];
668 assert(Label);
669
670 // If the current block is the header of a reducible cycle that
671 // contains the divergent branch, then the label should be
672 // propagated to the cycle exits. Such a header is the "last
673 // possible join" of any disjoint paths within this cycle. This
674 // prevents detection of spurious joins at the entries of any
675 // irreducible child cycles.
676 //
677 // This conclusion about the header is true for any choice of DFS:
678 //
679 // If some DFS has a reducible cycle C with header H, then for
680 // any other DFS, H is the header of a cycle C' that is a
681 // superset of C. For a divergent branch inside the subgraph
682 // C, any join node inside C is either H, or some node
683 // encountered without passing through H.
684 //
685 auto getReducibleParent = [&](const BlockT *Block) -> const CycleT * {
686 if (!CyclePOT.isReducibleCycleHeader(Block))
687 return nullptr;
688 const auto *BlockCycle = CI.getCycle(Block);
689 if (BlockCycle->contains(&DivTermBlock))
690 return BlockCycle;
691 return nullptr;
692 };
693
694 if (const auto *BlockCycle = getReducibleParent(Block)) {
695 SmallVector<BlockT *, 4> BlockCycleExits;
696 BlockCycle->getExitBlocks(BlockCycleExits);
697 for (auto *BlockCycleExit : BlockCycleExits)
698 visitCycleExitEdge(*BlockCycleExit, *Label);
699 } else {
700 for (const auto *SuccBlock : successors(Block))
701 visitEdge(*SuccBlock, *Label);
702 }
703 }
704
705 LLVM_DEBUG(dbgs() << "Final labeling:\n"; printDefs(dbgs()));
706
707 // Check every cycle containing DivTermBlock for exit divergence.
708 // A cycle has exit divergence if the label of an exit block does
709 // not match the label of its header.
710 for (const auto *Cycle = CI.getCycle(&DivTermBlock); Cycle;
711 Cycle = Cycle->getParentCycle()) {
712 if (Cycle->isReducible()) {
713 // The exit divergence of a reducible cycle is recorded while
714 // propagating labels.
715 continue;
716 }
718 Cycle->getExitBlocks(Exits);
719 auto *Header = Cycle->getHeader();
720 auto *HeaderLabel = BlockLabels[Header];
721 for (const auto *Exit : Exits) {
722 if (BlockLabels[Exit] != HeaderLabel) {
723 // Identified a divergent cycle exit
724 DivDesc->CycleDivBlocks.insert(Exit);
725 LLVM_DEBUG(dbgs() << "\tDivergent cycle exit: " << Context.print(Exit)
726 << "\n");
727 }
728 }
729 }
730
731 return std::move(DivDesc);
732 }
733};
734
735template <typename ContextT>
737 const ContextT &Context, const DominatorTreeT &DT, const CycleInfoT &CI)
738 : CyclePO(Context), DT(DT), CI(CI) {
739 CyclePO.compute(CI);
740}
741
742template <typename ContextT>
744 const BlockT *DivTermBlock) -> const DivergenceDescriptor & {
745 // trivial case
746 if (succ_size(DivTermBlock) <= 1) {
747 return EmptyDivergenceDesc;
748 }
749
750 // already available in cache?
751 auto ItCached = CachedControlDivDescs.find(DivTermBlock);
752 if (ItCached != CachedControlDivDescs.end())
753 return *ItCached->second;
754
755 // compute all join points
756 DivergencePropagatorT Propagator(CyclePO, DT, CI, *DivTermBlock);
757 auto DivDesc = Propagator.computeJoinPoints();
758
759 auto printBlockSet = [&](ConstBlockSet &Blocks) {
760 return Printable([&](raw_ostream &Out) {
761 Out << "[";
762 ListSeparator LS;
763 for (const auto *BB : Blocks) {
764 Out << LS << CI.getSSAContext().print(BB);
765 }
766 Out << "]\n";
767 });
768 };
769
771 dbgs() << "\nResult (" << CI.getSSAContext().print(DivTermBlock)
772 << "):\n JoinDivBlocks: " << printBlockSet(DivDesc->JoinDivBlocks)
773 << " CycleDivBlocks: " << printBlockSet(DivDesc->CycleDivBlocks)
774 << "\n");
775 (void)printBlockSet;
776
777 auto ItInserted =
778 CachedControlDivDescs.try_emplace(DivTermBlock, std::move(DivDesc));
779 assert(ItInserted.second);
780 return *ItInserted.first->second;
781}
782
783template <typename ContextT>
785 const InstructionT &I) {
786 if (isAlwaysUniform(I))
787 return;
788 bool Marked = false;
789 if (I.isTerminator()) {
790 Marked = DivergentTermBlocks.insert(I.getParent()).second;
791 if (Marked) {
792 LLVM_DEBUG(dbgs() << "marked divergent term block: "
793 << Context.print(I.getParent()) << "\n");
794 }
795 } else {
796 Marked = markDefsDivergent(I);
797 }
798
799 if (Marked)
800 Worklist.push_back(&I);
801}
802
803template <typename ContextT>
805 ConstValueRefT Val) {
806 if (DivergentValues.insert(Val).second) {
807 LLVM_DEBUG(dbgs() << "marked divergent: " << Context.print(Val) << "\n");
808 return true;
809 }
810 return false;
811}
812
813template <typename ContextT>
815 const InstructionT &Instr) {
816 UniformOverrides.insert(&Instr);
817}
818
819// Mark as divergent all external uses of values defined in \p DefCycle.
820//
821// A value V defined by a block B inside \p DefCycle may be used outside the
822// cycle only if the use is a PHI in some exit block, or B dominates some exit
823// block. Thus, we check uses as follows:
824//
825// - Check all PHIs in all exit blocks for inputs defined inside \p DefCycle.
826// - For every block B inside \p DefCycle that dominates at least one exit
827// block, check all uses outside \p DefCycle.
828//
829// FIXME: This function does not distinguish between divergent and uniform
830// exits. For each divergent exit, only the values that are live at that exit
831// need to be propagated as divergent at their use outside the cycle.
832template <typename ContextT>
833void GenericUniformityAnalysisImpl<ContextT>::analyzeCycleExitDivergence(
834 const CycleT &DefCycle) {
836 DefCycle.getExitBlocks(Exits);
837 for (auto *Exit : Exits) {
838 for (auto &Phi : Exit->phis()) {
839 if (usesValueFromCycle(Phi, DefCycle)) {
840 markDivergent(Phi);
841 }
842 }
843 }
844
845 for (auto *BB : DefCycle.blocks()) {
846 if (!llvm::any_of(Exits,
847 [&](BlockT *Exit) { return DT.dominates(BB, Exit); }))
848 continue;
849 for (auto &II : *BB) {
850 propagateTemporalDivergence(II, DefCycle);
851 }
852 }
853}
854
855template <typename ContextT>
856void GenericUniformityAnalysisImpl<ContextT>::propagateCycleExitDivergence(
857 const BlockT &DivExit, const CycleT &InnerDivCycle) {
858 LLVM_DEBUG(dbgs() << "\tpropCycleExitDiv " << Context.print(&DivExit)
859 << "\n");
860 auto *DivCycle = &InnerDivCycle;
861 auto *OuterDivCycle = DivCycle;
862 auto *ExitLevelCycle = CI.getCycle(&DivExit);
863 const unsigned CycleExitDepth =
864 ExitLevelCycle ? ExitLevelCycle->getDepth() : 0;
865
866 // Find outer-most cycle that does not contain \p DivExit
867 while (DivCycle && DivCycle->getDepth() > CycleExitDepth) {
868 LLVM_DEBUG(dbgs() << " Found exiting cycle: "
869 << Context.print(DivCycle->getHeader()) << "\n");
870 OuterDivCycle = DivCycle;
871 DivCycle = DivCycle->getParentCycle();
872 }
873 LLVM_DEBUG(dbgs() << "\tOuter-most exiting cycle: "
874 << Context.print(OuterDivCycle->getHeader()) << "\n");
875
876 if (!DivergentExitCycles.insert(OuterDivCycle).second)
877 return;
878
879 // Exit divergence does not matter if the cycle itself is assumed to
880 // be divergent.
881 for (const auto *C : AssumedDivergent) {
882 if (C->contains(OuterDivCycle))
883 return;
884 }
885
886 analyzeCycleExitDivergence(*OuterDivCycle);
887}
888
889template <typename ContextT>
890void GenericUniformityAnalysisImpl<ContextT>::taintAndPushAllDefs(
891 const BlockT &BB) {
892 LLVM_DEBUG(dbgs() << "taintAndPushAllDefs " << Context.print(&BB) << "\n");
893 for (const auto &I : instrs(BB)) {
894 // Terminators do not produce values; they are divergent only if
895 // the condition is divergent. That is handled when the divergent
896 // condition is placed in the worklist.
897 if (I.isTerminator())
898 break;
899
900 markDivergent(I);
901 }
902}
903
904/// Mark divergent phi nodes in a join block
905template <typename ContextT>
906void GenericUniformityAnalysisImpl<ContextT>::taintAndPushPhiNodes(
907 const BlockT &JoinBlock) {
908 LLVM_DEBUG(dbgs() << "taintAndPushPhiNodes in " << Context.print(&JoinBlock)
909 << "\n");
910 for (const auto &Phi : JoinBlock.phis()) {
911 // FIXME: The non-undef value is not constant per se; it just happens to be
912 // uniform and may not dominate this PHI. So assuming that the same value
913 // reaches along all incoming edges may itself be undefined behaviour. This
914 // particular interpretation of the undef value was added to
915 // DivergenceAnalysis in the following review:
916 //
917 // https://reviews.llvm.org/D19013
918 if (ContextT::isConstantOrUndefValuePhi(Phi))
919 continue;
920 markDivergent(Phi);
921 }
922}
923
924/// Add \p Candidate to \p Cycles if it is not already contained in \p Cycles.
925///
926/// \return true iff \p Candidate was added to \p Cycles.
927template <typename CycleT>
929 CycleT *Candidate) {
930 if (llvm::any_of(Cycles,
931 [Candidate](CycleT *C) { return C->contains(Candidate); }))
932 return false;
933 Cycles.push_back(Candidate);
934 return true;
935}
936
937/// Return the outermost cycle made divergent by branch outside it.
938///
939/// If two paths that diverged outside an irreducible cycle join
940/// inside that cycle, then that whole cycle is assumed to be
941/// divergent. This does not apply if the cycle is reducible.
942template <typename CycleT, typename BlockT>
943static const CycleT *getExtDivCycle(const CycleT *Cycle,
944 const BlockT *DivTermBlock,
945 const BlockT *JoinBlock) {
946 assert(Cycle);
947 assert(Cycle->contains(JoinBlock));
948
949 if (Cycle->contains(DivTermBlock))
950 return nullptr;
951
952 const auto *OriginalCycle = Cycle;
953 const auto *Parent = Cycle->getParentCycle();
954 while (Parent && !Parent->contains(DivTermBlock)) {
955 Cycle = Parent;
956 Parent = Cycle->getParentCycle();
957 }
958
959 // If the original cycle is not the outermost cycle, then the outermost cycle
960 // is irreducible. If the outermost cycle were reducible, then external
961 // diverged paths would not reach the original inner cycle.
962 (void)OriginalCycle;
963 assert(Cycle == OriginalCycle || !Cycle->isReducible());
964
965 if (Cycle->isReducible()) {
966 assert(Cycle->getHeader() == JoinBlock);
967 return nullptr;
968 }
969
970 LLVM_DEBUG(dbgs() << "cycle made divergent by external branch\n");
971 return Cycle;
972}
973
974/// Return the outermost cycle made divergent by branch inside it.
975///
976/// This checks the "diverged entry" criterion defined in the
977/// docs/ConvergenceAnalysis.html.
978template <typename ContextT, typename CycleT, typename BlockT,
979 typename DominatorTreeT>
980static const CycleT *
981getIntDivCycle(const CycleT *Cycle, const BlockT *DivTermBlock,
982 const BlockT *JoinBlock, const DominatorTreeT &DT,
983 ContextT &Context) {
984 LLVM_DEBUG(dbgs() << "examine join " << Context.print(JoinBlock)
985 << " for internal branch " << Context.print(DivTermBlock)
986 << "\n");
987 if (DT.properlyDominates(DivTermBlock, JoinBlock))
988 return nullptr;
989
990 // Find the smallest common cycle, if one exists.
991 assert(Cycle && Cycle->contains(JoinBlock));
992 while (Cycle && !Cycle->contains(DivTermBlock)) {
993 Cycle = Cycle->getParentCycle();
994 }
995 if (!Cycle || Cycle->isReducible())
996 return nullptr;
997
998 if (DT.properlyDominates(Cycle->getHeader(), JoinBlock))
999 return nullptr;
1000
1001 LLVM_DEBUG(dbgs() << " header " << Context.print(Cycle->getHeader())
1002 << " does not dominate join\n");
1003
1004 const auto *Parent = Cycle->getParentCycle();
1005 while (Parent && !DT.properlyDominates(Parent->getHeader(), JoinBlock)) {
1006 LLVM_DEBUG(dbgs() << " header " << Context.print(Parent->getHeader())
1007 << " does not dominate join\n");
1008 Cycle = Parent;
1009 Parent = Parent->getParentCycle();
1010 }
1011
1012 LLVM_DEBUG(dbgs() << " cycle made divergent by internal branch\n");
1013 return Cycle;
1014}
1015
1016template <typename ContextT, typename CycleT, typename BlockT,
1017 typename DominatorTreeT>
1018static const CycleT *
1019getOutermostDivergentCycle(const CycleT *Cycle, const BlockT *DivTermBlock,
1020 const BlockT *JoinBlock, const DominatorTreeT &DT,
1021 ContextT &Context) {
1022 if (!Cycle)
1023 return nullptr;
1024
1025 // First try to expand Cycle to the largest that contains JoinBlock
1026 // but not DivTermBlock.
1027 const auto *Ext = getExtDivCycle(Cycle, DivTermBlock, JoinBlock);
1028
1029 // Continue expanding to the largest cycle that contains both.
1030 const auto *Int = getIntDivCycle(Cycle, DivTermBlock, JoinBlock, DT, Context);
1031
1032 if (Int)
1033 return Int;
1034 return Ext;
1035}
1036
1037template <typename ContextT>
1038bool GenericUniformityAnalysisImpl<ContextT>::isTemporalDivergent(
1039 const BlockT &ObservingBlock, const InstructionT &Def) const {
1040 const BlockT *DefBlock = Def.getParent();
1041 for (const CycleT *Cycle = CI.getCycle(DefBlock);
1042 Cycle && !Cycle->contains(&ObservingBlock);
1043 Cycle = Cycle->getParentCycle()) {
1044 if (DivergentExitCycles.contains(Cycle)) {
1045 return true;
1046 }
1047 }
1048 return false;
1049}
1050
1051template <typename ContextT>
1053 const InstructionT &Term) {
1054 const auto *DivTermBlock = Term.getParent();
1055 DivergentTermBlocks.insert(DivTermBlock);
1056 LLVM_DEBUG(dbgs() << "analyzeControlDiv " << Context.print(DivTermBlock)
1057 << "\n");
1058
1059 // Don't propagate divergence from unreachable blocks.
1060 if (!DT.isReachableFromEntry(DivTermBlock))
1061 return;
1062
1063 const auto &DivDesc = SDA.getJoinBlocks(DivTermBlock);
1065
1066 // Iterate over all blocks now reachable by a disjoint path join
1067 for (const auto *JoinBlock : DivDesc.JoinDivBlocks) {
1068 const auto *Cycle = CI.getCycle(JoinBlock);
1069 LLVM_DEBUG(dbgs() << "visiting join block " << Context.print(JoinBlock)
1070 << "\n");
1071 if (const auto *Outermost = getOutermostDivergentCycle(
1072 Cycle, DivTermBlock, JoinBlock, DT, Context)) {
1073 LLVM_DEBUG(dbgs() << "found divergent cycle\n");
1074 DivCycles.push_back(Outermost);
1075 continue;
1076 }
1077 taintAndPushPhiNodes(*JoinBlock);
1078 }
1079
1080 // Sort by order of decreasing depth. This allows later cycles to be skipped
1081 // because they are already contained in earlier ones.
1082 llvm::sort(DivCycles, [](const CycleT *A, const CycleT *B) {
1083 return A->getDepth() > B->getDepth();
1084 });
1085
1086 // Cycles that are assumed divergent due to the diverged entry
1087 // criterion potentially contain temporal divergence depending on
1088 // the DFS chosen. Conservatively, all values produced in such a
1089 // cycle are assumed divergent. "Cycle invariant" values may be
1090 // assumed uniform, but that requires further analysis.
1091 for (auto *C : DivCycles) {
1092 if (!insertIfNotContained(AssumedDivergent, C))
1093 continue;
1094 LLVM_DEBUG(dbgs() << "process divergent cycle\n");
1095 for (const BlockT *BB : C->blocks()) {
1096 taintAndPushAllDefs(*BB);
1097 }
1098 }
1099
1100 const auto *BranchCycle = CI.getCycle(DivTermBlock);
1101 assert(DivDesc.CycleDivBlocks.empty() || BranchCycle);
1102 for (const auto *DivExitBlock : DivDesc.CycleDivBlocks) {
1103 propagateCycleExitDivergence(*DivExitBlock, *BranchCycle);
1104 }
1105}
1106
1107template <typename ContextT>
1109 // Initialize worklist.
1110 auto DivValuesCopy = DivergentValues;
1111 for (const auto DivVal : DivValuesCopy) {
1112 assert(isDivergent(DivVal) && "Worklist invariant violated!");
1113 pushUsers(DivVal);
1114 }
1115
1116 // All values on the Worklist are divergent.
1117 // Their users may not have been updated yet.
1118 while (!Worklist.empty()) {
1119 const InstructionT *I = Worklist.back();
1120 Worklist.pop_back();
1121
1122 LLVM_DEBUG(dbgs() << "worklist pop: " << Context.print(I) << "\n");
1123
1124 if (I->isTerminator()) {
1126 continue;
1127 }
1128
1129 // propagate value divergence to users
1130 assert(isDivergent(*I) && "Worklist invariant violated!");
1131 pushUsers(*I);
1132 }
1133}
1134
1135template <typename ContextT>
1141
1142template <typename ContextT>
1144 const InstructionT &Instr) const {
1145 return UniformOverrides.contains(&Instr);
1146}
1147
1148template <typename ContextT>
1150 const DominatorTreeT &DT, const CycleInfoT &CI,
1151 const TargetTransformInfo *TTI) {
1152 DA.reset(new ImplT{DT, CI, TTI});
1153}
1154
1155template <typename ContextT>
1157 bool haveDivergentArgs = false;
1158
1159 // When we print Value, LLVM IR instruction, we want to print extra new line.
1160 // In LLVM IR print function for Value does not print new line at the end.
1161 // In MIR print for MachineInstr prints new line at the end.
1162 constexpr bool IsMIR = std::is_same<InstructionT, MachineInstr>::value;
1163 std::string NewLine = IsMIR ? "" : "\n";
1164
1165 // Control flow instructions may be divergent even if their inputs are
1166 // uniform. Thus, although exceedingly rare, it is possible to have a program
1167 // with no divergent values but with divergent control structures.
1168 if (DivergentValues.empty() && DivergentTermBlocks.empty() &&
1169 DivergentExitCycles.empty()) {
1170 OS << "ALL VALUES UNIFORM\n";
1171 return;
1172 }
1173
1174 for (const auto &entry : DivergentValues) {
1175 const BlockT *parent = Context.getDefBlock(entry);
1176 if (!parent) {
1177 if (!haveDivergentArgs) {
1178 OS << "DIVERGENT ARGUMENTS:\n";
1179 haveDivergentArgs = true;
1180 }
1181 OS << " DIVERGENT: " << Context.print(entry) << '\n';
1182 }
1183 }
1184
1185 if (!AssumedDivergent.empty()) {
1186 OS << "CYCLES ASSUMED DIVERGENT:\n";
1187 for (const CycleT *cycle : AssumedDivergent) {
1188 OS << " " << cycle->print(Context) << '\n';
1189 }
1190 }
1191
1192 if (!DivergentExitCycles.empty()) {
1193 OS << "CYCLES WITH DIVERGENT EXIT:\n";
1194 for (const CycleT *cycle : DivergentExitCycles) {
1195 OS << " " << cycle->print(Context) << '\n';
1196 }
1197 }
1198
1199 if (!TemporalDivergenceList.empty()) {
1200 OS << "\nTEMPORAL DIVERGENCE LIST:\n";
1201
1202 for (auto [Val, UseInst, Cycle] : TemporalDivergenceList) {
1203 OS << "Value :" << Context.print(Val) << NewLine
1204 << "Used by :" << Context.print(UseInst) << NewLine
1205 << "Outside cycle :" << Cycle->print(Context) << "\n\n";
1206 }
1207 }
1208
1209 for (auto &block : F) {
1210 OS << "\nBLOCK " << Context.print(&block) << '\n';
1211
1212 OS << "DEFINITIONS\n";
1214 Context.appendBlockDefs(defs, block);
1215 for (auto value : defs) {
1216 if (isDivergent(value))
1217 OS << " DIVERGENT: ";
1218 else
1219 OS << " ";
1220 OS << Context.print(value) << NewLine;
1221 }
1222
1223 OS << "TERMINATORS\n";
1225 Context.appendBlockTerms(terms, block);
1226 bool divergentTerminators = hasDivergentTerminator(block);
1227 for (auto *T : terms) {
1228 if (divergentTerminators)
1229 OS << " DIVERGENT: ";
1230 else
1231 OS << " ";
1232 OS << Context.print(T) << NewLine;
1233 }
1234
1235 OS << "END BLOCK\n";
1236 }
1237}
1238
1239template <typename ContextT>
1243 return make_range(DA->TemporalDivergenceList.begin(),
1244 DA->TemporalDivergenceList.end());
1245}
1246
1247template <typename ContextT>
1249 return DA->hasDivergence();
1250}
1251
1252template <typename ContextT>
1253const typename ContextT::FunctionT &
1255 return DA->getFunction();
1256}
1257
1258/// Whether \p V is divergent at its definition.
1259template <typename ContextT>
1261 return DA->isDivergent(V);
1262}
1263
1264template <typename ContextT>
1266 return DA->isDivergent(*I);
1267}
1268
1269template <typename ContextT>
1271 return DA->isDivergentUse(U);
1272}
1273
1274template <typename ContextT>
1276 return DA->hasDivergentTerminator(B);
1277}
1278
1279/// \brief T helper function for printing.
1280template <typename ContextT>
1282 DA->print(out);
1283}
1284
1285template <typename ContextT>
1286void llvm::ModifiedPostOrder<ContextT>::computeStackPO(
1287 SmallVectorImpl<const BlockT *> &Stack, const CycleInfoT &CI,
1288 const CycleT *Cycle, SmallPtrSetImpl<const BlockT *> &Finalized) {
1289 LLVM_DEBUG(dbgs() << "inside computeStackPO\n");
1290 while (!Stack.empty()) {
1291 auto *NextBB = Stack.back();
1292 if (Finalized.count(NextBB)) {
1293 Stack.pop_back();
1294 continue;
1295 }
1296 LLVM_DEBUG(dbgs() << " visiting " << CI.getSSAContext().print(NextBB)
1297 << "\n");
1298 auto *NestedCycle = CI.getCycle(NextBB);
1299 if (Cycle != NestedCycle && (!Cycle || Cycle->contains(NestedCycle))) {
1300 LLVM_DEBUG(dbgs() << " found a cycle\n");
1301 while (NestedCycle->getParentCycle() != Cycle)
1302 NestedCycle = NestedCycle->getParentCycle();
1303
1304 SmallVector<BlockT *, 3> NestedExits;
1305 NestedCycle->getExitBlocks(NestedExits);
1306 bool PushedNodes = false;
1307 for (auto *NestedExitBB : NestedExits) {
1308 LLVM_DEBUG(dbgs() << " examine exit: "
1309 << CI.getSSAContext().print(NestedExitBB) << "\n");
1310 if (Cycle && !Cycle->contains(NestedExitBB))
1311 continue;
1312 if (Finalized.count(NestedExitBB))
1313 continue;
1314 PushedNodes = true;
1315 Stack.push_back(NestedExitBB);
1316 LLVM_DEBUG(dbgs() << " pushed exit: "
1317 << CI.getSSAContext().print(NestedExitBB) << "\n");
1318 }
1319 if (!PushedNodes) {
1320 // All loop exits finalized -> finish this node
1321 Stack.pop_back();
1322 computeCyclePO(CI, NestedCycle, Finalized);
1323 }
1324 continue;
1325 }
1326
1327 LLVM_DEBUG(dbgs() << " no nested cycle, going into DAG\n");
1328 // DAG-style
1329 bool PushedNodes = false;
1330 for (auto *SuccBB : successors(NextBB)) {
1331 LLVM_DEBUG(dbgs() << " examine succ: "
1332 << CI.getSSAContext().print(SuccBB) << "\n");
1333 if (Cycle && !Cycle->contains(SuccBB))
1334 continue;
1335 if (Finalized.count(SuccBB))
1336 continue;
1337 PushedNodes = true;
1338 Stack.push_back(SuccBB);
1339 LLVM_DEBUG(dbgs() << " pushed succ: " << CI.getSSAContext().print(SuccBB)
1340 << "\n");
1341 }
1342 if (!PushedNodes) {
1343 // Never push nodes twice
1344 LLVM_DEBUG(dbgs() << " finishing node: "
1345 << CI.getSSAContext().print(NextBB) << "\n");
1346 Stack.pop_back();
1347 Finalized.insert(NextBB);
1348 appendBlock(*NextBB);
1349 }
1350 }
1351 LLVM_DEBUG(dbgs() << "exited computeStackPO\n");
1352}
1353
1354template <typename ContextT>
1355void ModifiedPostOrder<ContextT>::computeCyclePO(
1356 const CycleInfoT &CI, const CycleT *Cycle,
1358 LLVM_DEBUG(dbgs() << "inside computeCyclePO\n");
1360 auto *CycleHeader = Cycle->getHeader();
1361
1362 LLVM_DEBUG(dbgs() << " noted header: "
1363 << CI.getSSAContext().print(CycleHeader) << "\n");
1364 assert(!Finalized.count(CycleHeader));
1365 Finalized.insert(CycleHeader);
1366
1367 // Visit the header last
1368 LLVM_DEBUG(dbgs() << " finishing header: "
1369 << CI.getSSAContext().print(CycleHeader) << "\n");
1370 appendBlock(*CycleHeader, Cycle->isReducible());
1371
1372 // Initialize with immediate successors
1373 for (auto *BB : successors(CycleHeader)) {
1374 LLVM_DEBUG(dbgs() << " examine succ: " << CI.getSSAContext().print(BB)
1375 << "\n");
1376 if (!Cycle->contains(BB))
1377 continue;
1378 if (BB == CycleHeader)
1379 continue;
1380 if (!Finalized.count(BB)) {
1381 LLVM_DEBUG(dbgs() << " pushed succ: " << CI.getSSAContext().print(BB)
1382 << "\n");
1383 Stack.push_back(BB);
1384 }
1385 }
1386
1387 // Compute PO inside region
1388 computeStackPO(Stack, CI, Cycle, Finalized);
1389
1390 LLVM_DEBUG(dbgs() << "exited computeCyclePO\n");
1391}
1392
1393/// \brief Generically compute the modified post order.
1394template <typename ContextT>
1398 auto *F = CI.getFunction();
1399 Stack.reserve(24); // FIXME made-up number
1400 Stack.push_back(&F->front());
1401 computeStackPO(Stack, CI, nullptr, Finalized);
1402}
1403
1404} // namespace llvm
1405
1406#undef DEBUG_TYPE
1407
1408#endif // LLVM_ADT_GENERICUNIFORMITYIMPL_H
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
This file defines the DenseSet and SmallDenseSet classes.
#define F(x, y, z)
Definition MD5.cpp:55
#define I(x, y, z)
Definition MD5.cpp:58
#define T
uint64_t IntrinsicInst * II
#define P(N)
This file contains some templates that are useful if you are working with the STL at all.
This file defines the SmallPtrSet class.
This file defines the SparseBitVector class.
This file contains some functions that are useful when dealing with strings.
#define LLVM_DEBUG(...)
Definition Debug.h:114
unify loop Fixup each natural loop to have a single exit block
Implements a dense probed hash-table based set.
Definition DenseSet.h:279
Compute divergence starting with a divergent branch.
typename SyncDependenceAnalysisT::BlockLabelMap BlockLabelMapT
GenericSyncDependenceAnalysis< ContextT > SyncDependenceAnalysisT
typename ContextT::DominatorTreeT DominatorTreeT
bool computeJoin(const BlockT &SuccBlock, const BlockT &PushedLabel)
std::unique_ptr< DivergenceDescriptorT > DivDesc
void printDefs(raw_ostream &Out)
typename ContextT::FunctionT FunctionT
GenericCycleInfo< ContextT > CycleInfoT
ModifiedPostOrder< ContextT > ModifiedPO
std::unique_ptr< DivergenceDescriptorT > computeJoinPoints()
bool visitCycleExitEdge(const BlockT &ExitBlock, const BlockT &Label)
typename ContextT::ValueRefT ValueRefT
typename ContextT::BlockT BlockT
typename CycleInfoT::CycleT CycleT
DivergencePropagator(const ModifiedPO &CyclePOT, const DominatorTreeT &DT, const CycleInfoT &CI, const BlockT &DivTermBlock)
bool visitEdge(const BlockT &SuccBlock, const BlockT &Label)
typename SyncDependenceAnalysisT::DivergenceDescriptor DivergenceDescriptorT
Cycle information for a function.
bool contains(const BlockT *Block) const
Return whether Block is contained in the cycle.
const GenericCycle * getParentCycle() const
Locate join blocks for disjoint paths starting at a divergent branch.
GenericSyncDependenceAnalysis(const ContextT &Context, const DominatorTreeT &DT, const CycleInfoT &CI)
ModifiedPostOrder< ContextT > ModifiedPO
DivergencePropagator< ContextT > DivergencePropagatorT
DenseMap< const BlockT *, const BlockT * > BlockLabelMap
SmallPtrSet< const BlockT *, 4 > ConstBlockSet
typename ContextT::DominatorTreeT DominatorTreeT
GenericCycleInfo< ContextT > CycleInfoT
typename ContextT::FunctionT FunctionT
typename ContextT::InstructionT InstructionT
typename ContextT::ValueRefT ValueRefT
const DivergenceDescriptor & getJoinBlocks(const BlockT *DivTermBlock)
Computes divergent join points and cycle exits caused by branch divergence in Term.
SmallVector< TemporalDivergenceTuple, 8 > TemporalDivergenceList
bool isAlwaysUniform(const InstructionT &Instr) const
Whether Val will always return a uniform value regardless of its operands.
typename ContextT::ValueRefT ValueRefT
void analyzeControlDivergence(const InstructionT &Term)
Mark Term as divergent and push all Instructions that become divergent as a result on the worklist.
bool isDivergent(ConstValueRefT V) const
Whether Val is divergent at its definition.
bool isDivergentUse(const UseT &U) const
bool hasDivergentDefs(const InstructionT &I) const
typename ContextT::InstructionT InstructionT
DenseSet< ConstValueRefT > DivergentValues
typename ContextT::ConstValueRefT ConstValueRefT
typename SyncDependenceAnalysisT::BlockLabelMap BlockLabelMapT
bool hasDivergence() const
Whether any value was marked or analyzed to be divergent.
typename ContextT::DominatorTreeT DominatorTreeT
void compute()
Propagate divergence to all instructions in the region.
bool hasDivergentTerminator(const BlockT &B) const
GenericUniformityAnalysisImpl(const DominatorTreeT &DT, const CycleInfoT &CI, const TargetTransformInfo *TTI)
bool markDefsDivergent(const InstructionT &Instr)
Mark outputs of Instr as divergent.
typename ContextT::FunctionT FunctionT
bool isDivergent(const InstructionT &I) const
std::tuple< ConstValueRefT, InstructionT *, const CycleT * > TemporalDivergenceTuple
std::vector< const InstructionT * > Worklist
void markDivergent(const InstructionT &I)
Examine I for divergent outputs and add to the worklist.
SmallPtrSet< const BlockT *, 32 > DivergentTermBlocks
GenericCycleInfo< ContextT > CycleInfoT
void addUniformOverride(const InstructionT &Instr)
Mark UniVal as a value that is always uniform.
void recordTemporalDivergence(ConstValueRefT, const InstructionT *, const CycleT *)
typename SyncDependenceAnalysisT::DivergenceDescriptor DivergenceDescriptorT
GenericSyncDependenceAnalysis< ContextT > SyncDependenceAnalysisT
bool hasDivergentTerminator(const BlockT &B)
bool isDivergentUse(const UseT &U) const
Whether U is divergent.
bool isDivergent(ConstValueRefT V) const
Whether V is divergent at its definition.
void print(raw_ostream &Out) const
T helper function for printing.
std::tuple< ConstValueRefT, InstructionT *, const CycleT * > TemporalDivergenceTuple
typename ContextT::ConstValueRefT ConstValueRefT
typename ContextT::BlockT BlockT
typename ContextT::InstructionT InstructionT
bool hasDivergence() const
Whether any divergence was detected.
const FunctionT & getFunction() const
The GPU kernel this analysis result is for.
GenericCycleInfo< ContextT > CycleInfoT
typename ContextT::DominatorTreeT DominatorTreeT
iterator_range< TemporalDivergenceTuple * > getTemporalDivergenceList() const
A helper class to return the specified delimiter string after the first invocation of operator String...
Construct a specially modified post-order traversal of cycles.
typename ContextT::FunctionT FunctionT
const BlockT * operator[](size_t idx) const
typename CycleInfoT::CycleT CycleT
bool isReducibleCycleHeader(const BlockT *BB) const
ModifiedPostOrder(const ContextT &C)
unsigned count(BlockT *BB) const
void compute(const CycleInfoT &CI)
Generically compute the modified post order.
GenericCycleInfo< ContextT > CycleInfoT
void appendBlock(const BlockT &BB, bool isReducibleCycleHeader=false)
unsigned getIndex(const BlockT *BB) const
typename std::vector< BlockT * >::const_iterator const_iterator
typename ContextT::DominatorTreeT DominatorTreeT
typename ContextT::BlockT BlockT
Simple wrapper around std::function<void(raw_ostream&)>.
Definition Printable.h:38
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
A range adaptor for a pair of iterators.
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition raw_ostream.h:53
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
This is an optimization pass for GlobalISel generic memory operations.
static const CycleT * getIntDivCycle(const CycleT *Cycle, const BlockT *DivTermBlock, const BlockT *JoinBlock, const DominatorTreeT &DT, ContextT &Context)
Return the outermost cycle made divergent by branch inside it.
static const CycleT * getExtDivCycle(const CycleT *Cycle, const BlockT *DivTermBlock, const BlockT *JoinBlock)
Return the outermost cycle made divergent by branch outside it.
auto successors(const MachineBasicBlock *BB)
static bool insertIfNotContained(SmallVector< CycleT * > &Cycles, CycleT *Candidate)
Add Candidate to Cycles if it is not already contained in Cycles.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
CycleInfo::CycleT Cycle
Definition CycleInfo.h:24
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1712
void sort(IteratorTy Start, IteratorTy End)
Definition STLExtras.h:1624
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:207
auto succ_size(const MachineBasicBlock *BB)
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
TargetTransformInfo TTI
auto instrs(const MachineBasicBlock &BB)
static const CycleT * getOutermostDivergentCycle(const CycleT *Cycle, const BlockT *DivTermBlock, const BlockT *JoinBlock, const DominatorTreeT &DT, ContextT &Context)
Information discovered by the sync dependence analysis for each divergent branch.