diff --git a/llvm/include/llvm/Transforms/Utils/Local.h b/llvm/include/llvm/Transforms/Utils/Local.h index db064e1f41f02..8ece8405ae5ce 100644 --- a/llvm/include/llvm/Transforms/Utils/Local.h +++ b/llvm/include/llvm/Transforms/Utils/Local.h @@ -175,6 +175,22 @@ bool EliminateDuplicatePHINodes(BasicBlock *BB); bool EliminateDuplicatePHINodes(BasicBlock *BB, SmallPtrSetImpl &ToRemove); +/// Check for and eliminate duplicate PHI nodes in the block. This function is +/// specifically designed for scenarios where new PHI nodes are inserted into +/// the beginning of the block, such when SSAUpdaterBulk::RewriteAllUses. It +/// compares the newly inserted PHI nodes against the existing ones and if a +/// new PHI node is found to be a duplicate of an existing one, the new node is +/// removed. Existing PHI nodes are left unmodified, even if they are +/// duplicates. New nodes are also deleted if they are duplicates of each other. +/// Similar to EliminateDuplicatePHINodes, this function assumes a consistent +/// order for all incoming values across PHI nodes in the block. FirstExistingPN +/// Points to the first existing PHI node in the block. Newly inserted PHI nodes +/// should not reference one another. However, they may reference themselves or +/// existing PHI nodes, and existing PHI nodes may reference the newly inserted +/// PHI nodes. +bool EliminateNewDuplicatePHINodes(BasicBlock *BB, + BasicBlock::phi_iterator FirstExistingPN); + /// This function is used to do simplification of a CFG. For example, it /// adjusts branches to branches to eliminate the extra hop, it eliminates /// unreachable basic blocks, and does other peephole optimization of the CFG. diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp index 809a0d7ebeea6..382f1d3fe1a44 100644 --- a/llvm/lib/Transforms/Utils/Local.cpp +++ b/llvm/lib/Transforms/Utils/Local.cpp @@ -1420,65 +1420,65 @@ EliminateDuplicatePHINodesNaiveImpl(BasicBlock *BB, return Changed; } -static bool -EliminateDuplicatePHINodesSetBasedImpl(BasicBlock *BB, - SmallPtrSetImpl &ToRemove) { - // This implementation doesn't currently consider undef operands - // specially. Theoretically, two phis which are identical except for - // one having an undef where the other doesn't could be collapsed. +// This implementation doesn't currently consider undef operands +// specially. Theoretically, two phis which are identical except for +// one having an undef where the other doesn't could be collapsed. - struct PHIDenseMapInfo { - static PHINode *getEmptyKey() { - return DenseMapInfo::getEmptyKey(); - } +struct PHIDenseMapInfo { + static PHINode *getEmptyKey() { + return DenseMapInfo::getEmptyKey(); + } - static PHINode *getTombstoneKey() { - return DenseMapInfo::getTombstoneKey(); - } + static PHINode *getTombstoneKey() { + return DenseMapInfo::getTombstoneKey(); + } - static bool isSentinel(PHINode *PN) { - return PN == getEmptyKey() || PN == getTombstoneKey(); - } + static bool isSentinel(const PHINode *PN) { + return PN == getEmptyKey() || PN == getTombstoneKey(); + } - // WARNING: this logic must be kept in sync with - // Instruction::isIdenticalToWhenDefined()! - static unsigned getHashValueImpl(PHINode *PN) { - // Compute a hash value on the operands. Instcombine will likely have - // sorted them, which helps expose duplicates, but we have to check all - // the operands to be safe in case instcombine hasn't run. - return static_cast( - hash_combine(hash_combine_range(PN->operand_values()), - hash_combine_range(PN->blocks()))); - } + // WARNING: this logic must be kept in sync with + // Instruction::isIdenticalToWhenDefined()! + static unsigned getHashValueImpl(const PHINode *PN) { + // Compute a hash value on the operands. Instcombine will likely have + // sorted them, which helps expose duplicates, but we have to check all + // the operands to be safe in case instcombine hasn't run. + return static_cast( + hash_combine(hash_combine_range(PN->operand_values()), + hash_combine_range(PN->blocks()))); + } - static unsigned getHashValue(PHINode *PN) { + static unsigned getHashValue(const PHINode *PN) { #ifndef NDEBUG - // If -phicse-debug-hash was specified, return a constant -- this - // will force all hashing to collide, so we'll exhaustively search - // the table for a match, and the assertion in isEqual will fire if - // there's a bug causing equal keys to hash differently. - if (PHICSEDebugHash) - return 0; + // If -phicse-debug-hash was specified, return a constant -- this + // will force all hashing to collide, so we'll exhaustively search + // the table for a match, and the assertion in isEqual will fire if + // there's a bug causing equal keys to hash differently. + if (PHICSEDebugHash) + return 0; #endif - return getHashValueImpl(PN); - } + return getHashValueImpl(PN); + } - static bool isEqualImpl(PHINode *LHS, PHINode *RHS) { - if (isSentinel(LHS) || isSentinel(RHS)) - return LHS == RHS; - return LHS->isIdenticalTo(RHS); - } + static bool isEqualImpl(const PHINode *LHS, const PHINode *RHS) { + if (isSentinel(LHS) || isSentinel(RHS)) + return LHS == RHS; + return LHS->isIdenticalTo(RHS); + } - static bool isEqual(PHINode *LHS, PHINode *RHS) { - // These comparisons are nontrivial, so assert that equality implies - // hash equality (DenseMap demands this as an invariant). - bool Result = isEqualImpl(LHS, RHS); - assert(!Result || (isSentinel(LHS) && LHS == RHS) || - getHashValueImpl(LHS) == getHashValueImpl(RHS)); - return Result; - } - }; + static bool isEqual(const PHINode *LHS, const PHINode *RHS) { + // These comparisons are nontrivial, so assert that equality implies + // hash equality (DenseMap demands this as an invariant). + bool Result = isEqualImpl(LHS, RHS); + assert(!Result || (isSentinel(LHS) && LHS == RHS) || + getHashValueImpl(LHS) == getHashValueImpl(RHS)); + return Result; + } +}; +static bool +EliminateDuplicatePHINodesSetBasedImpl(BasicBlock *BB, + SmallPtrSetImpl &ToRemove) { // Set of unique PHINodes. DenseSet PHISet; PHISet.reserve(4 * PHICSENumPHISmallSize); @@ -1525,6 +1525,111 @@ bool llvm::EliminateDuplicatePHINodes(BasicBlock *BB) { return Changed; } +#ifndef NDEBUG // Should this be under EXPENSIVE_CHECKS? +// New PHI nodes should not reference one another but they may reference +// themselves or existing PHI nodes, and existing PHI nodes may reference new +// PHI nodes. +static bool +PHIAreRefEachOther(const iterator_range &NewPHIs) { + SmallPtrSet NewPHISet; + for (PHINode &PN : NewPHIs) + NewPHISet.insert(&PN); + for (PHINode &PHI : NewPHIs) { + for (Value *V : PHI.incoming_values()) { + PHINode *IncPHI = dyn_cast(V); + if (IncPHI && IncPHI != &PHI && NewPHISet.contains(IncPHI)) + return true; + } + } + return false; +} +#endif + +bool EliminateNewDuplicatePHINodesN2(BasicBlock *BB, + BasicBlock::phi_iterator FirstExistingPN) { + auto ReplaceIfIdentical = [](PHINode &PHI, PHINode &ReplPHI) { + if (!PHI.isIdenticalToWhenDefined(&ReplPHI)) + return false; + PHI.replaceAllUsesWith(&ReplPHI); + PHI.eraseFromParent(); + return true; + }; + + // Deduplicate new PHIs first to reduce the number of comparisons on the + // following new -> existing pass. + bool Changed = false; + for (auto I = BB->phis().begin(); I != FirstExistingPN; ++I) { + for (auto J = std::next(I); J != FirstExistingPN;) { + Changed |= ReplaceIfIdentical(*J++, *I); + } + } + + // Iterate over existing PHIs and replace identical new PHIs. + for (PHINode &ExistingPHI : make_range(FirstExistingPN, BB->phis().end())) { + auto I = BB->phis().begin(); + assert(I != FirstExistingPN); // Should be at least one new PHI. + do { + Changed |= ReplaceIfIdentical(*I++, ExistingPHI); + } while (I != FirstExistingPN); + if (BB->phis().begin() == FirstExistingPN) + return Changed; + } + return Changed; +} + +bool EliminateNewDuplicatePHINodesSet( + BasicBlock *BB, BasicBlock::phi_iterator FirstExistingPN) { + auto Replace = [](PHINode &PHI, PHINode &ReplPHI) { + PHI.replaceAllUsesWith(&ReplPHI); + PHI.eraseFromParent(); + return true; + }; + + DenseSet NewPHISet; + NewPHISet.reserve(4 * PHICSENumPHISmallSize); + + // Deduplicate new PHIs, note that NewPHISet remains consistent because new + // PHIs are not reference each other. + bool Changed = false; + for (PHINode &NewPHI : + make_early_inc_range(make_range(BB->phis().begin(), FirstExistingPN))) { + auto [I, Inserted] = NewPHISet.insert(&NewPHI); + if (!Inserted) + Changed |= Replace(NewPHI, **I); + } + + // Iterate over existing PHIs and replace matching new PHIs. + for (PHINode &ExistingPHI : make_range(FirstExistingPN, BB->phis().end())) { + assert(!NewPHISet.empty()); // Should be at least one new PHI. + auto I = NewPHISet.find(&ExistingPHI); + if (I == NewPHISet.end()) + continue; + Changed |= Replace(**I, ExistingPHI); + NewPHISet.erase(I); + if (NewPHISet.empty()) + return Changed; + } + return Changed; +} + +bool llvm::EliminateNewDuplicatePHINodes( + BasicBlock *BB, BasicBlock::phi_iterator FirstExistingPN) { + + if (hasNItemsOrLess(BB->phis(), 1)) + return false; + + auto NewPHIs = make_range(BB->phis().begin(), FirstExistingPN); + assert(!PHIAreRefEachOther(NewPHIs)); + + // Both functions perform identical pass over existing PHIs and differ in time + // spent on new PHI duplicate check which depends on the number of new PHIs. + // Therefore make a choice based on the number of new PHIs and not the total + // number of PHIs in the block. + return hasNItemsOrLess(NewPHIs, PHICSENumPHISmallSize) + ? EliminateNewDuplicatePHINodesN2(BB, FirstExistingPN) + : EliminateNewDuplicatePHINodesSet(BB, FirstExistingPN); +} + Align llvm::tryEnforceAlignment(Value *V, Align PrefAlign, const DataLayout &DL) { V = V->stripPointerCasts(); diff --git a/llvm/unittests/Transforms/Utils/LocalTest.cpp b/llvm/unittests/Transforms/Utils/LocalTest.cpp index 3c7a892c9d65a..b90294fe0d3f2 100644 --- a/llvm/unittests/Transforms/Utils/LocalTest.cpp +++ b/llvm/unittests/Transforms/Utils/LocalTest.cpp @@ -1362,3 +1362,203 @@ TEST(Local, ReplaceDbgVariableRecord) { // Teardown. RetInst->DebugMarker->eraseFromParent(); } + +bool EliminateNewDuplicatePHINodesN2(BasicBlock *BB, + BasicBlock::phi_iterator FirstExistingPN); +bool EliminateNewDuplicatePHINodesSet(BasicBlock *BB, + BasicBlock::phi_iterator FirstExistingPN); + +TEST(Local, RaceEliminateNewDuplicatePHINodes) { + GTEST_SKIP(); // Comment out to run this test manually. + using namespace std::chrono; + + LLVMContext C; + IRBuilder<> B(C); + std::unique_ptr F( + Function::Create(FunctionType::get(B.getVoidTy(), false), + GlobalValue::ExternalLinkage, "F")); + BasicBlock *Entry(BasicBlock::Create(C, "", F.get())); + BasicBlock *BB(BasicBlock::Create(C, "", F.get())); + BranchInst::Create(BB, Entry); + B.SetInsertPoint(BB); + auto *Ret = B.CreateRetVoid(); + B.SetInsertPoint(Ret); + + const unsigned NumPreds = 5; + for (unsigned Pass = 1; Pass < 64; ++Pass) { + auto *PHI = B.CreatePHI(Type::getInt32Ty(C), NumPreds); + for (unsigned I = 0; I < NumPreds; ++I) + PHI->addIncoming(B.getInt32(Pass), Entry); + + if (Pass < 2) + continue; + + auto FirstExistingPN = std::next(BB->phis().begin(), Pass / 2); + const unsigned NumRuns = 1000000; + + outs() << "Num phis: " << Pass; + auto Start = high_resolution_clock::now(); + for (unsigned Run = NumRuns; Run > 0; --Run) + EliminateNewDuplicatePHINodesSet(BB, FirstExistingPN); + auto End = high_resolution_clock::now(); + auto TH = duration_cast(End - Start).count(); + outs() << " H: " << TH; + + Start = high_resolution_clock::now(); + for (unsigned Run = NumRuns; Run > 0; --Run) + EliminateNewDuplicatePHINodesN2(BB, FirstExistingPN); + End = high_resolution_clock::now(); + auto TN2 = duration_cast(End - Start).count(); + + outs() << " N2: " << TN2 << " Diff: " << (TH - TN2) + << " Ratio: " << format("%.3f", ((double)TH / TN2)) << "\n"; + } +} + +// Helper to run both versions on the same input. +static void RunEliminateNewDuplicatePHINode( + const char *AsmText, + std::function + Check) { + LLVMContext C; + for (int Pass = 0; Pass < 2; ++Pass) { + std::unique_ptr M = parseIR(C, AsmText); + Function *F = M->getFunction("main"); + auto BBIt = std::find_if(F->begin(), F->end(), [](const BasicBlock &Block) { + return Block.getName() == "testbb"; + }); + ASSERT_NE(BBIt, F->end()); + Check(*BBIt, Pass == 0 ? EliminateNewDuplicatePHINodesSet + : EliminateNewDuplicatePHINodesN2); + } +} + +static BasicBlock::phi_iterator getPhiIt(BasicBlock &BB, unsigned Idx) { + return std::next(BB.phis().begin(), Idx); +} + +static PHINode *getPhi(BasicBlock &BB, unsigned Idx) { + return &*getPhiIt(BB, Idx); +} + +static int getNumPHIs(BasicBlock &BB) { + return std::distance(BB.phis().begin(), BB.phis().end()); +} + +TEST(Local, EliminateNewDuplicatePHINodes_OrderExisting) { + RunEliminateNewDuplicatePHINode(R"( + define void @main() { + entry: + br label %testbb + testbb: + %np0 = phi i32 [ 1, %entry ] + %np1 = phi i32 [ 1, %entry ] + %ep0 = phi i32 [ 1, %entry ] + %ep1 = phi i32 [ 1, %entry ] + %u = add i32 %np0, %np1 + ret void + } + )", [](BasicBlock &BB, auto *ENDPN) { + AssertingVH EP0 = getPhi(BB, 2); + AssertingVH EP1 = getPhi(BB, 3); + EXPECT_TRUE(ENDPN(&BB, getPhiIt(BB, 2))); + // Expected: + // %ep0 = phi i32 [ 1, %entry ] + // %ep1 = phi i32 [ 1, %entry ] + // %u = add i32 %ep0, %ep0 + EXPECT_EQ(getNumPHIs(BB), 2); + Instruction &Add = *BB.getFirstNonPHIIt(); + EXPECT_EQ(Add.getOperand(0), EP0); + EXPECT_EQ(Add.getOperand(1), EP0); + (void)EP1; // Avoid "unused" warning. + }); +} + +TEST(Local, EliminateNewDuplicatePHINodes_OrderNew) { + RunEliminateNewDuplicatePHINode(R"( + define void @main() { + entry: + br label %testbb + testbb: + %np0 = phi i32 [ 1, %entry ] + %np1 = phi i32 [ 1, %entry ] + %ep0 = phi i32 [ 2, %entry ] + %ep1 = phi i32 [ 2, %entry ] + %u = add i32 %np0, %np1 + ret void + } + )", [](BasicBlock &BB, auto *ENDPN) { + AssertingVH NP0 = getPhi(BB, 0); + AssertingVH EP0 = getPhi(BB, 2); + AssertingVH EP1 = getPhi(BB, 3); + EXPECT_TRUE(ENDPN(&BB, getPhiIt(BB, 2))); + // Expected: + // %np0 = phi i32 [ 1, %entry ] + // %ep0 = phi i32 [ 2, %entry ] + // %ep1 = phi i32 [ 2, %entry ] + // %u = add i32 %np0, %np0 + EXPECT_EQ(getNumPHIs(BB), 3); + Instruction &Add = *BB.getFirstNonPHIIt(); + EXPECT_EQ(Add.getOperand(0), NP0); + EXPECT_EQ(Add.getOperand(1), NP0); + (void)EP0; + (void)EP1; // Avoid "unused" warning. + }); +} + +TEST(Local, EliminateNewDuplicatePHINodes_NewRefExisting) { + RunEliminateNewDuplicatePHINode(R"( + define void @main() { + entry: + br label %testbb + testbb: + %np0 = phi i32 [ 1, %entry ], [ %ep0, %testbb ] + %np1 = phi i32 [ 1, %entry ], [ %ep1, %testbb ] + %ep0 = phi i32 [ 1, %entry ], [ %ep0, %testbb ] + %ep1 = phi i32 [ 1, %entry ], [ %ep1, %testbb ] + %u = add i32 %np0, %np1 + br label %testbb + } + )", [](BasicBlock &BB, auto *ENDPN) { + AssertingVH EP0 = getPhi(BB, 2); + AssertingVH EP1 = getPhi(BB, 3); + EXPECT_TRUE(ENDPN(&BB, getPhiIt(BB, 2))); + // Expected: + // %ep0 = phi i32 [ 1, %entry ], [ %ep0, %testbb ] + // %ep1 = phi i32 [ 1, %entry ], [ %ep1, %testbb ] + // %u = add i32 %ep0, %ep1 + EXPECT_EQ(getNumPHIs(BB), 2); + Instruction &Add = *BB.getFirstNonPHIIt(); + EXPECT_EQ(Add.getOperand(0), EP0); + EXPECT_EQ(Add.getOperand(1), EP1); + }); +} + +TEST(Local, EliminateNewDuplicatePHINodes_ExistingRefNew) { + RunEliminateNewDuplicatePHINode(R"( + define void @main() { + entry: + br label %testbb + testbb: + %np0 = phi i32 [ 1, %entry ], [ %np0, %testbb ] + %np1 = phi i32 [ 1, %entry ], [ %np1, %testbb ] + %ep0 = phi i32 [ 1, %entry ], [ %np0, %testbb ] + %ep1 = phi i32 [ 1, %entry ], [ %np1, %testbb ] + %u = add i32 %np0, %np1 + br label %testbb + } + )", [](BasicBlock &BB, auto *ENDPN) { + AssertingVH EP0 = getPhi(BB, 2); + AssertingVH EP1 = getPhi(BB, 3); + EXPECT_TRUE(ENDPN(&BB, getPhiIt(BB, 2))); + // Expected: + // %ep0 = phi i32 [ 1, %entry ], [ %ep0, %testbb ] + // %ep1 = phi i32 [ 1, %entry ], [ %ep1, %testbb ] + // %u = add i32 %ep0, %ep1 + EXPECT_EQ(getNumPHIs(BB), 2); + Instruction &Add = *BB.getFirstNonPHIIt(); + EXPECT_EQ(Add.getOperand(0), EP0); + EXPECT_EQ(Add.getOperand(1), EP1); + }); +} \ No newline at end of file