diff --git a/src/coreclr/jit/CMakeLists.txt b/src/coreclr/jit/CMakeLists.txt index 480bfdc045e16c..ec5d3ac2c00411 100644 --- a/src/coreclr/jit/CMakeLists.txt +++ b/src/coreclr/jit/CMakeLists.txt @@ -126,6 +126,7 @@ set( JIT_SOURCES hwintrinsic.cpp hostallocator.cpp ifconversion.cpp + runtimelookup.cpp indirectcalltransformer.cpp importercalls.cpp importer.cpp diff --git a/src/coreclr/jit/arraystack.h b/src/coreclr/jit/arraystack.h index eb8a17932ca63d..83a43c9432ba0e 100644 --- a/src/coreclr/jit/arraystack.h +++ b/src/coreclr/jit/arraystack.h @@ -10,7 +10,7 @@ class ArrayStack static const int builtinSize = 8; public: - ArrayStack(CompAllocator alloc, int initialCapacity = builtinSize) : m_alloc(alloc) + explicit ArrayStack(CompAllocator alloc, int initialCapacity = builtinSize) : m_alloc(alloc) { if (initialCapacity > builtinSize) { diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index 13177d52753961..fe8a524ce66b9f 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -1951,6 +1951,7 @@ void Compiler::compInit(ArenaAllocator* pAlloc, vnStore = nullptr; m_outlinedCompositeSsaNums = nullptr; m_nodeToLoopMemoryBlockMap = nullptr; + m_signatureToLookupInfoMap = nullptr; fgSsaPassesCompleted = 0; fgSsaChecksEnabled = false; fgVNPassesCompleted = 0; @@ -5000,6 +5001,9 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl DoPhase(this, PHASE_STRESS_SPLIT_TREE, &Compiler::StressSplitTree); #endif + // Expand runtime lookups (an optimization but we'd better run it in tier0 too) + DoPhase(this, PHASE_EXPAND_RTLOOKUPS, &Compiler::fgExpandRuntimeLookups); + // Insert GC Polls DoPhase(this, PHASE_INSERT_GC_POLLS, &Compiler::fgInsertGCPolls); diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 308bfe0789299c..7ab54332866fb3 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -4814,6 +4814,7 @@ class Compiler BasicBlock* fgSplitBlockAfterStatement(BasicBlock* curr, Statement* stmt); BasicBlock* fgSplitBlockAfterNode(BasicBlock* curr, GenTree* node); // for LIR BasicBlock* fgSplitEdge(BasicBlock* curr, BasicBlock* succ); + BasicBlock* fgSplitBlockBeforeTree(BasicBlock* block, Statement* stmt, GenTree* splitPoint, Statement** firstNewStmt, GenTree*** splitNodeUse); Statement* fgNewStmtFromTree(GenTree* tree, BasicBlock* block, const DebugInfo& di); Statement* fgNewStmtFromTree(GenTree* tree); @@ -4949,6 +4950,17 @@ class Compiler return m_nodeToLoopMemoryBlockMap; } + typedef JitHashTable, CORINFO_RUNTIME_LOOKUP> SignatureToLookupInfoMap; + SignatureToLookupInfoMap* m_signatureToLookupInfoMap; + SignatureToLookupInfoMap* GetSignatureToLookupInfoMap() + { + if (m_signatureToLookupInfoMap == nullptr) + { + m_signatureToLookupInfoMap = new (getAllocator()) SignatureToLookupInfoMap(getAllocator()); + } + return m_signatureToLookupInfoMap; + } + void optRecordLoopMemoryDependence(GenTree* tree, BasicBlock* block, ValueNum memoryVN); void optCopyLoopMemoryDependence(GenTree* fromTree, GenTree* toTree); @@ -5281,6 +5293,7 @@ class Compiler PhaseStatus StressSplitTree(); void SplitTreesRandomly(); void SplitTreesRemoveCommas(); + PhaseStatus fgExpandRuntimeLookups(); PhaseStatus fgInsertGCPolls(); BasicBlock* fgCreateGCPoll(GCPollType pollType, BasicBlock* block); @@ -7083,13 +7096,6 @@ class Compiler optMethodFlags |= OMF_HAS_EXPRUNTIMELOOKUP; } - void clearMethodHasExpRuntimeLookup() - { - optMethodFlags &= ~OMF_HAS_EXPRUNTIMELOOKUP; - } - - void addExpRuntimeLookupCandidate(GenTreeCall* call); - bool doesMethodHavePatchpoints() { return (optMethodFlags & OMF_HAS_PATCHPOINT) != 0; diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 94dcf70963a413..8cec0408b70902 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -1610,6 +1610,27 @@ inline void GenTree::BashToZeroConst(var_types type) } } +//------------------------------------------------------------------------ +// BashToLclVar: Bash node to a LCL_VAR. +// +// Arguments: +// comp - compiler object +// lclNum - the local's number +// +// Return Value: +// The bashed node. +// +inline GenTreeLclVar* GenTree::BashToLclVar(Compiler* comp, unsigned lclNum) +{ + LclVarDsc* varDsc = comp->lvaGetDesc(lclNum); + + ChangeOper(GT_LCL_VAR); + ChangeType(varDsc->lvNormalizeOnLoad() ? varDsc->TypeGet() : genActualType(varDsc)); + AsLclVar()->SetLclNum(lclNum); + + return AsLclVar(); +} + /***************************************************************************** * * Returns true if the node is of the "ovf" variety, for example, add.ovf.i1. diff --git a/src/coreclr/jit/compphases.h b/src/coreclr/jit/compphases.h index c3f96621e80e6f..dfcd9da0ee9374 100644 --- a/src/coreclr/jit/compphases.h +++ b/src/coreclr/jit/compphases.h @@ -92,6 +92,7 @@ CompPhaseNameMacro(PHASE_VN_BASED_DEAD_STORE_REMOVAL,"VN-based dead store remova CompPhaseNameMacro(PHASE_OPT_UPDATE_FLOW_GRAPH, "Update flow graph opt pass", false, -1, false) CompPhaseNameMacro(PHASE_COMPUTE_EDGE_WEIGHTS2, "Compute edge weights (2, false)",false, -1, false) CompPhaseNameMacro(PHASE_STRESS_SPLIT_TREE, "Stress gtSplitTree", false, -1, false) +CompPhaseNameMacro(PHASE_EXPAND_RTLOOKUPS, "Expand runtime lookups", false, -1, true) CompPhaseNameMacro(PHASE_INSERT_GC_POLLS, "Insert GC Polls", false, -1, true) CompPhaseNameMacro(PHASE_DETERMINE_FIRST_COLD_BLOCK, "Determine first cold block", false, -1, true) CompPhaseNameMacro(PHASE_RATIONALIZE, "Rationalize IR", false, -1, false) diff --git a/src/coreclr/jit/fgbasic.cpp b/src/coreclr/jit/fgbasic.cpp index 9c33405a67f0f1..0824a63f576e0f 100644 --- a/src/coreclr/jit/fgbasic.cpp +++ b/src/coreclr/jit/fgbasic.cpp @@ -4618,6 +4618,50 @@ BasicBlock* Compiler::fgSplitBlockAfterStatement(BasicBlock* curr, Statement* st return newBlock; } +//------------------------------------------------------------------------------ +// fgSplitBlockBeforeTree : Split the given block right before the given tree +// +// Arguments: +// block - The block containing the statement. +// stmt - The statement containing the tree. +// splitPoint - A tree inside the statement. +// firstNewStmt - [out] The first new statement that was introduced. +// [firstNewStmt..stmt) are the statements added by this function. +// splitNodeUse - The use of the tree to split at. +// +// Returns: +// The last block after split +// +// Notes: +// See comments in gtSplitTree +// +BasicBlock* Compiler::fgSplitBlockBeforeTree( + BasicBlock* block, Statement* stmt, GenTree* splitPoint, Statement** firstNewStmt, GenTree*** splitNodeUse) +{ + gtSplitTree(block, stmt, splitPoint, firstNewStmt, splitNodeUse); + + BasicBlockFlags originalFlags = block->bbFlags; + BasicBlock* prevBb = block; + + if (stmt == block->firstStmt()) + { + block = fgSplitBlockAtBeginning(prevBb); + } + else + { + assert(stmt->GetPrevStmt() != block->lastStmt()); + JITDUMP("Splitting " FMT_BB " after statement " FMT_STMT "\n", prevBb->bbNum, stmt->GetPrevStmt()->GetID()); + block = fgSplitBlockAfterStatement(prevBb, stmt->GetPrevStmt()); + } + + // We split a block, possibly, in the middle - we need to propagate some flags + prevBb->bbFlags = originalFlags & (~(BBF_SPLIT_LOST | BBF_LOOP_PREHEADER | BBF_RETLESS_CALL) | BBF_GC_SAFE_POINT); + block->bbFlags |= + originalFlags & (BBF_SPLIT_GAINED | BBF_IMPORTED | BBF_GC_SAFE_POINT | BBF_LOOP_PREHEADER | BBF_RETLESS_CALL); + + return block; +} + //------------------------------------------------------------------------------ // fgSplitBlockAfterNode - Split the given block, with all code after // the given node going into the second block. diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 5ab36443360b8f..059f2b369db3a3 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -1947,8 +1947,8 @@ struct GenTree template void BashToConst(T value, var_types type = TYP_UNDEF); - void BashToZeroConst(var_types type); + GenTreeLclVar* BashToLclVar(Compiler* comp, unsigned lclNum); #if NODEBASH_STATS static void RecordOperBashing(genTreeOps operOld, genTreeOps operNew); diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index e0860c3565026d..1fff9332a8e9c6 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -1780,15 +1780,36 @@ GenTree* Compiler::impRuntimeLookupToTree(CORINFO_RESOLVED_TOKEN* pResolvedToken } #endif - // Slot pointer - GenTree* slotPtrTree = ctxTree; - if (pRuntimeLookup->testForNull) { - slotPtrTree = impCloneExpr(ctxTree, &ctxTree, NO_CLASS_HANDLE, CHECK_SPILL_ALL, - nullptr DEBUGARG("impRuntimeLookup slot")); + assert(pRuntimeLookup->indirections != 0); + + // Call the helper + // - Setup argNode with the pointer to the signature returned by the lookup + GenTree* argNode = + gtNewIconEmbHndNode(pRuntimeLookup->signature, nullptr, GTF_ICON_GLOBAL_PTR, compileTimeHandle); + GenTreeCall* helperCall = gtNewHelperCallNode(pRuntimeLookup->helper, TYP_I_IMPL, ctxTree, argNode); + + // No need to perform CSE/hoisting for signature node - it is expected to end up in a rarely-taken block after + // "Expand runtime lookups" phase. + argNode->gtFlags |= GTF_DONT_CSE; + + // Leave a note that this method has runtime lookups we might want to expand (nullchecks, size checks) later. + // We can also consider marking current block as a runtime lookup holder to improve TP for Tier0 + impInlineRoot()->setMethodHasExpRuntimeLookup(); + helperCall->SetExpRuntimeLookup(); + if (!impInlineRoot()->GetSignatureToLookupInfoMap()->Lookup(pRuntimeLookup->signature)) + { + JITDUMP("Registering %p in SignatureToLookupInfoMap\n", pRuntimeLookup->signature) + impInlineRoot()->GetSignatureToLookupInfoMap()->Set(pRuntimeLookup->signature, *pRuntimeLookup); + } + unsigned callLclNum = lvaGrabTemp(true DEBUGARG("spilling helperCall")); + impAssignTempGen(callLclNum, helperCall); + return gtNewLclvNode(callLclNum, helperCall->TypeGet()); } + // Slot pointer + GenTree* slotPtrTree = ctxTree; GenTree* indOffTree = nullptr; GenTree* lastIndOfTree = nullptr; @@ -1834,107 +1855,47 @@ GenTree* Compiler::impRuntimeLookupToTree(CORINFO_RESOLVED_TOKEN* pResolvedToken } // No null test required - if (!pRuntimeLookup->testForNull) - { - if (pRuntimeLookup->indirections == 0) - { - return slotPtrTree; - } - - slotPtrTree = gtNewOperNode(GT_IND, TYP_I_IMPL, slotPtrTree); - slotPtrTree->gtFlags |= GTF_IND_NONFAULTING; - - if (!pRuntimeLookup->testForFixup) - { - return slotPtrTree; - } - - impSpillSideEffects(true, CHECK_SPILL_ALL DEBUGARG("bubbling QMark0")); - - unsigned slotLclNum = lvaGrabTemp(true DEBUGARG("impRuntimeLookup test")); - impAssignTempGen(slotLclNum, slotPtrTree, NO_CLASS_HANDLE, CHECK_SPILL_ALL, nullptr, impCurStmtDI); - - GenTree* slot = gtNewLclvNode(slotLclNum, TYP_I_IMPL); - // downcast the pointer to a TYP_INT on 64-bit targets - slot = impImplicitIorI4Cast(slot, TYP_INT); - // Use a GT_AND to check for the lowest bit and indirect if it is set - GenTree* test = gtNewOperNode(GT_AND, TYP_INT, slot, gtNewIconNode(1)); - GenTree* relop = gtNewOperNode(GT_EQ, TYP_INT, test, gtNewIconNode(0)); + assert(!pRuntimeLookup->testForNull); - // slot = GT_IND(slot - 1) - slot = gtNewLclvNode(slotLclNum, TYP_I_IMPL); - GenTree* add = gtNewOperNode(GT_ADD, TYP_I_IMPL, slot, gtNewIconNode(-1, TYP_I_IMPL)); - GenTree* indir = gtNewOperNode(GT_IND, TYP_I_IMPL, add); - indir->gtFlags |= GTF_IND_NONFAULTING; - indir->gtFlags |= GTF_IND_INVARIANT; - - slot = gtNewLclvNode(slotLclNum, TYP_I_IMPL); - GenTree* asg = gtNewAssignNode(slot, indir); - GenTreeColon* colon = new (this, GT_COLON) GenTreeColon(TYP_VOID, gtNewNothingNode(), asg); - GenTreeQmark* qmark = gtNewQmarkNode(TYP_VOID, relop, colon); - impAppendTree(qmark, CHECK_SPILL_NONE, impCurStmtDI); - - return gtNewLclvNode(slotLclNum, TYP_I_IMPL); + if (pRuntimeLookup->indirections == 0) + { + return slotPtrTree; } - assert(pRuntimeLookup->indirections != 0); - - impSpillSideEffects(true, CHECK_SPILL_ALL DEBUGARG("bubbling QMark1")); - - // Extract the handle - GenTree* handleForNullCheck = gtNewOperNode(GT_IND, TYP_I_IMPL, slotPtrTree); - handleForNullCheck->gtFlags |= GTF_IND_NONFAULTING; - - // Call the helper - // - Setup argNode with the pointer to the signature returned by the lookup - GenTree* argNode = gtNewIconEmbHndNode(pRuntimeLookup->signature, nullptr, GTF_ICON_GLOBAL_PTR, compileTimeHandle); - - GenTreeCall* helperCall = gtNewHelperCallNode(pRuntimeLookup->helper, TYP_I_IMPL, ctxTree, argNode); - - // Check for null and possibly call helper - GenTree* nullCheck = gtNewOperNode(GT_NE, TYP_INT, handleForNullCheck, gtNewIconNode(0, TYP_I_IMPL)); - GenTree* handleForResult = gtCloneExpr(handleForNullCheck); - - GenTree* result = nullptr; + slotPtrTree = gtNewOperNode(GT_IND, TYP_I_IMPL, slotPtrTree); + slotPtrTree->gtFlags |= GTF_IND_NONFAULTING; - if (pRuntimeLookup->sizeOffset != CORINFO_NO_SIZE_CHECK) + if (!pRuntimeLookup->testForFixup) { - // Dynamic dictionary expansion support + return slotPtrTree; + } - assert((lastIndOfTree != nullptr) && (pRuntimeLookup->indirections > 0)); + impSpillSideEffects(true, CHECK_SPILL_ALL DEBUGARG("bubbling QMark0")); - // sizeValue = dictionary[pRuntimeLookup->sizeOffset] - GenTreeIntCon* sizeOffset = gtNewIconNode(pRuntimeLookup->sizeOffset, TYP_I_IMPL); - GenTree* sizeValueOffset = gtNewOperNode(GT_ADD, TYP_I_IMPL, lastIndOfTree, sizeOffset); - GenTree* sizeValue = gtNewOperNode(GT_IND, TYP_I_IMPL, sizeValueOffset); - sizeValue->gtFlags |= GTF_IND_NONFAULTING; + unsigned slotLclNum = lvaGrabTemp(true DEBUGARG("impRuntimeLookup test")); + impAssignTempGen(slotLclNum, slotPtrTree, NO_CLASS_HANDLE, CHECK_SPILL_ALL, nullptr, impCurStmtDI); - // sizeCheck fails if sizeValue < pRuntimeLookup->offsets[i] - GenTree* offsetValue = gtNewIconNode(pRuntimeLookup->offsets[pRuntimeLookup->indirections - 1], TYP_I_IMPL); - GenTree* sizeCheck = gtNewOperNode(GT_LE, TYP_INT, sizeValue, offsetValue); + GenTree* slot = gtNewLclvNode(slotLclNum, TYP_I_IMPL); + // downcast the pointer to a TYP_INT on 64-bit targets + slot = impImplicitIorI4Cast(slot, TYP_INT); + // Use a GT_AND to check for the lowest bit and indirect if it is set + GenTree* test = gtNewOperNode(GT_AND, TYP_INT, slot, gtNewIconNode(1)); + GenTree* relop = gtNewOperNode(GT_EQ, TYP_INT, test, gtNewIconNode(0)); - // revert null check condition. - nullCheck->ChangeOperUnchecked(GT_EQ); + // slot = GT_IND(slot - 1) + slot = gtNewLclvNode(slotLclNum, TYP_I_IMPL); + GenTree* add = gtNewOperNode(GT_ADD, TYP_I_IMPL, slot, gtNewIconNode(-1, TYP_I_IMPL)); + GenTree* indir = gtNewOperNode(GT_IND, TYP_I_IMPL, add); + indir->gtFlags |= GTF_IND_NONFAULTING; + indir->gtFlags |= GTF_IND_INVARIANT; - // ((sizeCheck fails || nullCheck fails))) ? (helperCall : handle). - // Add checks and the handle as call arguments, indirect call transformer will handle this. - NewCallArg nullCheckArg = NewCallArg::Primitive(nullCheck); - NewCallArg sizeCheckArg = NewCallArg::Primitive(sizeCheck); - NewCallArg handleForResultArg = NewCallArg::Primitive(handleForResult); - helperCall->gtArgs.PushFront(this, nullCheckArg, sizeCheckArg, handleForResultArg); - result = helperCall; - addExpRuntimeLookupCandidate(helperCall); - } - else - { - GenTreeColon* colonNullCheck = new (this, GT_COLON) GenTreeColon(TYP_I_IMPL, handleForResult, helperCall); - result = gtNewQmarkNode(TYP_I_IMPL, nullCheck, colonNullCheck); - } + slot = gtNewLclvNode(slotLclNum, TYP_I_IMPL); + GenTree* asg = gtNewAssignNode(slot, indir); + GenTreeColon* colon = new (this, GT_COLON) GenTreeColon(TYP_VOID, gtNewNothingNode(), asg); + GenTreeQmark* qmark = gtNewQmarkNode(TYP_VOID, relop, colon); + impAppendTree(qmark, CHECK_SPILL_NONE, impCurStmtDI); - unsigned tmp = lvaGrabTemp(true DEBUGARG("spilling Runtime Lookup tree")); - - impAssignTempGen(tmp, result, CHECK_SPILL_NONE); - return gtNewLclvNode(tmp, TYP_I_IMPL); + return gtNewLclvNode(slotLclNum, TYP_I_IMPL); } struct RecursiveGuard @@ -14081,12 +14042,6 @@ methodPointerInfo* Compiler::impAllocateMethodPointerInfo(const CORINFO_RESOLVED return memory; } -void Compiler::addExpRuntimeLookupCandidate(GenTreeCall* call) -{ - setMethodHasExpRuntimeLookup(); - call->SetExpRuntimeLookup(); -} - //------------------------------------------------------------------------ // impIsClassExact: check if a class handle can only describe values // of exactly one class. diff --git a/src/coreclr/jit/indirectcalltransformer.cpp b/src/coreclr/jit/indirectcalltransformer.cpp index 1d21937119c11c..b62ffc4e5a6752 100644 --- a/src/coreclr/jit/indirectcalltransformer.cpp +++ b/src/coreclr/jit/indirectcalltransformer.cpp @@ -115,12 +115,6 @@ class IndirectCallTransformer transformer.Run(); count++; } - else if (compiler->doesMethodHaveExpRuntimeLookup() && ContainsExpRuntimeLookup(stmt)) - { - ExpRuntimeLookupTransformer transformer(compiler, block, stmt); - transformer.Run(); - count++; - } } return count; @@ -159,28 +153,6 @@ class IndirectCallTransformer return candidate->IsCall() && candidate->AsCall()->IsGuardedDevirtualizationCandidate(); } - //------------------------------------------------------------------------ - // ContainsExpRuntimeLookup: check if this statement contains a dictionary - // with dynamic dictionary expansion that we want to transform in CFG. - // - // Return Value: - // true if contains, false otherwise. - // - bool ContainsExpRuntimeLookup(Statement* stmt) - { - GenTree* candidate = stmt->GetRootNode(); - if (candidate->OperIs(GT_ASG)) - { - candidate = candidate->gtGetOp2(); - } - if (candidate->OperIs(GT_CALL)) - { - GenTreeCall* call = candidate->AsCall(); - return call->IsExpRuntimeLookup(); - } - return false; - } - class Transformer { public: @@ -1221,191 +1193,6 @@ class IndirectCallTransformer } }; - // Runtime lookup with dynamic dictionary expansion transformer, - // it expects helper runtime lookup call with additional arguments that are: - // result handle, nullCheck tree, sizeCheck tree. - // before: - // current block - // { - // previous statements - // transforming statement - // { - // ASG lclVar, call with GTF_CALL_M_EXP_RUNTIME_LOOKUP flag set and additional arguments. - // } - // subsequent statements - // } - // - // after: - // current block - // { - // previous statements - // } BBJ_NONE check block - // check block - // { - // jump to else if the handle fails size check - // } BBJ_COND check block2, else block - // check block2 - // { - // jump to else if the handle fails null check - // } BBJ_COND then block, else block - // then block - // { - // return handle - // } BBJ_ALWAYS remainder block - // else block - // { - // do a helper call - // } BBJ_NONE remainder block - // remainder block - // { - // subsequent statements - // } - // - class ExpRuntimeLookupTransformer final : public Transformer - { - public: - ExpRuntimeLookupTransformer(Compiler* compiler, BasicBlock* block, Statement* stmt) - : Transformer(compiler, block, stmt) - { - GenTreeOp* asg = stmt->GetRootNode()->AsOp(); - resultLclNum = asg->gtOp1->AsLclVar()->GetLclNum(); - origCall = GetCall(stmt); - checkBlock2 = nullptr; - } - - protected: - virtual const char* Name() override - { - return "ExpRuntimeLookup"; - } - - //------------------------------------------------------------------------ - // GetCall: find a call in a statement. - // - // Arguments: - // callStmt - the statement with the call inside. - // - // Return Value: - // call tree node pointer. - virtual GenTreeCall* GetCall(Statement* callStmt) override - { - GenTree* tree = callStmt->GetRootNode(); - assert(tree->OperIs(GT_ASG)); - GenTreeCall* call = tree->gtGetOp2()->AsCall(); - return call; - } - - //------------------------------------------------------------------------ - // ClearFlag: clear runtime exp lookup flag from the original call. - // - virtual void ClearFlag() override - { - origCall->ClearExpRuntimeLookup(); - } - - // FixupRetExpr: no action needed. - virtual void FixupRetExpr() override - { - } - - //------------------------------------------------------------------------ - // CreateCheck: create check blocks, that checks dictionary size and does null test. - // - virtual void CreateCheck() override - { - CallArg* nullCheck = origCall->gtArgs.GetArgByIndex(0); - CallArg* sizeCheck = origCall->gtArgs.GetArgByIndex(1); - origCall->gtArgs.Remove(nullCheck); - origCall->gtArgs.Remove(sizeCheck); - // The first argument is the handle now. - checkBlock = CreateAndInsertBasicBlock(BBJ_COND, currBlock); - - assert(sizeCheck->GetEarlyNode()->OperIs(GT_LE)); - GenTree* sizeJmpTree = compiler->gtNewOperNode(GT_JTRUE, TYP_VOID, sizeCheck->GetNode()); - Statement* sizeJmpStmt = compiler->fgNewStmtFromTree(sizeJmpTree, stmt->GetDebugInfo()); - compiler->fgInsertStmtAtEnd(checkBlock, sizeJmpStmt); - - checkBlock2 = CreateAndInsertBasicBlock(BBJ_COND, checkBlock); - assert(nullCheck->GetEarlyNode()->OperIs(GT_EQ)); - GenTree* nullJmpTree = compiler->gtNewOperNode(GT_JTRUE, TYP_VOID, nullCheck->GetNode()); - Statement* nullJmpStmt = compiler->fgNewStmtFromTree(nullJmpTree, stmt->GetDebugInfo()); - compiler->fgInsertStmtAtEnd(checkBlock2, nullJmpStmt); - } - - //------------------------------------------------------------------------ - // CreateThen: create then block, that is executed if the checks succeed. - // This simply returns the handle. - // - virtual void CreateThen() override - { - thenBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, checkBlock2); - - CallArg* resultHandle = origCall->gtArgs.GetArgByIndex(0); - // The first argument is the real first argument for the call now. - origCall->gtArgs.Remove(resultHandle); - - GenTree* asg = compiler->gtNewTempAssign(resultLclNum, resultHandle->GetNode()); - Statement* asgStmt = compiler->gtNewStmt(asg, stmt->GetDebugInfo()); - compiler->fgInsertStmtAtEnd(thenBlock, asgStmt); - } - - //------------------------------------------------------------------------ - // CreateElse: create else block, that is executed if the checks fail. - // - virtual void CreateElse() override - { - elseBlock = CreateAndInsertBasicBlock(BBJ_NONE, thenBlock); - GenTree* asg = compiler->gtNewTempAssign(resultLclNum, origCall); - Statement* asgStmt = compiler->gtNewStmt(asg, stmt->GetDebugInfo()); - compiler->fgInsertStmtAtEnd(elseBlock, asgStmt); - } - - //------------------------------------------------------------------------ - // SetWeights: set weights for new blocks. - // - virtual void SetWeights() override - { - remainderBlock->inheritWeight(currBlock); - checkBlock->inheritWeight(currBlock); - checkBlock2->inheritWeightPercentage(checkBlock, HIGH_PROBABILITY); - thenBlock->inheritWeightPercentage(currBlock, HIGH_PROBABILITY); - elseBlock->inheritWeightPercentage(currBlock, 100 - HIGH_PROBABILITY); - } - - //------------------------------------------------------------------------ - // ChainFlow: link new blocks into correct cfg. - // - virtual void ChainFlow() override - { - assert(compiler->fgPredsComputed); - - // currBlock - compiler->fgRemoveRefPred(remainderBlock, currBlock); - compiler->fgAddRefPred(checkBlock, currBlock); - - // checkBlock - checkBlock->bbJumpDest = elseBlock; - compiler->fgAddRefPred(elseBlock, checkBlock); - compiler->fgAddRefPred(checkBlock2, checkBlock); - - // checkBlock2 - checkBlock2->bbJumpDest = elseBlock; - compiler->fgAddRefPred(elseBlock, checkBlock2); - compiler->fgAddRefPred(thenBlock, checkBlock2); - - // thenBlock - thenBlock->bbJumpDest = remainderBlock; - compiler->fgAddRefPred(remainderBlock, thenBlock); - - // elseBlock - compiler->fgAddRefPred(remainderBlock, elseBlock); - } - - private: - BasicBlock* checkBlock2; - unsigned resultLclNum; - }; - Compiler* compiler; }; @@ -1424,7 +1211,6 @@ Compiler::fgWalkResult Compiler::fgDebugCheckForTransformableIndirectCalls(GenTr GenTreeCall* call = tree->AsCall(); assert(!call->IsFatPointerCandidate()); assert(!call->IsGuardedDevirtualizationCandidate()); - assert(!call->IsExpRuntimeLookup()); } return WALK_CONTINUE; } @@ -1436,7 +1222,6 @@ Compiler::fgWalkResult Compiler::fgDebugCheckForTransformableIndirectCalls(GenTr void Compiler::CheckNoTransformableIndirectCallsRemain() { assert(!doesMethodHaveFatPointer()); - assert(!doesMethodHaveExpRuntimeLookup()); for (BasicBlock* const block : Blocks()) { @@ -1460,7 +1245,7 @@ void Compiler::CheckNoTransformableIndirectCallsRemain() PhaseStatus Compiler::fgTransformIndirectCalls() { int count = 0; - if (doesMethodHaveFatPointer() || doesMethodHaveGuardedDevirtualization() || doesMethodHaveExpRuntimeLookup()) + if (doesMethodHaveFatPointer() || doesMethodHaveGuardedDevirtualization()) { IndirectCallTransformer indirectCallTransformer(this); count = indirectCallTransformer.Run(); @@ -1475,7 +1260,6 @@ PhaseStatus Compiler::fgTransformIndirectCalls() } clearMethodHasFatPointer(); - clearMethodHasExpRuntimeLookup(); } else { diff --git a/src/coreclr/jit/lclmorph.cpp b/src/coreclr/jit/lclmorph.cpp index 0a55c73d1272e4..80104f7674e6c3 100644 --- a/src/coreclr/jit/lclmorph.cpp +++ b/src/coreclr/jit/lclmorph.cpp @@ -1154,7 +1154,7 @@ class LocalAddressVisitor final : public GenTreeVisitor case IndirTransform::BitCast: indir->ChangeOper(GT_BITCAST); - lclNode = BashToLclVar(indir->gtGetOp1(), lclNum); + lclNode = indir->gtGetOp1()->BashToLclVar(m_compiler, lclNum); break; #ifdef FEATURE_HW_INTRINSICS @@ -1166,7 +1166,7 @@ class LocalAddressVisitor final : public GenTreeVisitor { GenTree* hwiNode = nullptr; var_types elementType = indir->TypeGet(); - lclNode = BashToLclVar(indir->gtGetOp1(), lclNum); + lclNode = indir->gtGetOp1()->BashToLclVar(m_compiler, lclNum); if (elementType == TYP_FLOAT) { @@ -1195,7 +1195,7 @@ class LocalAddressVisitor final : public GenTreeVisitor GenTree* hwiNode = nullptr; var_types elementType = indir->TypeGet(); - lclNode = BashToLclVar(indir, lclNum); + lclNode = indir->BashToLclVar(m_compiler, lclNum); GenTree* simdLclNode = m_compiler->gtNewLclvNode(lclNum, varDsc->TypeGet()); GenTree* elementNode = user->gtGetOp2(); @@ -1252,7 +1252,7 @@ class LocalAddressVisitor final : public GenTreeVisitor assert(genTypeSize(varDsc) >= genTypeSize(indir)); assert(!isDef); - lclNode = BashToLclVar(indir->gtGetOp1(), lclNum); + lclNode = indir->gtGetOp1()->BashToLclVar(m_compiler, lclNum); *val.Use() = m_compiler->gtNewCastNode(genActualType(indir), lclNode, false, indir->TypeGet()); break; @@ -1670,27 +1670,6 @@ class LocalAddressVisitor final : public GenTreeVisitor return (user == nullptr) || (user->OperIs(GT_COMMA) && (user->AsOp()->gtGetOp1() == node)); } - //------------------------------------------------------------------------ - // BashToLclVar: Bash node to a LCL_VAR. - // - // Arguments: - // node - the node to bash - // lclNum - the local's number - // - // Return Value: - // The bashed node. - // - GenTreeLclVar* BashToLclVar(GenTree* node, unsigned lclNum) - { - LclVarDsc* varDsc = m_compiler->lvaGetDesc(lclNum); - - node->ChangeOper(GT_LCL_VAR); - node->ChangeType(varDsc->lvNormalizeOnLoad() ? varDsc->TypeGet() : genActualType(varDsc)); - node->AsLclVar()->SetLclNum(lclNum); - - return node->AsLclVar(); - } - void SequenceLocal(GenTreeLclVarCommon* lcl) { if (m_sequencer != nullptr) diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index 00ab6f19b44fe5..614ca071c7ed41 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -1790,6 +1790,9 @@ void Lowering::LowerCall(GenTree* node) DISPTREERANGE(BlockRange(), call); JITDUMP("\n"); + // All runtime lookups are expected to be expanded in fgExpandRuntimeLookups + assert(!call->IsExpRuntimeLookup()); + call->ClearOtherRegs(); LowerArgsForCall(call); diff --git a/src/coreclr/jit/runtimelookup.cpp b/src/coreclr/jit/runtimelookup.cpp new file mode 100644 index 00000000000000..58e7d75ae4ba90 --- /dev/null +++ b/src/coreclr/jit/runtimelookup.cpp @@ -0,0 +1,424 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#include "jitpch.h" +#ifdef _MSC_VER +#pragma hdrstop +#endif + +// Obtain constant pointer from a tree +static void* GetConstantPointer(Compiler* comp, GenTree* tree) +{ + void* cns = nullptr; + if (tree->gtEffectiveVal()->IsCnsIntOrI()) + { + cns = (void*)tree->gtEffectiveVal()->AsIntCon()->IconValue(); + } + else if (comp->vnStore->IsVNConstant(tree->gtVNPair.GetLiberal())) + { + cns = (void*)comp->vnStore->CoercedConstantValue(tree->gtVNPair.GetLiberal()); + } + return cns; +} + +// Save expression to a local and append it as the last statement in exprBlock +static GenTree* SpillExpression(Compiler* comp, GenTree* expr, BasicBlock* exprBlock, DebugInfo& debugInfo) +{ + unsigned const tmpNum = comp->lvaGrabTemp(true DEBUGARG("spilling expr")); + Statement* asgStmt = comp->fgNewStmtAtEnd(exprBlock, comp->gtNewTempAssign(tmpNum, expr), debugInfo); + comp->gtSetStmtInfo(asgStmt); + comp->fgSetStmtSeq(asgStmt); + return comp->gtNewLclvNode(tmpNum, genActualType(expr)); +}; + +// Create block from the given tree +static BasicBlock* CreateBlockFromTree(Compiler* comp, + BasicBlock* insertAfter, + BBjumpKinds blockKind, + GenTree* tree, + DebugInfo& debugInfo, + bool updateSideEffects = false) +{ + // Fast-path basic block + BasicBlock* newBlock = comp->fgNewBBafter(blockKind, insertAfter, true); + newBlock->bbFlags |= BBF_INTERNAL; + Statement* stmt = comp->fgNewStmtFromTree(tree, debugInfo); + comp->fgInsertStmtAtEnd(newBlock, stmt); + newBlock->bbCodeOffs = insertAfter->bbCodeOffsEnd; + newBlock->bbCodeOffsEnd = insertAfter->bbCodeOffsEnd; + if (updateSideEffects) + { + comp->gtUpdateStmtSideEffects(stmt); + } + return newBlock; +} + +//------------------------------------------------------------------------------ +// fgExpandRuntimeLookups : partially expand runtime lookups helper calls +// to add a nullcheck [+ size check] and a fast path +// Returns: +// PhaseStatus indicating what, if anything, was changed. +// +// Notes: +// The runtime lookup itself is needed to access a handle in code shared between +// generic instantiations. The lookup depends on the typeContext which is only available at +// runtime, and not at compile - time. See ASCII block diagrams in comments below for +// better understanding how this phase expands runtime lookups. +// +PhaseStatus Compiler::fgExpandRuntimeLookups() +{ + PhaseStatus result = PhaseStatus::MODIFIED_NOTHING; + + if (!doesMethodHaveExpRuntimeLookup()) + { + // The method being compiled doesn't have expandable runtime lookups. If it does + // and doesMethodHaveExpRuntimeLookup() still returns false we'll assert in LowerCall + return result; + } + + // Find all calls with GTF_CALL_M_EXP_RUNTIME_LOOKUP flag + // We don't use Blocks() iterator here as we modify `block` variable + for (BasicBlock* block = fgFirstBB; block != nullptr; block = block->bbNext) + { + SCAN_BLOCK_AGAIN: + for (Statement* const stmt : block->Statements()) + { + if ((stmt->GetRootNode()->gtFlags & GTF_CALL) == 0) + { + // TP: Stmt has no calls - bail out + continue; + } + + for (GenTree* const tree : stmt->TreeList()) + { + // We only need calls with IsExpRuntimeLookup() flag + if (!tree->IsCall() || !tree->AsCall()->IsExpRuntimeLookup()) + { + continue; + } + assert(tree->IsHelperCall()); + JITDUMP("Expanding runtime lookup for [%06d] in " FMT_BB ":\n", dspTreeID(tree), block->bbNum) + DISPTREE(tree) + JITDUMP("\n") + + GenTreeCall* call = tree->AsCall(); + + // Clear ExpRuntimeLookup flag so we won't miss any runtime lookup that needs partial expansion + call->ClearExpRuntimeLookup(); + + if (call->IsTailCall()) + { + // It is very unlikely to happen and is impossible to represent in C# + continue; + } + + assert(call->gtArgs.CountArgs() == 2); + // The call has the following signature: + // + // type = call(genericCtx, signatureCns); + // + void* signature = GetConstantPointer(this, call->gtArgs.GetArgByIndex(1)->GetNode()); + if (signature == nullptr) + { + // Technically, it is possible (e.g. it was CSE'd and then VN was erased), but for Debug mode we + // want to catch such cases as we really don't want to emit just a fallback call - it's too slow + assert(!"can't restore signature argument value"); + continue; + } + + // Restore runtimeLookup using signature argument via a global dictionary + CORINFO_RUNTIME_LOOKUP runtimeLookup = {}; + const bool lookupFound = GetSignatureToLookupInfoMap()->Lookup(signature, &runtimeLookup); + assert(lookupFound); + + const bool needsSizeCheck = runtimeLookup.sizeOffset != CORINFO_NO_SIZE_CHECK; + if (needsSizeCheck) + { + JITDUMP("dynamic expansion, needs size check.\n") + } + + DebugInfo debugInfo = stmt->GetDebugInfo(); + + assert(runtimeLookup.indirections != 0); + assert(runtimeLookup.testForNull); + + // Split block right before the call tree + BasicBlock* prevBb = block; + GenTree** callUse = nullptr; + Statement* newFirstStmt = nullptr; + block = fgSplitBlockBeforeTree(block, stmt, call, &newFirstStmt, &callUse); + assert(prevBb != nullptr && block != nullptr); + + // Block ops inserted by the split need to be morphed here since we are after morph. + // We cannot morph stmt yet as we may modify it further below, and the morphing + // could invalidate callUse. + while ((newFirstStmt != nullptr) && (newFirstStmt != stmt)) + { + fgMorphStmtBlockOps(block, newFirstStmt); + newFirstStmt = newFirstStmt->GetNextStmt(); + } + + GenTreeLclVar* rtLookupLcl = nullptr; + + // Mostly for Tier0: if the current statement is ASG(LCL, RuntimeLookup) + // we can drop it and use that LCL as the destination + if (stmt->GetRootNode()->OperIs(GT_ASG)) + { + GenTree* lhs = stmt->GetRootNode()->gtGetOp1(); + GenTree* rhs = stmt->GetRootNode()->gtGetOp2(); + if (lhs->OperIs(GT_LCL_VAR) && rhs == *callUse) + { + rtLookupLcl = gtClone(lhs)->AsLclVar(); + fgRemoveStmt(block, stmt); + } + } + + // Grab a temp to store result (it's assigned from either fastPathBb or fallbackBb) + if (rtLookupLcl == nullptr) + { + // Define a local for the result + unsigned rtLookupLclNum = lvaGrabTemp(true DEBUGARG("runtime lookup")); + lvaTable[rtLookupLclNum].lvType = TYP_I_IMPL; + rtLookupLcl = gtNewLclvNode(rtLookupLclNum, call->TypeGet()); + + *callUse = gtClone(rtLookupLcl); + + fgMorphStmtBlockOps(block, stmt); + gtUpdateStmtSideEffects(stmt); + } + + GenTree* ctxTree = call->gtArgs.GetArgByIndex(0)->GetNode(); + GenTree* sigNode = call->gtArgs.GetArgByIndex(1)->GetNode(); + + // Prepare slotPtr tree (TODO: consider sharing this part with impRuntimeLookup) + GenTree* slotPtrTree = gtCloneExpr(ctxTree); + GenTree* indOffTree = nullptr; + GenTree* lastIndOfTree = nullptr; + for (WORD i = 0; i < runtimeLookup.indirections; i++) + { + if ((i == 1 && runtimeLookup.indirectFirstOffset) || (i == 2 && runtimeLookup.indirectSecondOffset)) + { + indOffTree = SpillExpression(this, slotPtrTree, prevBb, debugInfo); + slotPtrTree = gtCloneExpr(indOffTree); + } + + // The last indirection could be subject to a size check (dynamic dictionary expansion) + const bool isLastIndirectionWithSizeCheck = (i == runtimeLookup.indirections - 1) && needsSizeCheck; + if (i != 0) + { + slotPtrTree = gtNewOperNode(GT_IND, TYP_I_IMPL, slotPtrTree); + slotPtrTree->gtFlags |= GTF_IND_NONFAULTING; + if (!isLastIndirectionWithSizeCheck) + { + slotPtrTree->gtFlags |= GTF_IND_INVARIANT; + } + } + + if ((i == 1 && runtimeLookup.indirectFirstOffset) || (i == 2 && runtimeLookup.indirectSecondOffset)) + { + slotPtrTree = gtNewOperNode(GT_ADD, TYP_I_IMPL, indOffTree, slotPtrTree); + } + if (runtimeLookup.offsets[i] != 0) + { + if (isLastIndirectionWithSizeCheck) + { + lastIndOfTree = SpillExpression(this, slotPtrTree, prevBb, debugInfo); + slotPtrTree = gtCloneExpr(lastIndOfTree); + } + slotPtrTree = gtNewOperNode(GT_ADD, TYP_I_IMPL, slotPtrTree, + gtNewIconNode(runtimeLookup.offsets[i], TYP_I_IMPL)); + } + } + + // Non-dynamic expansion case (no size check): + // + // prevBb(BBJ_NONE): [weight: 1.0] + // ... + // + // nullcheckBb(BBJ_COND): [weight: 1.0] + // if (*fastPathValue == null) + // goto fallbackBb; + // + // fastPathBb(BBJ_ALWAYS): [weight: 0.8] + // rtLookupLcl = *fastPathValue; + // goto block; + // + // fallbackBb(BBJ_NONE): [weight: 0.2] + // rtLookupLcl = HelperCall(); + // + // block(...): [weight: 1.0] + // use(rtLookupLcl); + // + + // null-check basic block + GenTree* fastPathValue = gtNewOperNode(GT_IND, TYP_I_IMPL, gtCloneExpr(slotPtrTree)); + fastPathValue->gtFlags |= GTF_IND_NONFAULTING; + // Save dictionary slot to a local (to be used by fast path) + GenTree* fastPathValueClone = + opts.OptimizationEnabled() ? fgMakeMultiUse(&fastPathValue) : gtCloneExpr(fastPathValue); + GenTree* nullcheckOp = gtNewOperNode(GT_EQ, TYP_INT, fastPathValue, gtNewIconNode(0, TYP_I_IMPL)); + nullcheckOp->gtFlags |= GTF_RELOP_JMP_USED; + BasicBlock* nullcheckBb = + CreateBlockFromTree(this, prevBb, BBJ_COND, gtNewOperNode(GT_JTRUE, TYP_VOID, nullcheckOp), + debugInfo); + + // Fallback basic block + GenTree* asgFallbackValue = gtNewAssignNode(gtClone(rtLookupLcl), call); + BasicBlock* fallbackBb = + CreateBlockFromTree(this, nullcheckBb, BBJ_NONE, asgFallbackValue, debugInfo, true); + + // Fast-path basic block + GenTree* asgFastpathValue = gtNewAssignNode(gtClone(rtLookupLcl), fastPathValueClone); + BasicBlock* fastPathBb = + CreateBlockFromTree(this, nullcheckBb, BBJ_ALWAYS, asgFastpathValue, debugInfo); + + BasicBlock* sizeCheckBb = nullptr; + if (needsSizeCheck) + { + // Dynamic expansion case (sizeCheckBb is added and some preds are changed): + // + // prevBb(BBJ_NONE): [weight: 1.0] + // + // sizeCheckBb(BBJ_COND): [weight: 1.0] + // if (sizeValue <= offsetValue) + // goto fallbackBb; + // ... + // + // nullcheckBb(BBJ_COND): [weight: 0.8] + // if (*fastPathValue == null) + // goto fallbackBb; + // + // fastPathBb(BBJ_ALWAYS): [weight: 0.64] + // rtLookupLcl = *fastPathValue; + // goto block; + // + // fallbackBb(BBJ_NONE): [weight: 0.36] + // rtLookupLcl = HelperCall(); + // + // block(...): [weight: 1.0] + // use(rtLookupLcl); + // + + // sizeValue = dictionary[pRuntimeLookup->sizeOffset] + GenTreeIntCon* sizeOffset = gtNewIconNode(runtimeLookup.sizeOffset, TYP_I_IMPL); + assert(lastIndOfTree != nullptr); + GenTree* sizeValueOffset = gtNewOperNode(GT_ADD, TYP_I_IMPL, lastIndOfTree, sizeOffset); + GenTree* sizeValue = gtNewOperNode(GT_IND, TYP_I_IMPL, sizeValueOffset); + sizeValue->gtFlags |= GTF_IND_NONFAULTING; + + // sizeCheck fails if sizeValue <= pRuntimeLookup->offsets[i] + GenTree* offsetValue = + gtNewIconNode(runtimeLookup.offsets[runtimeLookup.indirections - 1], TYP_I_IMPL); + GenTree* sizeCheck = gtNewOperNode(GT_LE, TYP_INT, sizeValue, offsetValue); + sizeCheck->gtFlags |= GTF_RELOP_JMP_USED; + + GenTree* jtrue = gtNewOperNode(GT_JTRUE, TYP_VOID, sizeCheck); + sizeCheckBb = CreateBlockFromTree(this, prevBb, BBJ_COND, jtrue, debugInfo); + } + + // + // Update preds in all new blocks + // + fgRemoveRefPred(block, prevBb); + fgAddRefPred(block, fastPathBb); + fgAddRefPred(block, fallbackBb); + nullcheckBb->bbJumpDest = fallbackBb; + fastPathBb->bbJumpDest = block; + + if (needsSizeCheck) + { + // sizeCheckBb is the first block after prevBb + fgAddRefPred(sizeCheckBb, prevBb); + // sizeCheckBb flows into nullcheckBb in case if the size check passes + fgAddRefPred(nullcheckBb, sizeCheckBb); + // fallbackBb is reachable from both nullcheckBb and sizeCheckBb + fgAddRefPred(fallbackBb, nullcheckBb); + fgAddRefPred(fallbackBb, sizeCheckBb); + // fastPathBb is only reachable from successful nullcheckBb + fgAddRefPred(fastPathBb, nullcheckBb); + // sizeCheckBb fails - jump to fallbackBb + sizeCheckBb->bbJumpDest = fallbackBb; + } + else + { + // nullcheckBb is the first block after prevBb + fgAddRefPred(nullcheckBb, prevBb); + // No size check, nullcheckBb jumps to fast path + fgAddRefPred(fastPathBb, nullcheckBb); + // fallbackBb is only reachable from nullcheckBb (jump destination) + fgAddRefPred(fallbackBb, nullcheckBb); + } + + // + // Re-distribute weights (see '[weight: X]' on the diagrams above) + // TODO: consider marking fallbackBb as rarely-taken + // + block->inheritWeight(prevBb); + if (needsSizeCheck) + { + sizeCheckBb->inheritWeight(prevBb); + // 80% chance we pass nullcheck + nullcheckBb->inheritWeightPercentage(sizeCheckBb, 80); + // 64% (0.8 * 0.8) chance we pass both nullcheck and sizecheck + fastPathBb->inheritWeightPercentage(nullcheckBb, 80); + // 100-64=36% chance we fail either nullcheck or sizecheck + fallbackBb->inheritWeightPercentage(sizeCheckBb, 36); + } + else + { + nullcheckBb->inheritWeight(prevBb); + // 80% chance we pass nullcheck + fastPathBb->inheritWeightPercentage(nullcheckBb, 80); + // 20% chance we fail nullcheck (TODO: Consider making it cold (0%)) + fallbackBb->inheritWeightPercentage(nullcheckBb, 20); + } + + // + // Update loop info if loop table is known to be valid + // + if (optLoopTableValid && prevBb->bbNatLoopNum != BasicBlock::NOT_IN_LOOP) + { + nullcheckBb->bbNatLoopNum = prevBb->bbNatLoopNum; + fastPathBb->bbNatLoopNum = prevBb->bbNatLoopNum; + fallbackBb->bbNatLoopNum = prevBb->bbNatLoopNum; + if (needsSizeCheck) + { + sizeCheckBb->bbNatLoopNum = prevBb->bbNatLoopNum; + } + // Update lpBottom after block split + if (optLoopTable[prevBb->bbNatLoopNum].lpBottom == prevBb) + { + optLoopTable[prevBb->bbNatLoopNum].lpBottom = block; + } + } + + // All blocks are expected to be in the same EH region + assert(BasicBlock::sameEHRegion(prevBb, block)); + assert(BasicBlock::sameEHRegion(prevBb, nullcheckBb)); + assert(BasicBlock::sameEHRegion(prevBb, fastPathBb)); + if (needsSizeCheck) + { + assert(BasicBlock::sameEHRegion(prevBb, sizeCheckBb)); + } + + // Scan current block again, the current call will be ignored because of ClearExpRuntimeLookup. + // We don't try to re-use expansions for the same lookups in the current block here - CSE is responsible + // for that + result = PhaseStatus::MODIFIED_EVERYTHING; + + // We've modified the graph and the current "block" might still have more runtime lookups + goto SCAN_BLOCK_AGAIN; + } + } + } + + if (result == PhaseStatus::MODIFIED_EVERYTHING) + { + if (opts.OptimizationEnabled()) + { + fgReorderBlocks(/* useProfileData */ false); + fgUpdateChangedFlowGraph(FlowGraphUpdates::COMPUTE_BASICS); + } + } + return result; +}