From 16bfc57b89bef3a0d5c2fef774d7eb8737305a3a Mon Sep 17 00:00:00 2001 From: EgorBo Date: Wed, 10 Jan 2024 02:50:12 +0100 Subject: [PATCH 1/4] Keep weights in Qmark --- src/coreclr/jit/compiler.h | 2 +- src/coreclr/jit/gentree.cpp | 3 +- src/coreclr/jit/gentree.h | 27 ++++++++++++++++-- src/coreclr/jit/importer.cpp | 19 +++++++++++-- src/coreclr/jit/morph.cpp | 55 +++++++++++++++++++++++++----------- 5 files changed, 82 insertions(+), 24 deletions(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 6883e062a6039c..39102d478dd374 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -5366,7 +5366,7 @@ class Compiler Statement* fgNewStmtFromTree(GenTree* tree, BasicBlock* block); Statement* fgNewStmtFromTree(GenTree* tree, const DebugInfo& di); - GenTree* fgGetTopLevelQmark(GenTree* expr, GenTree** ppDst = nullptr); + GenTreeQmark* fgGetTopLevelQmark(GenTree* expr, GenTree** ppDst = nullptr); bool fgExpandQmarkForCastInstOf(BasicBlock* block, Statement* stmt); bool fgExpandQmarkStmt(BasicBlock* block, Statement* stmt); void fgExpandQmarkNodes(); diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 0e1f86701138ea..86caf8654cd9e5 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -9380,7 +9380,8 @@ GenTree* Compiler::gtCloneExpr( case GT_QMARK: copy = new (this, GT_QMARK) - GenTreeQmark(tree->TypeGet(), tree->AsOp()->gtGetOp1(), tree->AsOp()->gtGetOp2()->AsColon()); + GenTreeQmark(tree->TypeGet(), tree->AsOp()->gtGetOp1(), tree->AsOp()->gtGetOp2()->AsColon(), + tree->AsQmark()->ThenNodeLikelihood()); break; case GT_BLK: diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 7788fbc519c3b8..f06b0cd4386716 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -5801,23 +5801,44 @@ struct GenTreeFptrVal : public GenTree /* gtQmark */ struct GenTreeQmark : public GenTreeOp { - GenTreeQmark(var_types type, GenTree* cond, GenTreeColon* colon) : GenTreeOp(GT_QMARK, type, cond, colon) + unsigned gtThenLikelihood; + + GenTreeQmark(var_types type, GenTree* cond, GenTreeColon* colon, unsigned thenLikelihood = 50) + : GenTreeOp(GT_QMARK, type, cond, colon), gtThenLikelihood(thenLikelihood) { // These must follow a specific form. assert((cond != nullptr) && cond->TypeIs(TYP_INT)); assert((colon != nullptr) && colon->OperIs(GT_COLON)); } - GenTree* ThenNode() + GenTree* ThenNode() const { return gtOp2->AsColon()->ThenNode(); } - GenTree* ElseNode() + GenTree* ElseNode() const { return gtOp2->AsColon()->ElseNode(); } + unsigned ThenNodeLikelihood() const + { + assert(gtThenLikelihood <= 100); + return gtThenLikelihood; + } + + unsigned ElseNodeLikelihood() const + { + assert(gtThenLikelihood <= 100); + return 100 - gtThenLikelihood; + } + + void SetThenNodeLikelihood(unsigned thenLikelihood) + { + assert(thenLikelihood <= 100); + gtThenLikelihood = thenLikelihood; + } + #if DEBUGGABLE_GENTREE GenTreeQmark() : GenTreeOp() { diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index fed5a6ad0f2d64..2feed548bf7699 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -5448,6 +5448,9 @@ GenTree* Compiler::impCastClassOrIsInstToTree( CORINFO_CLASS_HANDLE exactCls = NO_CLASS_HANDLE; + // By default, we assume it's 50/50 with the slow path. + unsigned fastPathLikelihood = 50; + // Legality check. // // Not all classclass/isinst operations can be inline expanded. @@ -5507,6 +5510,13 @@ GenTree* Compiler::impCastClassOrIsInstToTree( if (likelyClassCount > 0) { #ifdef DEBUG + for (UINT32 i = 0; i < likelyClassCount; i++) + { + const char* className = eeGetClassName((CORINFO_CLASS_HANDLE)likelyClasses[i].handle); + printf(" %u) %p (%s) [likelihood:%u%%]\n", i + 1, likelyClasses[i].handle, className, + likelyClasses[i].likelihood); + } + // Optional stress mode to pick a random known class, rather than // the most likely known class. if (JitConfig.JitRandomGuardedDevirtualization() != 0) @@ -5525,8 +5535,9 @@ GenTree* Compiler::impCastClassOrIsInstToTree( LikelyClassMethodRecord likelyClass = likelyClasses[0]; CORINFO_CLASS_HANDLE likelyCls = (CORINFO_CLASS_HANDLE)likelyClass.handle; - if ((likelyCls != NO_CLASS_HANDLE) && - (likelyClass.likelihood > (UINT32)JitConfig.JitGuardedDevirtualizationChainLikelihood())) + // if there is a dominating candidate with >= 40% likelihood, use it + const unsigned likelihoodMinThreshold = 40; + if ((likelyCls != NO_CLASS_HANDLE) && (likelyClass.likelihood > likelihoodMinThreshold)) { TypeCompareState castResult = info.compCompHnd->compareTypesForCast(likelyCls, pResolvedToken->hClass); @@ -5552,6 +5563,7 @@ GenTree* Compiler::impCastClassOrIsInstToTree( canExpandInline = true; partialExpand = true; exactCls = likelyCls; + fastPathLikelihood= likelyClass.likelihood; } } } @@ -5664,7 +5676,7 @@ GenTree* Compiler::impCastClassOrIsInstToTree( condTrue = gtNewIconNode(0, TYP_REF); } - GenTree* qmarkMT; + GenTreeQmark* qmarkMT; // // Generate first QMARK - COLON tree // @@ -5676,6 +5688,7 @@ GenTree* Compiler::impCastClassOrIsInstToTree( // temp = new (this, GT_COLON) GenTreeColon(TYP_REF, condTrue, condFalse); qmarkMT = gtNewQmarkNode(TYP_REF, condMT, temp->AsColon()); + qmarkMT->SetThenNodeLikelihood(fastPathLikelihood); if (isCastClass && isClassExact && condTrue->OperIs(GT_CALL)) { diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index fdf57289e74de0..fd7a3d77968735 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -14517,22 +14517,22 @@ void Compiler::fgPostExpandQmarkChecks() // Returns: // The GT_QMARK node, or nullptr if there is no top level qmark. // -GenTree* Compiler::fgGetTopLevelQmark(GenTree* expr, GenTree** ppDst /* = NULL */) +GenTreeQmark* Compiler::fgGetTopLevelQmark(GenTree* expr, GenTree** ppDst /* = NULL */) { if (ppDst != nullptr) { *ppDst = nullptr; } - GenTree* topQmark = nullptr; + GenTreeQmark* topQmark = nullptr; if (expr->gtOper == GT_QMARK) { - topQmark = expr; + topQmark = expr->AsQmark(); } else if (expr->OperIsLocalStore() && expr->AsLclVarCommon()->Data()->OperIs(GT_QMARK)) { - topQmark = expr->AsLclVarCommon()->Data(); + topQmark = expr->AsLclVarCommon()->Data()->AsQmark(); if (ppDst != nullptr) { @@ -14595,8 +14595,8 @@ bool Compiler::fgExpandQmarkForCastInstOf(BasicBlock* block, Statement* stmt) bool introducedThrow = false; GenTree* expr = stmt->GetRootNode(); - GenTree* dst = nullptr; - GenTree* qmark = fgGetTopLevelQmark(expr, &dst); + GenTree* dst = nullptr; + GenTreeQmark* qmark = fgGetTopLevelQmark(expr, &dst); noway_assert(dst != nullptr); assert(dst->OperIsLocalStore()); @@ -14613,11 +14613,13 @@ bool Compiler::fgExpandQmarkForCastInstOf(BasicBlock* block, Statement* stmt) GenTree* true2Expr; GenTree* false2Expr; + unsigned nestedQmarkElseLikelihood = 50; if (nestedQmark->gtOper == GT_QMARK) { - cond2Expr = nestedQmark->gtGetOp1(); - true2Expr = nestedQmark->gtGetOp2()->AsColon()->ThenNode(); - false2Expr = nestedQmark->gtGetOp2()->AsColon()->ElseNode(); + cond2Expr = nestedQmark->gtGetOp1(); + true2Expr = nestedQmark->gtGetOp2()->AsColon()->ThenNode(); + false2Expr = nestedQmark->gtGetOp2()->AsColon()->ElseNode(); + nestedQmarkElseLikelihood = nestedQmark->AsQmark()->ElseNodeLikelihood(); } else { @@ -14688,8 +14690,26 @@ bool Compiler::fgExpandQmarkForCastInstOf(BasicBlock* block, Statement* stmt) // Set the weights; some are guesses. asgBlock->inheritWeight(block); cond1Block->inheritWeight(block); + + // We only have likelihood for the fast path (and fallback), but we don't know + // how often we have null in the root QMARK (although, we might be able to guess it too) + // so leave 50/50 for now. Thus, we have: + // + // [weight 1.0] + // if (obj != null) + // { + // [weight 0.5] + // if (obj.GetType() == typeof(FastType)) + // { + // [weight 0.5 * ] + // } + // else + // { + // [weight 0.5 * <100 - likelihood of FastType>] + // } + // cond2Block->inheritWeightPercentage(cond1Block, 50); - helperBlock->inheritWeightPercentage(cond2Block, 50); + helperBlock->inheritWeightPercentage(cond2Block, nestedQmarkElseLikelihood); // Append cond1 as JTRUE to cond1Block GenTree* jmpTree = gtNewOperNode(GT_JTRUE, TYP_VOID, condExpr); @@ -14812,8 +14832,8 @@ bool Compiler::fgExpandQmarkStmt(BasicBlock* block, Statement* stmt) GenTree* expr = stmt->GetRootNode(); // Retrieve the Qmark node to be expanded. - GenTree* dst = nullptr; - GenTree* qmark = fgGetTopLevelQmark(expr, &dst); + GenTree* dst = nullptr; + GenTreeQmark* qmark = fgGetTopLevelQmark(expr, &dst); if (qmark == nullptr) { return false; @@ -14874,6 +14894,9 @@ bool Compiler::fgExpandQmarkStmt(BasicBlock* block, Statement* stmt) condBlock->inheritWeight(block); + // Make sure remainderBlock gets exactly the same weight as block after split + assert(condBlock->bbWeight == remainderBlock->bbWeight); + assert(block->KindIs(BBJ_ALWAYS)); block->SetTarget(condBlock); condBlock->SetTarget(elseBlock); @@ -14913,8 +14936,8 @@ bool Compiler::fgExpandQmarkStmt(BasicBlock* block, Statement* stmt) fgAddRefPred(thenBlock, condBlock); fgAddRefPred(remainderBlock, thenBlock); - thenBlock->inheritWeightPercentage(condBlock, 50); - elseBlock->inheritWeightPercentage(condBlock, 50); + thenBlock->inheritWeightPercentage(condBlock, qmark->ThenNodeLikelihood()); + elseBlock->inheritWeightPercentage(condBlock, qmark->ElseNodeLikelihood()); } else if (hasTrueExpr) { @@ -14931,7 +14954,7 @@ bool Compiler::fgExpandQmarkStmt(BasicBlock* block, Statement* stmt) thenBlock = elseBlock; elseBlock = nullptr; - thenBlock->inheritWeightPercentage(condBlock, 50); + thenBlock->inheritWeightPercentage(condBlock, qmark->ThenNodeLikelihood()); } else if (hasFalseExpr) { @@ -14944,7 +14967,7 @@ bool Compiler::fgExpandQmarkStmt(BasicBlock* block, Statement* stmt) condBlock->SetCond(remainderBlock, elseBlock); fgAddRefPred(remainderBlock, condBlock); - elseBlock->inheritWeightPercentage(condBlock, 50); + elseBlock->inheritWeightPercentage(condBlock, qmark->ElseNodeLikelihood()); } assert(condBlock->KindIs(BBJ_COND)); From 537ecc9ffcf22669247d140c8017f6e9e3a82a8b Mon Sep 17 00:00:00 2001 From: EgorBo Date: Wed, 10 Jan 2024 11:30:42 +0100 Subject: [PATCH 2/4] formatting --- src/coreclr/jit/importer.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 2feed548bf7699..4108f424e77598 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -5514,7 +5514,7 @@ GenTree* Compiler::impCastClassOrIsInstToTree( { const char* className = eeGetClassName((CORINFO_CLASS_HANDLE)likelyClasses[i].handle); printf(" %u) %p (%s) [likelihood:%u%%]\n", i + 1, likelyClasses[i].handle, className, - likelyClasses[i].likelihood); + likelyClasses[i].likelihood); } // Optional stress mode to pick a random known class, rather than @@ -5559,11 +5559,11 @@ GenTree* Compiler::impCastClassOrIsInstToTree( JITDUMP("Adding \"is %s (%X)\" check as a fast path for %s using PGO data.\n", eeGetClassName(likelyCls), likelyCls, isCastClass ? "castclass" : "isinst"); - reversedMTCheck = castResult == TypeCompareState::MustNot; - canExpandInline = true; - partialExpand = true; - exactCls = likelyCls; - fastPathLikelihood= likelyClass.likelihood; + reversedMTCheck = castResult == TypeCompareState::MustNot; + canExpandInline = true; + partialExpand = true; + exactCls = likelyCls; + fastPathLikelihood = likelyClass.likelihood; } } } From 7114cb35874279f8827080a77847fe2ef998ac64 Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Wed, 10 Jan 2024 19:03:16 +0100 Subject: [PATCH 3/4] Update importer.cpp --- src/coreclr/jit/importer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 4108f424e77598..9ce87d2bc0b98f 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -5513,7 +5513,7 @@ GenTree* Compiler::impCastClassOrIsInstToTree( for (UINT32 i = 0; i < likelyClassCount; i++) { const char* className = eeGetClassName((CORINFO_CLASS_HANDLE)likelyClasses[i].handle); - printf(" %u) %p (%s) [likelihood:%u%%]\n", i + 1, likelyClasses[i].handle, className, + JITDUMP(" %u) %p (%s) [likelihood:%u%%]\n", i + 1, likelyClasses[i].handle, className, likelyClasses[i].likelihood); } From b1ea263875ad4843db44fd21b556509f40ed3045 Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Wed, 10 Jan 2024 20:18:16 +0100 Subject: [PATCH 4/4] Update importer.cpp --- src/coreclr/jit/importer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 9ce87d2bc0b98f..2ba0643887987d 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -5514,7 +5514,7 @@ GenTree* Compiler::impCastClassOrIsInstToTree( { const char* className = eeGetClassName((CORINFO_CLASS_HANDLE)likelyClasses[i].handle); JITDUMP(" %u) %p (%s) [likelihood:%u%%]\n", i + 1, likelyClasses[i].handle, className, - likelyClasses[i].likelihood); + likelyClasses[i].likelihood); } // Optional stress mode to pick a random known class, rather than