Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Slightly improve profiled casts #96731

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jan 11, 2024
Merged

Slightly improve profiled casts #96731

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
16bfc57
Keep weights in Qmark
EgorBo Jan 10, 2024
537ecc9
formatting
EgorBo Jan 10, 2024
7114cb3
Update importer.cpp
EgorBo Jan 10, 2024
b1ea263
Update importer.cpp
EgorBo Jan 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/coreclr/jit/compiler.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5366,7 +5366,7 @@ class Compiler
Statement* fgNewStmtFromTree(GenTree* tree, BasicBlock* block);
Statement* fgNewStmtFromTree(GenTree* tree, const DebugInfo& di);

GenTree* fgGetTopLevelQmark(GenTree* expr, GenTree** ppDst = nullptr);
GenTreeQmark* fgGetTopLevelQmark(GenTree* expr, GenTree** ppDst = nullptr);
bool fgExpandQmarkForCastInstOf(BasicBlock* block, Statement* stmt);
bool fgExpandQmarkStmt(BasicBlock* block, Statement* stmt);
void fgExpandQmarkNodes();
Expand Down
3 changes: 2 additions & 1 deletion src/coreclr/jit/gentree.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9380,7 +9380,8 @@ GenTree* Compiler::gtCloneExpr(

case GT_QMARK:
copy = new (this, GT_QMARK)
GenTreeQmark(tree->TypeGet(), tree->AsOp()->gtGetOp1(), tree->AsOp()->gtGetOp2()->AsColon());
GenTreeQmark(tree->TypeGet(), tree->AsOp()->gtGetOp1(), tree->AsOp()->gtGetOp2()->AsColon(),
tree->AsQmark()->ThenNodeLikelihood());
break;

case GT_BLK:
Expand Down
27 changes: 24 additions & 3 deletions src/coreclr/jit/gentree.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5801,23 +5801,44 @@ struct GenTreeFptrVal : public GenTree
/* gtQmark */
struct GenTreeQmark : public GenTreeOp
{
GenTreeQmark(var_types type, GenTree* cond, GenTreeColon* colon) : GenTreeOp(GT_QMARK, type, cond, colon)
unsigned gtThenLikelihood;

GenTreeQmark(var_types type, GenTree* cond, GenTreeColon* colon, unsigned thenLikelihood = 50)
: GenTreeOp(GT_QMARK, type, cond, colon), gtThenLikelihood(thenLikelihood)
{
// These must follow a specific form.
assert((cond != nullptr) && cond->TypeIs(TYP_INT));
assert((colon != nullptr) && colon->OperIs(GT_COLON));
}

GenTree* ThenNode()
GenTree* ThenNode() const
{
return gtOp2->AsColon()->ThenNode();
}

GenTree* ElseNode()
GenTree* ElseNode() const
{
return gtOp2->AsColon()->ElseNode();
}

unsigned ThenNodeLikelihood() const
{
assert(gtThenLikelihood <= 100);
return gtThenLikelihood;
}

unsigned ElseNodeLikelihood() const
{
assert(gtThenLikelihood <= 100);
return 100 - gtThenLikelihood;
}

void SetThenNodeLikelihood(unsigned thenLikelihood)
{
assert(thenLikelihood <= 100);
gtThenLikelihood = thenLikelihood;
}

#if DEBUGGABLE_GENTREE
GenTreeQmark() : GenTreeOp()
{
Expand Down
27 changes: 20 additions & 7 deletions src/coreclr/jit/importer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5448,6 +5448,9 @@ GenTree* Compiler::impCastClassOrIsInstToTree(

CORINFO_CLASS_HANDLE exactCls = NO_CLASS_HANDLE;

// By default, we assume it's 50/50 with the slow path.
unsigned fastPathLikelihood = 50;

// Legality check.
//
// Not all classclass/isinst operations can be inline expanded.
Expand Down Expand Up @@ -5507,6 +5510,13 @@ GenTree* Compiler::impCastClassOrIsInstToTree(
if (likelyClassCount > 0)
{
#ifdef DEBUG
for (UINT32 i = 0; i < likelyClassCount; i++)
{
const char* className = eeGetClassName((CORINFO_CLASS_HANDLE)likelyClasses[i].handle);
JITDUMP(" %u) %p (%s) [likelihood:%u%%]\n", i + 1, likelyClasses[i].handle, className,
likelyClasses[i].likelihood);
}

// Optional stress mode to pick a random known class, rather than
// the most likely known class.
if (JitConfig.JitRandomGuardedDevirtualization() != 0)
Expand All @@ -5525,8 +5535,9 @@ GenTree* Compiler::impCastClassOrIsInstToTree(
LikelyClassMethodRecord likelyClass = likelyClasses[0];
CORINFO_CLASS_HANDLE likelyCls = (CORINFO_CLASS_HANDLE)likelyClass.handle;

if ((likelyCls != NO_CLASS_HANDLE) &&
(likelyClass.likelihood > (UINT32)JitConfig.JitGuardedDevirtualizationChainLikelihood()))
// if there is a dominating candidate with >= 40% likelihood, use it
const unsigned likelihoodMinThreshold = 40;
if ((likelyCls != NO_CLASS_HANDLE) && (likelyClass.likelihood > likelihoodMinThreshold))
{
TypeCompareState castResult =
info.compCompHnd->compareTypesForCast(likelyCls, pResolvedToken->hClass);
Expand All @@ -5548,10 +5559,11 @@ GenTree* Compiler::impCastClassOrIsInstToTree(
JITDUMP("Adding \"is %s (%X)\" check as a fast path for %s using PGO data.\n",
eeGetClassName(likelyCls), likelyCls, isCastClass ? "castclass" : "isinst");

reversedMTCheck = castResult == TypeCompareState::MustNot;
canExpandInline = true;
partialExpand = true;
exactCls = likelyCls;
reversedMTCheck = castResult == TypeCompareState::MustNot;
canExpandInline = true;
partialExpand = true;
exactCls = likelyCls;
fastPathLikelihood = likelyClass.likelihood;
}
}
}
Expand Down Expand Up @@ -5664,7 +5676,7 @@ GenTree* Compiler::impCastClassOrIsInstToTree(
condTrue = gtNewIconNode(0, TYP_REF);
}

GenTree* qmarkMT;
GenTreeQmark* qmarkMT;
//
// Generate first QMARK - COLON tree
//
Expand All @@ -5676,6 +5688,7 @@ GenTree* Compiler::impCastClassOrIsInstToTree(
//
temp = new (this, GT_COLON) GenTreeColon(TYP_REF, condTrue, condFalse);
qmarkMT = gtNewQmarkNode(TYP_REF, condMT, temp->AsColon());
qmarkMT->SetThenNodeLikelihood(fastPathLikelihood);

if (isCastClass && isClassExact && condTrue->OperIs(GT_CALL))
{
Expand Down
55 changes: 39 additions & 16 deletions src/coreclr/jit/morph.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14517,22 +14517,22 @@ void Compiler::fgPostExpandQmarkChecks()
// Returns:
// The GT_QMARK node, or nullptr if there is no top level qmark.
//
GenTree* Compiler::fgGetTopLevelQmark(GenTree* expr, GenTree** ppDst /* = NULL */)
GenTreeQmark* Compiler::fgGetTopLevelQmark(GenTree* expr, GenTree** ppDst /* = NULL */)
{
if (ppDst != nullptr)
{
*ppDst = nullptr;
}

GenTree* topQmark = nullptr;
GenTreeQmark* topQmark = nullptr;

if (expr->gtOper == GT_QMARK)
{
topQmark = expr;
topQmark = expr->AsQmark();
}
else if (expr->OperIsLocalStore() && expr->AsLclVarCommon()->Data()->OperIs(GT_QMARK))
{
topQmark = expr->AsLclVarCommon()->Data();
topQmark = expr->AsLclVarCommon()->Data()->AsQmark();

if (ppDst != nullptr)
{
Expand Down Expand Up @@ -14595,8 +14595,8 @@ bool Compiler::fgExpandQmarkForCastInstOf(BasicBlock* block, Statement* stmt)
bool introducedThrow = false;
GenTree* expr = stmt->GetRootNode();

GenTree* dst = nullptr;
GenTree* qmark = fgGetTopLevelQmark(expr, &dst);
GenTree* dst = nullptr;
GenTreeQmark* qmark = fgGetTopLevelQmark(expr, &dst);

noway_assert(dst != nullptr);
assert(dst->OperIsLocalStore());
Expand All @@ -14613,11 +14613,13 @@ bool Compiler::fgExpandQmarkForCastInstOf(BasicBlock* block, Statement* stmt)
GenTree* true2Expr;
GenTree* false2Expr;

unsigned nestedQmarkElseLikelihood = 50;
if (nestedQmark->gtOper == GT_QMARK)
{
cond2Expr = nestedQmark->gtGetOp1();
true2Expr = nestedQmark->gtGetOp2()->AsColon()->ThenNode();
false2Expr = nestedQmark->gtGetOp2()->AsColon()->ElseNode();
cond2Expr = nestedQmark->gtGetOp1();
true2Expr = nestedQmark->gtGetOp2()->AsColon()->ThenNode();
false2Expr = nestedQmark->gtGetOp2()->AsColon()->ElseNode();
nestedQmarkElseLikelihood = nestedQmark->AsQmark()->ElseNodeLikelihood();
}
else
{
Expand Down Expand Up @@ -14688,8 +14690,26 @@ bool Compiler::fgExpandQmarkForCastInstOf(BasicBlock* block, Statement* stmt)
// Set the weights; some are guesses.
asgBlock->inheritWeight(block);
cond1Block->inheritWeight(block);

// We only have likelihood for the fast path (and fallback), but we don't know
// how often we have null in the root QMARK (although, we might be able to guess it too)
// so leave 50/50 for now. Thus, we have:
//
// [weight 1.0]
// if (obj != null)
// {
// [weight 0.5]
// if (obj.GetType() == typeof(FastType))
// {
// [weight 0.5 * <likelihood of FastType>]
// }
// else
// {
// [weight 0.5 * <100 - likelihood of FastType>]
// }
//
cond2Block->inheritWeightPercentage(cond1Block, 50);
helperBlock->inheritWeightPercentage(cond2Block, 50);
helperBlock->inheritWeightPercentage(cond2Block, nestedQmarkElseLikelihood);

// Append cond1 as JTRUE to cond1Block
GenTree* jmpTree = gtNewOperNode(GT_JTRUE, TYP_VOID, condExpr);
Expand Down Expand Up @@ -14812,8 +14832,8 @@ bool Compiler::fgExpandQmarkStmt(BasicBlock* block, Statement* stmt)
GenTree* expr = stmt->GetRootNode();

// Retrieve the Qmark node to be expanded.
GenTree* dst = nullptr;
GenTree* qmark = fgGetTopLevelQmark(expr, &dst);
GenTree* dst = nullptr;
GenTreeQmark* qmark = fgGetTopLevelQmark(expr, &dst);
if (qmark == nullptr)
{
return false;
Expand Down Expand Up @@ -14874,6 +14894,9 @@ bool Compiler::fgExpandQmarkStmt(BasicBlock* block, Statement* stmt)

condBlock->inheritWeight(block);

// Make sure remainderBlock gets exactly the same weight as block after split
assert(condBlock->bbWeight == remainderBlock->bbWeight);

assert(block->KindIs(BBJ_ALWAYS));
block->SetTarget(condBlock);
condBlock->SetTarget(elseBlock);
Expand Down Expand Up @@ -14913,8 +14936,8 @@ bool Compiler::fgExpandQmarkStmt(BasicBlock* block, Statement* stmt)
fgAddRefPred(thenBlock, condBlock);
fgAddRefPred(remainderBlock, thenBlock);

thenBlock->inheritWeightPercentage(condBlock, 50);
elseBlock->inheritWeightPercentage(condBlock, 50);
thenBlock->inheritWeightPercentage(condBlock, qmark->ThenNodeLikelihood());
elseBlock->inheritWeightPercentage(condBlock, qmark->ElseNodeLikelihood());
}
else if (hasTrueExpr)
{
Expand All @@ -14931,7 +14954,7 @@ bool Compiler::fgExpandQmarkStmt(BasicBlock* block, Statement* stmt)
thenBlock = elseBlock;
elseBlock = nullptr;

thenBlock->inheritWeightPercentage(condBlock, 50);
thenBlock->inheritWeightPercentage(condBlock, qmark->ThenNodeLikelihood());
}
else if (hasFalseExpr)
{
Expand All @@ -14944,7 +14967,7 @@ bool Compiler::fgExpandQmarkStmt(BasicBlock* block, Statement* stmt)
condBlock->SetCond(remainderBlock, elseBlock);
fgAddRefPred(remainderBlock, condBlock);

elseBlock->inheritWeightPercentage(condBlock, 50);
elseBlock->inheritWeightPercentage(condBlock, qmark->ElseNodeLikelihood());
}

assert(condBlock->KindIs(BBJ_COND));
Expand Down