diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index 76b30f063f0205..510cb750378d0f 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -5031,6 +5031,8 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl optLoopTableValid = false; optLoopsRequirePreHeaders = false; + DoPhase(this, PHASE_RATIONALIZE_ASSIGNMENTS, &Compiler::fgRationalizeAssignments); + #ifdef DEBUG DoPhase(this, PHASE_STRESS_SPLIT_TREE, &Compiler::StressSplitTree); #endif @@ -5069,8 +5071,6 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl // DoPhase(this, PHASE_DETERMINE_FIRST_COLD_BLOCK, &Compiler::fgDetermineFirstColdBlock); - DoPhase(this, PHASE_RATIONALIZE_ASSIGNMENTS, &Compiler::fgRationalizeAssignments); - #ifdef DEBUG // Stash the current estimate of the function's size if necessary. if (verbose) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 4420a7662a36cc..4fc53728d118c4 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -2536,6 +2536,7 @@ class Compiler void* compileTimeHandle); GenTreeLclVar* gtNewLclvNode(unsigned lnum, var_types type DEBUGARG(IL_OFFSET offs = BAD_IL_OFFSET)); + GenTreeLclVar* gtNewLclVarNode(unsigned lclNum, var_types type = TYP_UNDEF); GenTreeLclVar* gtNewLclLNode(unsigned lnum, var_types type DEBUGARG(IL_OFFSET offs = BAD_IL_OFFSET)); GenTreeLclFld* gtNewLclVarAddrNode(unsigned lclNum, var_types type = TYP_I_IMPL); @@ -2838,6 +2839,12 @@ class Compiler GenTreeIndir* gtNewIndir(var_types typ, GenTree* addr, GenTreeFlags indirFlags = GTF_EMPTY); + GenTreeBlk* gtNewStoreBlkNode( + ClassLayout* layout, GenTree* addr, GenTree* data, GenTreeFlags indirFlags = GTF_EMPTY); + + GenTreeStoreInd* gtNewStoreIndNode( + var_types type, GenTree* addr, GenTree* data, GenTreeFlags indirFlags = GTF_EMPTY); + GenTree* gtNewLoadValueNode( var_types type, ClassLayout* layout, GenTree* addr, GenTreeFlags indirFlags = GTF_EMPTY); @@ -2851,6 +2858,14 @@ class Compiler return gtNewLoadValueNode(type, nullptr, addr, indirFlags); } + GenTree* gtNewStoreValueNode( + var_types type, ClassLayout* layout, GenTree* addr, GenTree* data, GenTreeFlags indirFlags = GTF_EMPTY); + + GenTree* gtNewStoreValueNode(ClassLayout* layout, GenTree* addr, GenTree* data, GenTreeFlags indirFlags = GTF_EMPTY) + { + return gtNewStoreValueNode(layout->GetType(), layout, addr, data, indirFlags); + } + GenTree* gtNewNullCheck(GenTree* addr, BasicBlock* basicBlock); var_types gtTypeForNullCheck(GenTree* tree); @@ -2962,6 +2977,9 @@ class Compiler void gtPrepareCost(GenTree* tree); bool gtIsLikelyRegVar(GenTree* tree); + void gtGetLclVarNodeCost(GenTreeLclVar* node, int* pCostEx, int* pCostSz, bool isLikelyRegVar); + void gtGetLclFldNodeCost(GenTreeLclFld* node, int* pCostEx, int* pCostSz); + bool gtGetIndNodeCost(GenTreeIndir* node, int* pCostEx, int* pCostSz); // Returns true iff the secondNode can be swapped with firstNode. bool gtCanSwapOrder(GenTree* firstNode, GenTree* secondNode); diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 72e5553ebb7c09..4e92a57cb006b9 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -1230,6 +1230,7 @@ inline void GenTree::SetOper(genTreeOps oper, ValueNumberUpdate vnUpdate) break; #endif case GT_LCL_FLD: + case GT_STORE_LCL_FLD: AsLclFld()->SetLclOffs(0); AsLclFld()->SetLayout(nullptr); break; diff --git a/src/coreclr/jit/decomposelongs.cpp b/src/coreclr/jit/decomposelongs.cpp index c7e16f5e7551e8..e1b36b3f97e02c 100644 --- a/src/coreclr/jit/decomposelongs.cpp +++ b/src/coreclr/jit/decomposelongs.cpp @@ -514,10 +514,8 @@ GenTree* DecomposeLongs::DecomposeStoreLclFld(LIR::Use& use) loStore->gtFlags |= GTF_VAR_USEASG; // Create the store for the upper half of the GT_LONG and insert it after the low store. - GenTreeLclFld* hiStore = m_compiler->gtNewLclFldNode(loStore->GetLclNum(), TYP_INT, loStore->GetLclOffs() + 4); - hiStore->SetOper(GT_STORE_LCL_FLD); - hiStore->gtOp1 = value->gtOp2; - hiStore->gtFlags |= (GTF_VAR_DEF | GTF_VAR_USEASG); + GenTreeLclFld* hiStore = + m_compiler->gtNewStoreLclFldNode(loStore->GetLclNum(), TYP_INT, loStore->GetLclOffs() + 4, value->gtOp2); Range().InsertAfter(loStore, hiStore); diff --git a/src/coreclr/jit/fgopt.cpp b/src/coreclr/jit/fgopt.cpp index 526cea6443d4ba..8c4b9d665cf213 100644 --- a/src/coreclr/jit/fgopt.cpp +++ b/src/coreclr/jit/fgopt.cpp @@ -3445,24 +3445,14 @@ bool Compiler::fgBlockEndFavorsTailDuplication(BasicBlock* block, unsigned lclNu while (count < limit) { count++; + unsigned storeLclNum; GenTree* const tree = stmt->GetRootNode(); - if (tree->OperIs(GT_ASG) && !tree->OperIsBlkOp()) + if (tree->OperIsStoreLcl(&storeLclNum) && (storeLclNum == lclNum) && !tree->OperIsBlkOp()) { - GenTree* const op1 = tree->AsOp()->gtOp1; - - if (op1->IsLocal()) + GenTree* const data = tree->Data(); + if (data->OperIsArrLength() || data->OperIsConst() || data->OperIsCompare()) { - const unsigned op1LclNum = op1->AsLclVarCommon()->GetLclNum(); - - if (op1LclNum == lclNum) - { - GenTree* const op2 = tree->AsOp()->gtOp2; - - if (op2->OperIsArrLength() || op2->OperIsConst() || op2->OperIsCompare()) - { - return true; - } - } + return true; } } @@ -3616,36 +3606,29 @@ bool Compiler::fgBlockIsGoodTailDuplicationCandidate(BasicBlock* target, unsigne // Otherwise check the first stmt. // Verify the branch is just a simple local compare. // + unsigned storeLclNum; GenTree* const firstTree = firstStmt->GetRootNode(); - - if (firstTree->gtOper != GT_ASG) - { - return false; - } - - GenTree* const lhs = firstTree->AsOp()->gtOp1; - if (!lhs->OperIs(GT_LCL_VAR)) + if (!firstTree->OperIsStoreLclVar(&storeLclNum)) { return false; } - const unsigned lhsLcl = lhs->AsLclVarCommon()->GetLclNum(); - if (lhsLcl != *lclNum) + if (storeLclNum != *lclNum) { return false; } // Could allow unary here too... // - GenTree* const rhs = firstTree->AsOp()->gtOp2; - if (!rhs->OperIsBinary()) + GenTree* const data = firstTree->Data(); + if (!data->OperIsBinary()) { return false; } // op1 must be some combinations of casts of local or constant // (or unary) - op1 = rhs->AsOp()->gtOp1; + op1 = data->AsOp()->gtOp1; while (op1->gtOper == GT_CAST) { op1 = op1->AsOp()->gtOp1; @@ -3658,7 +3641,7 @@ bool Compiler::fgBlockIsGoodTailDuplicationCandidate(BasicBlock* target, unsigne // op2 must be some combinations of casts of local or constant // (or unary) - op2 = rhs->AsOp()->gtOp2; + op2 = data->AsOp()->gtOp2; // A binop may not actually have an op2. // diff --git a/src/coreclr/jit/flowgraph.cpp b/src/coreclr/jit/flowgraph.cpp index 50d9052e1f92bd..e521469fb99771 100644 --- a/src/coreclr/jit/flowgraph.cpp +++ b/src/coreclr/jit/flowgraph.cpp @@ -2979,6 +2979,7 @@ GenTree* Compiler::fgRationalizeAssignment(GenTreeOp* assignment) { store->SetReverseOp(); } + store->CopyRawCosts(assignment); if (storeOp == GT_STOREIND) { diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index ae7f23375a2f82..e2fbbd8f23d419 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -730,12 +730,15 @@ ClassLayout* GenTree::GetLayout(Compiler* compiler) const switch (OperGet()) { case GT_LCL_VAR: + case GT_STORE_LCL_VAR: return compiler->lvaGetDesc(AsLclVar())->GetLayout(); case GT_LCL_FLD: + case GT_STORE_LCL_FLD: return AsLclFld()->GetLayout(); case GT_BLK: + case GT_STORE_BLK: return AsBlk()->GetLayout(); case GT_COMMA: @@ -4128,7 +4131,7 @@ void Compiler::gtPrepareCost(GenTree* tree) bool Compiler::gtIsLikelyRegVar(GenTree* tree) { - if (tree->gtOper != GT_LCL_VAR) + if (!tree->OperIsScalarLocal()) { return false; } @@ -4171,6 +4174,176 @@ bool Compiler::gtIsLikelyRegVar(GenTree* tree) return true; } +//------------------------------------------------------------------------ +// gtGetLclVarNodeCost: Calculate the cost for a local variable node. +// +// Used for both uses and defs. Only the node's own cost is calculated. +// +// Arguments: +// node - The node in question +// pCostEx - [out] parameter for the execution cost +// pCostSz - [out] parameter for the size cost +// isLikelyRegVar - Is the local likely to end up enregistered +// +void Compiler::gtGetLclVarNodeCost(GenTreeLclVar* node, int* pCostEx, int* pCostSz, bool isLikelyRegVar) +{ + int costEx; + int costSz; + if (isLikelyRegVar) + { + costEx = 1; + costSz = 1; + /* Sign-extend and zero-extend are more expensive to load */ + if (lvaGetDesc(node)->lvNormalizeOnLoad()) + { + costEx += 1; + costSz += 1; + } + } + else + { + costEx = IND_COST_EX; + costSz = 2; + + // Some types are more expensive to load than others. + if (varTypeIsSmall(node)) + { + costEx += 1; + costSz += 1; + } + else if (node->TypeIs(TYP_STRUCT)) + { + costEx += 2 * IND_COST_EX; + costSz += 2 * 2; + } + } +#if defined(TARGET_AMD64) + // increase costSz for floating point locals + if (varTypeIsFloating(node)) + { + costSz += 1; + if (!isLikelyRegVar) + { + costSz += 1; + } + } +#endif + + *pCostEx = costEx; + *pCostSz = costSz; +} + +//------------------------------------------------------------------------ +// gtGetLclFldNodeCost: Calculate the cost for a local field node. +// +// Used for both uses and defs. Only the node's own cost is calculated. +// +// Arguments: +// node - The node in question +// pCostEx - [out] parameter for the execution cost +// pCostSz - [out] parameter for the size cost +// +void Compiler::gtGetLclFldNodeCost(GenTreeLclFld* node, int* pCostEx, int* pCostSz) +{ + int costEx = IND_COST_EX; + int costSz = 4; + if (varTypeIsSmall(node->TypeGet())) + { + costEx += 1; + costSz += 1; + } + else if (node->TypeIs(TYP_STRUCT)) + { + costEx += 2 * IND_COST_EX; + costSz += 2 * 2; + } + + *pCostEx = costEx; + *pCostSz = costSz; +} + +//------------------------------------------------------------------------ +// gtGetLclFldNodeCost: Calculate the cost for a primitive indir node. +// +// Used for both loads and stores. +// +// Arguments: +// node - The node in question +// pCostEx - [out] parameter for the execution cost +// pCostSz - [out] parameter for the size cost +// +// Return Value: +// Whether the cost calculated includes that of the node's address. +// +bool Compiler::gtGetIndNodeCost(GenTreeIndir* node, int* pCostEx, int* pCostSz) +{ + assert(node->isIndir()); + + // Indirections have a costEx of IND_COST_EX. + int costEx = IND_COST_EX; + int costSz = 2; + bool includesAddrCost = false; + + // If we have to sign-extend or zero-extend, bump the cost. + if (varTypeIsSmall(node)) + { + costEx += 1; + costSz += 1; + } + +#ifdef TARGET_ARM + if (varTypeIsFloating(node)) + { + costSz += 2; + } +#endif // TARGET_ARM + + // Can we form an addressing mode with this indirection? + GenTree* addr = node->Addr(); + + if (addr->gtEffectiveVal()->OperIs(GT_ADD)) + { + // See if we can form a complex addressing mode. + bool doAddrMode = true; + + // TODO-1stClassStructs: delete once IND nodes are no more. + if (node->TypeGet() == TYP_STRUCT) + { + doAddrMode = false; + } +#ifdef TARGET_ARM64 + if (node->IsVolatile()) + { + // For volatile store/loads when address is contained we always emit `dmb` + // if it's not - we emit one-way barriers i.e. ldar/stlr + doAddrMode = false; + } +#endif // TARGET_ARM64 + if (doAddrMode && gtMarkAddrMode(addr, &costEx, &costSz, node->TypeGet())) + { + includesAddrCost = true; + } + } + else if (gtIsLikelyRegVar(addr)) + { + // Indirection of an enregister LCL_VAR, don't increase costEx/costSz. + includesAddrCost = true; + } +#ifdef TARGET_XARCH + else if (addr->IsCnsIntOrI()) + { + // Indirection of a CNS_INT, subtract 1 from costEx makes costEx 3 for x86 and 4 for amd64. + costEx += (addr->GetCostEx() - 1); + costSz += addr->GetCostSz(); + includesAddrCost = true; + } +#endif + + *pCostEx = costEx; + *pCostSz = costSz; + return includesAddrCost; +} + //------------------------------------------------------------------------ // gtCanSwapOrder: Returns true iff the secondNode can be swapped with firstNode. // @@ -5063,61 +5236,12 @@ unsigned Compiler::gtSetEvalOrder(GenTree* tree) case GT_LCL_VAR: level = 1; - if (gtIsLikelyRegVar(tree)) - { - costEx = 1; - costSz = 1; - /* Sign-extend and zero-extend are more expensive to load */ - if (lvaTable[tree->AsLclVar()->GetLclNum()].lvNormalizeOnLoad()) - { - costEx += 1; - costSz += 1; - } - } - else - { - costEx = IND_COST_EX; - costSz = 2; - - // Some types are more expensive to load than others. - if (varTypeIsSmall(tree->TypeGet())) - { - costEx += 1; - costSz += 1; - } - else if (tree->TypeIs(TYP_STRUCT)) - { - costEx += 2 * IND_COST_EX; - costSz += 2 * 2; - } - } -#if defined(TARGET_AMD64) - // increase costSz for floating point locals - if (isflt) - { - costSz += 1; - if (!gtIsLikelyRegVar(tree)) - { - costSz += 1; - } - } -#endif + gtGetLclVarNodeCost(tree->AsLclVar(), &costEx, &costSz, gtIsLikelyRegVar(tree)); break; case GT_LCL_FLD: - level = 1; - costEx = IND_COST_EX; - costSz = 4; - if (varTypeIsSmall(tree->TypeGet())) - { - costEx += 1; - costSz += 1; - } - else if (tree->TypeIs(TYP_STRUCT)) - { - costEx += 2 * IND_COST_EX; - costSz += 2 * 2; - } + level = 1; + gtGetLclFldNodeCost(tree->AsLclFld(), &costEx, &costSz); break; case GT_LCL_ADDR: @@ -5145,9 +5269,6 @@ unsigned Compiler::gtSetEvalOrder(GenTree* tree) if (kind & GTK_SMPOP) { - int lvlb; // preference for op2 - unsigned lvl2; // scratch variable - GenTree* op1 = tree->AsOp()->gtOp1; GenTree* op2 = tree->gtGetOp2IfPresent(); @@ -5398,78 +5519,52 @@ unsigned Compiler::gtSetEvalOrder(GenTree* tree) break; case GT_IND: - { - /* An indirection should always have a non-zero level. - * Only constant leaf nodes have level 0. - */ - + // An indirection should always have a non-zero level. + // Only constant leaf nodes have level 0. if (level == 0) { level = 1; } - /* Indirections have a costEx of IND_COST_EX */ - costEx = IND_COST_EX; - costSz = 2; - - /* If we have to sign-extend or zero-extend, bump the cost */ - if (varTypeIsSmall(tree->TypeGet())) + if (gtGetIndNodeCost(tree->AsIndir(), &costEx, &costSz)) { - costEx += 1; - costSz += 1; + goto DONE; } + break; -#ifdef TARGET_ARM - if (isflt) + case GT_STORE_LCL_VAR: + if (gtIsLikelyRegVar(tree)) { - costSz += 2; + // Store to an enregistered local. + costEx = op1->GetCostEx(); + costSz = max(3, op1->GetCostSz()); // 3 is an estimate for a reg-reg move. + goto DONE; } -#endif // TARGET_ARM - // Can we form an addressing mode with this indirection? - GenTree* addr = op1->gtEffectiveVal(); + gtGetLclVarNodeCost(tree->AsLclVar(), &costEx, &costSz, /* isLikelyRegVar */ false); + goto SET_LCL_STORE_COSTS; - if (addr->OperIs(GT_ADD)) - { - // See if we can form a complex addressing mode. - bool doAddrMode = true; + case GT_STORE_LCL_FLD: + gtGetLclFldNodeCost(tree->AsLclFld(), &costEx, &costSz); - // TODO-1stClassStructs: Always do this, but first make sure it's done in Lowering as well. - if (tree->TypeGet() == TYP_STRUCT) - { - doAddrMode = false; - } -#ifdef TARGET_ARM64 - if (tree->AsIndir()->IsVolatile()) - { - // For volatile store/loads when address is contained we always emit `dmb` - // if it's not - we emit one-way barriers i.e. ldar/stlr - doAddrMode = false; - } -#endif // TARGET_ARM64 - if (doAddrMode && gtMarkAddrMode(op1, &costEx, &costSz, tree->TypeGet())) - { - goto DONE; - } - } - else if (gtIsLikelyRegVar(op1)) + SET_LCL_STORE_COSTS: + costEx += 1; + costSz += 1; +#ifdef TARGET_ARM + if (isflt) { - /* Indirection of an enregister LCL_VAR, don't increase costEx/costSz */ - goto DONE; + costSz += 2; } -#ifdef TARGET_XARCH - else if (op1->IsCnsIntOrI()) +#endif // TARGET_ARM +#ifndef TARGET_64BIT + if (tree->TypeIs(TYP_LONG)) { - // Indirection of a CNS_INT, subtract 1 from costEx - // makes costEx 3 for x86 and 4 for amd64 - // - costEx += (op1->GetCostEx() - 1); - costSz += op1->GetCostSz(); - goto DONE; + // Operations on longs are more expensive. + costEx += 3; + costSz += 3; } -#endif - } - break; +#endif // !TARGET_64BIT + break; default: break; @@ -5480,8 +5575,9 @@ unsigned Compiler::gtSetEvalOrder(GenTree* tree) } /* Binary operator - check for certain special cases */ - - lvlb = 0; + bool includeOp1Cost = true; + bool includeOp2Cost = true; + bool allowReversal = true; /* Default Binary ops have a cost of 1,1 */ costEx = 1; @@ -5501,6 +5597,9 @@ unsigned Compiler::gtSetEvalOrder(GenTree* tree) costSz += 3; } #endif + level = gtSetEvalOrder(op1); + unsigned lvl2 = gtSetEvalOrder(op2); + switch (oper) { case GT_MOD: @@ -5536,7 +5635,7 @@ unsigned Compiler::gtSetEvalOrder(GenTree* tree) costSz += 2; // Encourage the first operand to be evaluated (into EAX/EDX) first */ - lvlb -= 3; + level += 3; } break; @@ -5566,7 +5665,7 @@ unsigned Compiler::gtSetEvalOrder(GenTree* tree) { /* We use imulEAX for TYP_LONG and overflow multiplications */ // Encourage the first operand to be evaluated (into EAX/EDX) first */ - lvlb -= 4; + level += 4; /* The 64-bit imul instruction costs more */ costEx += 4; @@ -5593,21 +5692,94 @@ unsigned Compiler::gtSetEvalOrder(GenTree* tree) } break; + case GT_LSH: + case GT_RSH: + case GT_RSZ: + case GT_ROL: + case GT_ROR: + /* Variable sized shifts are more expensive and use REG_SHIFT */ + + if (!op2->IsCnsIntOrI()) + { + costEx += 3; +#ifndef TARGET_64BIT + // Variable sized LONG shifts require the use of a helper call + // + if (tree->gtType == TYP_LONG) + { + level += 5; + lvl2 += 5; + costEx += 3 * IND_COST_EX; + costSz += 4; + } +#endif // !TARGET_64BIT + } + break; + + case GT_EQ: + case GT_NE: + case GT_LT: + case GT_LE: + case GT_GE: + case GT_GT: + /* Float compares remove both operands from the FP stack */ + /* Also FP comparison uses EAX for flags */ + + if (varTypeIsFloating(op1->TypeGet())) + { + level++; + lvl2++; + } + if ((tree->gtFlags & GTF_RELOP_JMP_USED) == 0) + { + /* Using a setcc instruction is more expensive */ + costEx += 3; + } + break; + case GT_BOUNDS_CHECK: costEx = 4; // cmp reg,reg and jae throw (not taken) costSz = 7; // jump to cold section + // Bounds check nodes used to not be binary, thus GTF_REVERSE_OPS was not enabled for them. This + // condition preserves that behavior. Additionally, CQ analysis shows that enabling GTF_REVERSE_OPS + // for these nodes leads to mixed results at best. + allowReversal = false; break; - case GT_COMMA: + case GT_INTRINSIC: + // We do not swap operand execution order for intrinsics that are implemented by user calls because + // of trickiness around ensuring the execution order does not change during rationalization. + if (IsIntrinsicImplementedByUserCall(tree->AsIntrinsic()->gtIntrinsicName)) + { + allowReversal = false; + } - /* Comma tosses the result of the left operand */ - gtSetEvalOrder(op1); - level = gtSetEvalOrder(op2); + switch (tree->AsIntrinsic()->gtIntrinsicName) + { + case NI_System_Math_Atan2: + case NI_System_Math_Pow: + // These math intrinsics are actually implemented by user calls. Increase the + // Sethi 'complexity' by two to reflect the argument register requirement. + level += 2; + break; + + case NI_System_Math_Max: + case NI_System_Math_Min: + level++; + break; + + default: + assert(!"Unknown binary GT_INTRINSIC operator"); + break; + } + break; + case GT_COMMA: + /* Comma tosses the result of the left operand */ + level = lvl2; /* GT_COMMA cost is the sum of op1 and op2 costs */ costEx = (op1->GetCostEx() + op2->GetCostEx()); costSz = (op1->GetCostSz() + op2->GetCostSz()); - goto DONE; case GT_INDEX_ADDR: @@ -5615,171 +5787,143 @@ unsigned Compiler::gtSetEvalOrder(GenTree* tree) costSz = 9; // jump to cold section break; - case GT_ASG: - /* Assignments need a bit of special handling */ - /* Process the target */ - level = gtSetEvalOrder(op1); + case GT_STORE_BLK: + // We estimate the cost of a GT_STORE_BLK to be two stores. + costEx += 2 * IND_COST_EX; + costSz += 2 * 2; + goto SET_INDIRECT_STORE_ORDER; - if (gtIsLikelyRegVar(op1)) + case GT_STOREIND: + if (gtGetIndNodeCost(tree->AsIndir(), &costEx, &costSz)) { - assert(lvlb == 0); - lvl2 = gtSetEvalOrder(op2); - - /* Assignment to an enregistered LCL_VAR */ - costEx = op2->GetCostEx(); - costSz = max(3, op2->GetCostSz()); // 3 is an estimate for a reg-reg assignment - goto DONE_OP1_AFTER_COST; + includeOp1Cost = false; } - goto DONE_OP1; - - default: - break; - } - - /* Process the sub-operands */ - level = gtSetEvalOrder(op1); - if (lvlb < 0) - { - level -= lvlb; // lvlb is negative, so this increases level - lvlb = 0; - } - - DONE_OP1: - assert(lvlb >= 0); - lvl2 = gtSetEvalOrder(op2) + lvlb; - - costEx += (op1->GetCostEx() + op2->GetCostEx()); - costSz += (op1->GetCostSz() + op2->GetCostSz()); - - DONE_OP1_AFTER_COST: - - bool bReverseInAssignment = false; - if (oper == GT_ASG && (!optValnumCSE_phase || optCSE_canSwap(op1, op2))) - { - switch (op1->OperGet()) - { - case GT_IND: - case GT_BLK: + costEx += 1; + costSz += 1; +#ifndef TARGET_64BIT + if (tree->TypeIs(TYP_LONG)) { - // In an ASG(IND(addr), ...), the "IND" is a pure syntactical element, - // the actual indirection will only be realized at the point of the ASG - // itself. As such, we can discard any side effects "induced" by it in - // this logic. - // - GenTree* op1Addr = op1->AsIndir()->Addr(); + // Operations on longs are more expensive. + costEx += 3; + costSz += 3; + } +#endif // !TARGET_64BIT - if (op1Addr->IsInvariant()) + SET_INDIRECT_STORE_ORDER: + // TODO-ASG-Cleanup: this logic emulates the ASG case below. See how of much of it can be deleted. + if (!optValnumCSE_phase || optCSE_canSwap(op1, op2)) + { + if (op1->IsInvariant()) { - bReverseInAssignment = true; - tree->gtFlags |= GTF_REVERSE_OPS; + allowReversal = false; + tree->SetReverseOp(); break; } - if (op1Addr->gtFlags & GTF_ALL_EFFECT) + if ((op1->gtFlags & GTF_ALL_EFFECT) != 0) { break; } // In case op2 assigns to a local var that is used in op1, we have to evaluate op1 first. - if (op2->gtFlags & GTF_ASG) + if ((op2->gtFlags & GTF_ASG) != 0) { + // TODO-ASG-Cleanup: move this guard to "gtCanSwapOrder". + allowReversal = false; break; } // If op2 is simple then evaluate op1 first - if (op2->OperKind() & GTK_LEAF) + if (op2->OperIsLeaf()) { break; } - } - // fall through and set GTF_REVERSE_OPS - FALLTHROUGH; - - case GT_LCL_VAR: - case GT_LCL_FLD: - // Note that for local stores, liveness depends on seeing the defs and - // uses in correct order, and so we MUST reverse the ASG in that case. - bReverseInAssignment = true; - tree->gtFlags |= GTF_REVERSE_OPS; - break; - - default: - break; - } - } - else if (GenTree::OperIsCompare(oper)) - { - /* Float compares remove both operands from the FP stack */ - /* Also FP comparison uses EAX for flags */ - - if (varTypeIsFloating(op1->TypeGet())) - { - level++; - lvl2++; - } - if ((tree->gtFlags & GTF_RELOP_JMP_USED) == 0) - { - /* Using a setcc instruction is more expensive */ - costEx += 3; - } - } - - /* Check for other interesting cases */ - - switch (oper) - { - case GT_LSH: - case GT_RSH: - case GT_RSZ: - case GT_ROL: - case GT_ROR: - /* Variable sized shifts are more expensive and use REG_SHIFT */ - if (!op2->IsCnsIntOrI()) - { - costEx += 3; -#ifndef TARGET_64BIT - // Variable sized LONG shifts require the use of a helper call - // - if (tree->gtType == TYP_LONG) - { - level += 5; - lvl2 += 5; - costEx += 3 * IND_COST_EX; - costSz += 4; - } -#endif // !TARGET_64BIT + allowReversal = false; + tree->SetReverseOp(); } break; - case GT_INTRINSIC: + case GT_ASG: + /* Assignments need a bit of special handling */ + if (gtIsLikelyRegVar(op1)) + { + /* Assignment to an enregistered LCL_VAR */ + costEx = op2->GetCostEx(); + costSz = max(3, op2->GetCostSz()); // 3 is an estimate for a reg-reg assignment + includeOp1Cost = false; + includeOp2Cost = false; + } - switch (tree->AsIntrinsic()->gtIntrinsicName) + if (!optValnumCSE_phase || optCSE_canSwap(op1, op2)) { - case NI_System_Math_Atan2: - case NI_System_Math_Pow: - // These math intrinsics are actually implemented by user calls. - // Increase the Sethi 'complexity' by two to reflect the argument - // register requirement. - level += 2; - break; + switch (op1->OperGet()) + { + case GT_IND: + case GT_BLK: + { + // In an ASG(IND(addr), ...), the "IND" is a pure syntactical element, + // the actual indirection will only be realized at the point of the ASG + // itself. As such, we can discard any side effects "induced" by it in + // this logic. + // + GenTree* op1Addr = op1->AsIndir()->Addr(); + + if (op1Addr->IsInvariant()) + { + allowReversal = false; + tree->gtFlags |= GTF_REVERSE_OPS; + break; + } + if (op1Addr->gtFlags & GTF_ALL_EFFECT) + { + break; + } - case NI_System_Math_Max: - case NI_System_Math_Min: - level++; - break; + // In case op2 assigns to a local var that is used in op1, we have to evaluate op1 first. + if (op2->gtFlags & GTF_ASG) + { + break; + } - default: - assert(!"Unknown binary GT_INTRINSIC operator"); - break; - } + // If op2 is simple then evaluate op1 first + if (op2->OperKind() & GTK_LEAF) + { + break; + } + } + // fall through and set GTF_REVERSE_OPS + FALLTHROUGH; + + case GT_LCL_VAR: + case GT_LCL_FLD: + // Note that for local stores, liveness depends on seeing the defs and + // uses in correct order, and so we MUST reverse the ASG in that case. + allowReversal = false; + tree->gtFlags |= GTF_REVERSE_OPS; + break; + default: + break; + } + } break; default: break; } + if (includeOp1Cost) + { + costEx += op1->GetCostEx(); + costSz += op1->GetCostSz(); + } + if (includeOp2Cost) + { + costEx += op2->GetCostEx(); + costSz += op2->GetCostSz(); + } + /* We need to evaluate constants later as many places in codegen can't handle op1 being a constant. This is normally naturally enforced as constants have the least level of 0. However, @@ -5793,50 +5937,12 @@ unsigned Compiler::gtSetEvalOrder(GenTree* tree) lvl2++; } - /* We try to swap operands if the second one is more expensive */ - bool tryToSwap; - GenTree* opA; - GenTree* opB; - - if (tree->gtFlags & GTF_REVERSE_OPS) + // We try to swap operands if the second one is more expensive. + // Don't swap anything if we're in linear order; we're really just interested in the costs. + bool tryToSwap = false; + if (allowReversal && (fgOrder != FGOrderLinear)) { - opA = op2; - opB = op1; - } - else - { - opA = op1; - opB = op2; - } - - if (fgOrder == FGOrderLinear) - { - // Don't swap anything if we're in linear order; we're really just interested in the costs. - tryToSwap = false; - } - else if (bReverseInAssignment) - { - // Assignments are special, we want the reverseops flags - // so if possible it was set above. - tryToSwap = false; - } - else if ((oper == GT_INTRINSIC) && IsIntrinsicImplementedByUserCall(tree->AsIntrinsic()->gtIntrinsicName)) - { - // We do not swap operand execution order for intrinsics that are implemented by user calls - // because of trickiness around ensuring the execution order does not change during rationalization. - tryToSwap = false; - } - else if (oper == GT_BOUNDS_CHECK) - { - // Bounds check nodes used to not be binary, thus GTF_REVERSE_OPS was - // not enabled for them. This condition preserves that behavior. - // Additionally, CQ analysis shows that enabling GTF_REVERSE_OPS - // for these nodes leads to mixed results at best. - tryToSwap = false; - } - else - { - if (tree->gtFlags & GTF_REVERSE_OPS) + if (tree->IsReverseOp()) { tryToSwap = (level > lvl2); } @@ -5846,8 +5952,7 @@ unsigned Compiler::gtSetEvalOrder(GenTree* tree) } // Try to force extra swapping when in the stress mode: - if (compStressCompile(STRESS_REVERSE_FLAG, 60) && ((tree->gtFlags & GTF_REVERSE_OPS) == 0) && - !op2->OperIsConst()) + if (compStressCompile(STRESS_REVERSE_FLAG, 60) && !tree->IsReverseOp() && !op2->OperIsConst()) { tryToSwap = true; } @@ -5855,7 +5960,7 @@ unsigned Compiler::gtSetEvalOrder(GenTree* tree) if (tryToSwap) { - bool canSwap = gtCanSwapOrder(opA, opB); + bool canSwap = tree->IsReverseOp() ? gtCanSwapOrder(op2, op1) : gtCanSwapOrder(op1, op2); if (canSwap) { @@ -5895,30 +6000,17 @@ unsigned Compiler::gtSetEvalOrder(GenTree* tree) break; default: - - /* Mark the operand's evaluation order to be swapped */ - if (tree->gtFlags & GTF_REVERSE_OPS) - { - tree->gtFlags &= ~GTF_REVERSE_OPS; - } - else - { - tree->gtFlags |= GTF_REVERSE_OPS; - } - + // Mark the operand's evaluation order to be swapped. + tree->gtFlags ^= GTF_REVERSE_OPS; break; } } } /* Swap the level counts */ - if (tree->gtFlags & GTF_REVERSE_OPS) + if (tree->IsReverseOp()) { - unsigned tmpl; - - tmpl = level; - level = lvl2; - lvl2 = tmpl; + std::swap(level, lvl2); } /* Compute the sethi number for this binary operator */ @@ -7766,6 +7858,23 @@ GenTreeLclVar* Compiler::gtNewLclvNode(unsigned lnum, var_types type DEBUGARG(IL return node; } +GenTreeLclVar* Compiler::gtNewLclVarNode(unsigned lclNum, var_types type) +{ + LclVarDsc* varDsc = lvaGetDesc(lclNum); + if (type == TYP_UNDEF) + { + type = varDsc->lvNormalizeOnLoad() ? varDsc->TypeGet() : genActualType(varDsc); + } + + GenTreeLclVar* lclVar = gtNewLclvNode(lclNum, type); + if (varDsc->IsAddressExposed()) + { + lclVar->gtFlags |= GTF_GLOB_REF; + } + + return lclVar; +} + GenTreeLclVar* Compiler::gtNewLclLNode(unsigned lnum, var_types type DEBUGARG(IL_OFFSET offs)) { // We need to ensure that all struct values are normalized. @@ -8013,6 +8122,97 @@ GenTree* Compiler::gtNewLoadValueNode(var_types type, ClassLayout* layout, GenTr return (type == TYP_STRUCT) ? gtNewBlkIndir(layout, addr, indirFlags) : gtNewIndir(type, addr, indirFlags); } +//------------------------------------------------------------------------------ +// gtNewStoreBlkNode : Create an indirect struct store node. +// +// Arguments: +// layout - The struct layout +// addr - Destionation address +// data - Value to store +// indirFlags - Indirection flags +// +// Return Value: +// The created GT_STORE_BLK node. +// +GenTreeBlk* Compiler::gtNewStoreBlkNode(ClassLayout* layout, GenTree* addr, GenTree* data, GenTreeFlags indirFlags) +{ + assert((indirFlags & GTF_IND_INVARIANT) == 0); + assert(data->IsInitVal() || ClassLayout::AreCompatible(layout, data->GetLayout(this))); + + GenTreeBlk* store = new (this, GT_STORE_BLK) GenTreeBlk(GT_STORE_BLK, TYP_STRUCT, addr, data, layout); + store->gtFlags |= GTF_ASG; + gtInitializeIndirNode(store, indirFlags); + + return store; +} + +//------------------------------------------------------------------------------ +// gtNewStoreIndNode : Create an indirect store node. +// +// Arguments: +// type - Type of the store +// addr - Destionation address +// data - Value to store +// indirFlags - Indirection flags +// +// Return Value: +// The created GT_STOREIND node. +// +GenTreeStoreInd* Compiler::gtNewStoreIndNode(var_types type, GenTree* addr, GenTree* data, GenTreeFlags indirFlags) +{ + assert((indirFlags & GTF_IND_INVARIANT) == 0); + assert((type != TYP_STRUCT) && (genActualType(type) == genActualType(data))); + + GenTreeStoreInd* store = new (this, GT_STOREIND) GenTreeStoreInd(type, addr, data); + store->gtFlags |= GTF_ASG; + gtInitializeIndirNode(store, indirFlags); + + return store; +} + +//------------------------------------------------------------------------ +// gtNewStoreValueNode: Return a node that represents a store. +// +// Arguments: +// type - Type to store +// layout - Struct layout for the store +// addr - Destination address +// data - Value to store +// indirFlags - Indirection flags +// +// Return Value: +// A "STORE_BLK/STORE_IND" node, or "STORE_LCL_VAR" if "addr" points to +// a compatible local. +// +GenTree* Compiler::gtNewStoreValueNode( + var_types type, ClassLayout* layout, GenTree* addr, GenTree* data, GenTreeFlags indirFlags) +{ + assert((type != TYP_STRUCT) || (layout != nullptr)); + + if (((indirFlags & GTF_IND_VOLATILE) == 0) && addr->IsLclVarAddr()) + { + unsigned lclNum = addr->AsLclFld()->GetLclNum(); + LclVarDsc* varDsc = lvaGetDesc(lclNum); + if ((varDsc->TypeGet() == type) && + ((type != TYP_STRUCT) || ClassLayout::AreCompatible(layout, varDsc->GetLayout()))) + { + return gtNewStoreLclVarNode(lclNum, data); + } + } + + GenTree* store; + if (type == TYP_STRUCT) + { + store = gtNewStoreBlkNode(layout, addr, data, indirFlags); + } + else + { + store = gtNewStoreIndNode(type, addr, data, indirFlags); + } + + return store; +} + /***************************************************************************** * * Create a node that will assign 'src' to 'dst'. @@ -8020,7 +8220,7 @@ GenTree* Compiler::gtNewLoadValueNode(var_types type, ClassLayout* layout, GenTr GenTreeOp* Compiler::gtNewAssignNode(GenTree* dst, GenTree* src) { - assert(!src->TypeIs(TYP_VOID)); + assert(!src->TypeIs(TYP_VOID) && !compAssignmentRationalized); /* Mark the target as being assigned */ if ((dst->gtOper == GT_LCL_VAR) || (dst->OperGet() == GT_LCL_FLD)) @@ -10680,12 +10880,11 @@ void Compiler::gtDispNode(GenTree* tree, IndentStack* indentStack, _In_ _In_opt_ /* First print the flags associated with the node */ switch (tree->gtOper) { - case GT_LEA: case GT_BLK: + case GT_IND: + case GT_STOREIND: case GT_STORE_BLK: case GT_STORE_DYN_BLK: - - case GT_IND: // We prefer printing V or U if ((tree->gtFlags & (GTF_IND_VOLATILE | GTF_IND_UNALIGNED)) == 0) { @@ -15889,7 +16088,7 @@ GenTree* Compiler::gtFoldIndirConst(GenTreeIndir* indir) // May update the type of the temp, if it was previously unknown. // // May set compFloatingPointUsed. - +// GenTree* Compiler::gtNewTempAssign( unsigned tmp, GenTree* val, Statement** pAfterStmt, const DebugInfo& di, BasicBlock* block) { @@ -15976,26 +16175,21 @@ GenTree* Compiler::gtNewTempAssign( compFloatingPointUsed = true; } - /* Create the assignment node */ - - GenTree* asg; - GenTree* dest = gtNewLclvNode(tmp, dstTyp); - - if (varTypeIsStruct(varDsc) && !val->IsInitVal()) + GenTree* store; + if (compAssignmentRationalized) { - asg = impAssignStruct(dest, val, CHECK_SPILL_NONE, pAfterStmt, di, block); + store = gtNewStoreLclVarNode(tmp, val); } - else + else if (varTypeIsStruct(varDsc) && !val->IsInitVal()) { - asg = gtNewAssignNode(dest, val); + store = impAssignStruct(gtNewLclvNode(tmp, dstTyp), val, CHECK_SPILL_NONE, pAfterStmt, di, block); } - - if (compAssignmentRationalized) + else { - asg = fgRationalizeAssignment(asg->AsOp()); + store = gtNewAssignNode(gtNewLclvNode(tmp, dstTyp), val); } - return asg; + return store; } /***************************************************************************** diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index f376044e6a6b1e..1e8ebaa72721e9 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -1692,6 +1692,9 @@ struct GenTree return OperIs(GT_JCC, GT_SETCC, GT_SELECTCC); } + bool OperIsStoreLclVar(unsigned* pLclNum); + bool OperIsStoreLcl(unsigned* pLclNum); + #ifdef DEBUG static const GenTreeDebugOperKind gtDebugOperKindTable[]; @@ -1786,6 +1789,8 @@ struct GenTree // The returned pointer might be nullptr if the node is not binary, or if non-null op2 is not required. inline GenTree* gtGetOp2IfPresent() const; + inline GenTree* GetStoreDestination(); + inline GenTree*& Data(); bool TryGetUse(GenTree* operand, GenTree*** pUse); @@ -8841,6 +8846,40 @@ inline bool GenTree::OperIsCopyBlkOp() return OperIsBlkOp() && !OperIsInitBlkOp(); } +inline bool GenTree::OperIsStoreLclVar(unsigned* pLclNum) // TODO-ASG: delete. +{ + if (OperIs(GT_STORE_LCL_VAR)) + { + *pLclNum = AsLclVar()->GetLclNum(); + return true; + } + if (OperIs(GT_ASG) && gtGetOp1()->OperIs(GT_LCL_VAR)) + { + *pLclNum = gtGetOp1()->AsLclVar()->GetLclNum(); + return true; + } + + *pLclNum = BAD_VAR_NUM; + return false; +} + +inline bool GenTree::OperIsStoreLcl(unsigned* pLclNum) // TODO-ASG: delete. +{ + if (OperIsLocalStore()) + { + *pLclNum = AsLclVarCommon()->GetLclNum(); + return true; + } + if (OperIs(GT_ASG) && gtGetOp1()->OperIsLocal()) + { + *pLclNum = gtGetOp1()->AsLclVarCommon()->GetLclNum(); + return true; + } + + *pLclNum = BAD_VAR_NUM; + return false; +} + //------------------------------------------------------------------------ // IsIntegralConst: Checks whether this is a constant node with the given value // @@ -9222,6 +9261,12 @@ inline GenTree* GenTree::gtGetOp2IfPresent() const return op2; } +inline GenTree* GenTree::GetStoreDestination() // TODO-ASG: delete. +{ + assert(OperIs(GT_ASG) || OperIsStore()); + return OperIs(GT_ASG) ? gtGetOp1() : this; +} + inline GenTree*& GenTree::Data() { assert(OperIsStore() || OperIs(GT_STORE_DYN_BLK, GT_ASG)); diff --git a/src/coreclr/jit/helperexpansion.cpp b/src/coreclr/jit/helperexpansion.cpp index e614e781eb388a..e920e769e9c32a 100644 --- a/src/coreclr/jit/helperexpansion.cpp +++ b/src/coreclr/jit/helperexpansion.cpp @@ -186,15 +186,10 @@ bool Compiler::fgExpandRuntimeLookupsForCall(BasicBlock** pBlock, Statement* stm // Mostly for Tier0: if the current statement is ASG(LCL, RuntimeLookup) // we can drop it and use that LCL as the destination - if (stmt->GetRootNode()->OperIs(GT_ASG)) + if (stmt->GetRootNode()->OperIs(GT_STORE_LCL_VAR) && (stmt->GetRootNode()->AsLclVar()->Data() == *callUse)) { - GenTree* lhs = stmt->GetRootNode()->gtGetOp1(); - GenTree* rhs = stmt->GetRootNode()->gtGetOp2(); - if (lhs->OperIs(GT_LCL_VAR) && rhs == *callUse) - { - rtLookupLcl = gtClone(lhs)->AsLclVar(); - fgRemoveStmt(block, stmt); - } + rtLookupLcl = gtNewLclVarNode(stmt->GetRootNode()->AsLclVar()->GetLclNum()); + fgRemoveStmt(block, stmt); } // Grab a temp to store result (it's assigned from either fastPathBb or fallbackBb) @@ -285,12 +280,12 @@ bool Compiler::fgExpandRuntimeLookupsForCall(BasicBlock** pBlock, Statement* stm fgNewBBFromTreeAfter(BBJ_COND, prevBb, gtNewOperNode(GT_JTRUE, TYP_VOID, nullcheckOp), debugInfo); // Fallback basic block - GenTree* asgFallbackValue = gtNewAssignNode(gtClone(rtLookupLcl), call); - BasicBlock* fallbackBb = fgNewBBFromTreeAfter(BBJ_NONE, nullcheckBb, asgFallbackValue, debugInfo, true); + GenTree* fallbackValueDef = gtNewStoreLclVarNode(rtLookupLcl->GetLclNum(), call); + BasicBlock* fallbackBb = fgNewBBFromTreeAfter(BBJ_NONE, nullcheckBb, fallbackValueDef, debugInfo, true); // Fast-path basic block - GenTree* asgFastpathValue = gtNewAssignNode(gtClone(rtLookupLcl), fastPathValueClone); - BasicBlock* fastPathBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, nullcheckBb, asgFastpathValue, debugInfo); + GenTree* fastpathValueDef = gtNewStoreLclVarNode(rtLookupLcl->GetLclNum(), fastPathValueClone); + BasicBlock* fastPathBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, nullcheckBb, fastpathValueDef, debugInfo); BasicBlock* sizeCheckBb = nullptr; if (needsSizeCheck) @@ -565,15 +560,14 @@ bool Compiler::fgExpandThreadLocalAccessForCall(BasicBlock** pBlock, Statement* // Cache the tls value unsigned tlsLclNum = lvaGrabTemp(true DEBUGARG("TLS access")); lvaTable[tlsLclNum].lvType = TYP_I_IMPL; - GenTree* defTlsLclValue = gtNewLclvNode(tlsLclNum, TYP_I_IMPL); - GenTree* useTlsLclValue = gtCloneExpr(defTlsLclValue); // Create a use for tlsLclValue - GenTree* asgTlsValue = gtNewAssignNode(defTlsLclValue, tlsValue); + GenTree* tlsValueDef = gtNewStoreLclVarNode(tlsLclNum, tlsValue); + GenTree* tlsLclValueUse = gtNewLclVarNode(tlsLclNum); // Create tree for "maxThreadStaticBlocks = tls[offsetOfMaxThreadStaticBlocks]" GenTree* offsetOfMaxThreadStaticBlocks = gtNewIconNode(threadStaticBlocksInfo.offsetOfMaxThreadStaticBlocks, TYP_I_IMPL); GenTree* maxThreadStaticBlocksRef = - gtNewOperNode(GT_ADD, TYP_I_IMPL, gtCloneExpr(useTlsLclValue), offsetOfMaxThreadStaticBlocks); + gtNewOperNode(GT_ADD, TYP_I_IMPL, gtCloneExpr(tlsLclValueUse), offsetOfMaxThreadStaticBlocks); GenTree* maxThreadStaticBlocksValue = gtNewIndir(TYP_INT, maxThreadStaticBlocksRef, GTF_IND_NONFAULTING | GTF_IND_INVARIANT); @@ -585,7 +579,7 @@ bool Compiler::fgExpandThreadLocalAccessForCall(BasicBlock** pBlock, Statement* // Create tree for "threadStaticBlockBase = tls[offsetOfThreadStaticBlocks]" GenTree* offsetOfThreadStaticBlocks = gtNewIconNode(threadStaticBlocksInfo.offsetOfThreadStaticBlocks, TYP_I_IMPL); GenTree* threadStaticBlocksRef = - gtNewOperNode(GT_ADD, TYP_I_IMPL, gtCloneExpr(useTlsLclValue), offsetOfThreadStaticBlocks); + gtNewOperNode(GT_ADD, TYP_I_IMPL, gtCloneExpr(tlsLclValueUse), offsetOfThreadStaticBlocks); GenTree* threadStaticBlocksValue = gtNewIndir(TYP_I_IMPL, threadStaticBlocksRef, GTF_IND_NONFAULTING | GTF_IND_INVARIANT); @@ -599,14 +593,12 @@ bool Compiler::fgExpandThreadLocalAccessForCall(BasicBlock** pBlock, Statement* // Cache the threadStaticBlock value unsigned threadStaticBlockBaseLclNum = lvaGrabTemp(true DEBUGARG("ThreadStaticBlockBase access")); lvaTable[threadStaticBlockBaseLclNum].lvType = TYP_I_IMPL; - GenTree* defThreadStaticBlockBaseLclValue = gtNewLclvNode(threadStaticBlockBaseLclNum, TYP_I_IMPL); - GenTree* useThreadStaticBlockBaseLclValue = - gtCloneExpr(defThreadStaticBlockBaseLclValue); // StaticBlockBaseLclValue that will be used - GenTree* asgThreadStaticBlockBase = gtNewAssignNode(defThreadStaticBlockBaseLclValue, typeThreadStaticBlockValue); + GenTree* threadStaticBlockBaseDef = gtNewStoreLclVarNode(threadStaticBlockBaseLclNum, typeThreadStaticBlockValue); + GenTree* threadStaticBlockBaseLclValueUse = gtNewLclVarNode(threadStaticBlockBaseLclNum); // Create tree for "if (threadStaticBlockValue != nullptr)" GenTree* threadStaticBlockNullCond = - gtNewOperNode(GT_NE, TYP_INT, useThreadStaticBlockBaseLclValue, gtNewIconNode(0, TYP_I_IMPL)); + gtNewOperNode(GT_NE, TYP_INT, threadStaticBlockBaseLclValueUse, gtNewIconNode(0, TYP_I_IMPL)); threadStaticBlockNullCond = gtNewOperNode(GT_JTRUE, TYP_VOID, threadStaticBlockNullCond); // prevBb (BBJ_NONE): [weight: 1.0] @@ -633,26 +625,26 @@ bool Compiler::fgExpandThreadLocalAccessForCall(BasicBlock** pBlock, Statement* // use(threadStaticBlockBase); // maxThreadStaticBlocksCondBB - BasicBlock* maxThreadStaticBlocksCondBB = fgNewBBFromTreeAfter(BBJ_COND, prevBb, asgTlsValue, debugInfo); + BasicBlock* maxThreadStaticBlocksCondBB = fgNewBBFromTreeAfter(BBJ_COND, prevBb, tlsValueDef, debugInfo); fgInsertStmtAfter(maxThreadStaticBlocksCondBB, maxThreadStaticBlocksCondBB->firstStmt(), fgNewStmtFromTree(maxThreadStaticBlocksCond)); // threadStaticBlockNullCondBB BasicBlock* threadStaticBlockNullCondBB = - fgNewBBFromTreeAfter(BBJ_COND, maxThreadStaticBlocksCondBB, asgThreadStaticBlockBase, debugInfo); + fgNewBBFromTreeAfter(BBJ_COND, maxThreadStaticBlocksCondBB, threadStaticBlockBaseDef, debugInfo); fgInsertStmtAfter(threadStaticBlockNullCondBB, threadStaticBlockNullCondBB->firstStmt(), fgNewStmtFromTree(threadStaticBlockNullCond)); // fallbackBb - GenTree* asgFallbackValue = gtNewAssignNode(gtClone(threadStaticBlockLcl), call); + GenTree* fallbackValueDef = gtNewStoreLclVarNode(threadStaticBlockLclNum, call); BasicBlock* fallbackBb = - fgNewBBFromTreeAfter(BBJ_ALWAYS, threadStaticBlockNullCondBB, asgFallbackValue, debugInfo, true); + fgNewBBFromTreeAfter(BBJ_ALWAYS, threadStaticBlockNullCondBB, fallbackValueDef, debugInfo, true); // fastPathBb - GenTree* asgFastPathValue = - gtNewAssignNode(gtClone(threadStaticBlockLcl), gtCloneExpr(useThreadStaticBlockBaseLclValue)); - BasicBlock* fastPathBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, fallbackBb, asgFastPathValue, debugInfo, true); + GenTree* fastPathValueDef = + gtNewStoreLclVarNode(threadStaticBlockLclNum, gtCloneExpr(threadStaticBlockBaseLclValueUse)); + BasicBlock* fastPathBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, fallbackBb, fastPathValueDef, debugInfo, true); // // Update preds in all new blocks diff --git a/src/coreclr/jit/ifconversion.cpp b/src/coreclr/jit/ifconversion.cpp index d7194d36c0df12..c7441458556bc4 100644 --- a/src/coreclr/jit/ifconversion.cpp +++ b/src/coreclr/jit/ifconversion.cpp @@ -144,7 +144,7 @@ bool OptIfConversionDsc::IfConvertCheckThenFlow() } else { - m_mainOper = GT_ASG; + m_mainOper = GT_STORE_LCL_VAR; } return true; } @@ -233,19 +233,16 @@ bool OptIfConversionDsc::IfConvertCheckStmts(BasicBlock* fromBlock, IfConvertOpe GenTree* tree = stmt->GetRootNode(); switch (tree->gtOper) { - case GT_ASG: + case GT_STORE_LCL_VAR: { - GenTree* op1 = tree->gtGetOp1(); - GenTree* op2 = tree->gtGetOp2(); - // Only one per operation per block can be conditionally executed. if (found) { return false; } - // Ensure the operarand is a local variable with integer type. - if (!op1->OperIs(GT_LCL_VAR) || !varTypeIsIntegralOrI(op1)) + // Ensure the local has integer type. + if (!varTypeIsIntegralOrI(tree)) { return false; } @@ -258,16 +255,16 @@ bool OptIfConversionDsc::IfConvertCheckStmts(BasicBlock* fromBlock, IfConvertOpe return false; } #endif + GenTree* op1 = tree->AsLclVar()->Data(); // Ensure it won't cause any additional side effects. - if ((op1->gtFlags & (GTF_SIDE_EFFECT | GTF_ORDER_SIDEEFF)) != 0 || - (op2->gtFlags & (GTF_SIDE_EFFECT | GTF_ORDER_SIDEEFF)) != 0) + if ((op1->gtFlags & (GTF_SIDE_EFFECT | GTF_ORDER_SIDEEFF)) != 0) { return false; } // Ensure the source isn't a phi. - if (op2->OperIs(GT_PHI)) + if (op1->OperIs(GT_PHI)) { return false; } @@ -276,7 +273,7 @@ bool OptIfConversionDsc::IfConvertCheckStmts(BasicBlock* fromBlock, IfConvertOpe // with the condition (for example, the condition could be an explicit bounds // check and the operand could read an array element). Disallow this except // for some common cases that we know are always side effect free. - if (((m_cond->gtFlags & GTF_ORDER_SIDEEFF) != 0) && !op2->IsInvariant() && !op2->OperIsLocal()) + if (((m_cond->gtFlags & GTF_ORDER_SIDEEFF) != 0) && !op1->IsInvariant() && !op1->OperIsLocal()) { return false; } @@ -424,7 +421,7 @@ void OptIfConversionDsc::IfConvertDump() // // This is represented in IR by two basic blocks. The first block (block) ends with // a JTRUE statement which conditionally jumps to the second block (thenBlock). -// The second block just contains a single assign statement. Both blocks then jump +// The second block just contains a single store statement. Both blocks then jump // to the same destination (finalBlock). Note that the first block may contain // additional statements prior to the JTRUE statement. // @@ -439,15 +436,14 @@ void OptIfConversionDsc::IfConvertDump() // // ------------ BB04 [00D..010), preds={BB03} succs={BB05} // STMT00005 -// * ASG int $VN.Void -// +--* LCL_VAR int V00 arg0 -// \--* CNS_INT int 5 $47 +// * STORE_LCL_VAR int V00 arg0 +// \--* CNS_INT int 5 $47 // // // This is optimised by conditionally executing the store and removing the conditional -// jumps. First the JTRUE is replaced with a NOP. The assignment is updated so that -// the source of the store is a SELECT node with the condition set to the inverse of -// the original JTRUE condition. If the condition passes the original assign happens, +// jumps. First the JTRUE is replaced with a NOP. The store is updated so that the +// source of the store is a SELECT node with the condition set to the inverse of the +// original JTRUE condition. If the condition passes the original store happens, // otherwise the existing source value is used. // // In the example above, local var 0 is set to 5 if the LT returns true, otherwise @@ -458,8 +454,7 @@ void OptIfConversionDsc::IfConvertDump() // * NOP void // // STMT00005 -// * ASG int $VN.Void -// +--* LCL_VAR int V00 arg0 +// * STORE_LCL_VAR int V00 arg0 // \--* SELECT int // +--* LT int $102 // | +--* LCL_VAR int V02 @@ -485,15 +480,13 @@ void OptIfConversionDsc::IfConvertDump() // // ------------ BB04 [00D..010), preds={BB03} succs={BB06} // STMT00005 -// * ASG int $VN.Void -// +--* LCL_VAR int V00 arg0 -// \--* CNS_INT int 5 $47 +// * STORE_LCL_VAR int V00 arg0 +// \--* CNS_INT int 5 $47 // // ------------ BB05 [00D..010), preds={BB03} succs={BB06} // STMT00006 -// * ASG int $VN.Void -// +--* LCL_VAR int V00 arg0 -// \--* CNS_INT int 9 $48 +// * STORE_LCL_VAR int V00 arg0 +// \--* CNS_INT int 9 $48 // // Again this is squashed into a single block, with the SELECT node handling both cases. // @@ -502,8 +495,7 @@ void OptIfConversionDsc::IfConvertDump() // * NOP void // // STMT00005 -// * ASG int $VN.Void -// +--* LCL_VAR int V00 arg0 +// * STORE_LCL_VAR int V00 arg0 // \--* SELECT int // +--* LT int $102 // | +--* LCL_VAR int V02 @@ -587,7 +579,7 @@ bool OptIfConversionDsc::optIfConvert() { return false; } - assert(m_thenOperation.node->gtOper == GT_ASG || m_thenOperation.node->gtOper == GT_RETURN); + assert(m_thenOperation.node->OperIs(GT_STORE_LCL_VAR, GT_RETURN)); if (m_doElseConversion) { if (!IfConvertCheckStmts(m_startBlock->bbJumpDest, &m_elseOperation)) @@ -596,16 +588,16 @@ bool OptIfConversionDsc::optIfConvert() } // Both operations must be the same node type. - if (m_thenOperation.node->gtOper != m_elseOperation.node->gtOper) + if (m_thenOperation.node->OperGet() != m_elseOperation.node->OperGet()) { return false; } - // Currently can only support Else Asg Blocks that have the same destination as the Then block. - if (m_thenOperation.node->gtOper == GT_ASG) + // Currently can only support Else Store Blocks that have the same destination as the Then block. + if (m_thenOperation.node->OperIs(GT_STORE_LCL_VAR)) { - unsigned lclNumThen = m_thenOperation.node->gtGetOp1()->AsLclVarCommon()->GetLclNum(); - unsigned lclNumElse = m_elseOperation.node->gtGetOp1()->AsLclVarCommon()->GetLclNum(); + unsigned lclNumThen = m_thenOperation.node->AsLclVarCommon()->GetLclNum(); + unsigned lclNumElse = m_elseOperation.node->AsLclVarCommon()->GetLclNum(); if (lclNumThen != lclNumElse) { return false; @@ -633,14 +625,14 @@ bool OptIfConversionDsc::optIfConvert() int thenCost = 0; int elseCost = 0; - if (m_mainOper == GT_ASG) + if (m_mainOper == GT_STORE_LCL_VAR) { - thenCost = m_thenOperation.node->gtGetOp2()->GetCostEx() + - (m_comp->gtIsLikelyRegVar(m_thenOperation.node->gtGetOp1()) ? 0 : 2); + thenCost = m_thenOperation.node->AsLclVar()->Data()->GetCostEx() + + (m_comp->gtIsLikelyRegVar(m_thenOperation.node) ? 0 : 2); if (m_doElseConversion) { - elseCost = m_elseOperation.node->gtGetOp2()->GetCostEx() + - (m_comp->gtIsLikelyRegVar(m_elseOperation.node->gtGetOp1()) ? 0 : 2); + elseCost = m_elseOperation.node->AsLclVar()->Data()->GetCostEx() + + (m_comp->gtIsLikelyRegVar(m_elseOperation.node) ? 0 : 2); } } else @@ -685,25 +677,22 @@ bool OptIfConversionDsc::optIfConvert() var_types selectType; GenTree* selectTrueInput; GenTree* selectFalseInput; - if (m_mainOper == GT_ASG) + if (m_mainOper == GT_STORE_LCL_VAR) { if (m_doElseConversion) { - selectTrueInput = m_elseOperation.node->gtGetOp2(); - selectFalseInput = m_thenOperation.node->gtGetOp2(); + selectTrueInput = m_elseOperation.node->AsLclVar()->Data(); + selectFalseInput = m_thenOperation.node->AsLclVar()->Data(); } - else + else // Duplicate the destination of the Then store. { - // Duplicate the destination of the Then assignment. - assert(m_thenOperation.node->gtGetOp1()->IsLocal()); - selectTrueInput = m_comp->gtCloneExpr(m_thenOperation.node->gtGetOp1()); - selectTrueInput->gtFlags &= GTF_EMPTY; - - selectFalseInput = m_thenOperation.node->gtGetOp2(); + GenTreeLclVar* store = m_thenOperation.node->AsLclVar(); + selectTrueInput = m_comp->gtNewLclVarNode(store->GetLclNum(), store->TypeGet()); + selectFalseInput = m_thenOperation.node->AsLclVar()->Data(); } // Pick the type as the type of the local, which should always be compatible even for implicit coercions. - selectType = genActualType(m_thenOperation.node->gtGetOp1()); + selectType = genActualType(m_thenOperation.node); } else { @@ -719,12 +708,12 @@ bool OptIfConversionDsc::optIfConvert() // Create a select node. GenTreeConditional* select = m_comp->gtNewConditionalNode(GT_SELECT, m_cond, selectTrueInput, selectFalseInput, selectType); - m_thenOperation.node->AsOp()->gtFlags |= (select->gtFlags & GTF_ALL_EFFECT); + m_thenOperation.node->AddAllEffectsFlags(select); // Use the select as the source of the Then operation. - if (m_mainOper == GT_ASG) + if (m_mainOper == GT_STORE_LCL_VAR) { - m_thenOperation.node->AsOp()->gtOp2 = select; + m_thenOperation.node->AsLclVar()->Data() = select; } else { @@ -734,12 +723,12 @@ bool OptIfConversionDsc::optIfConvert() m_comp->fgSetStmtSeq(m_thenOperation.stmt); // Remove statements. - last->ReplaceWith(m_comp->gtNewNothingNode(), m_comp); + last->gtBashToNOP(); m_comp->gtSetEvalOrder(last); m_comp->fgSetStmtSeq(m_startBlock->lastStmt()); if (m_doElseConversion) { - m_elseOperation.node->ReplaceWith(m_comp->gtNewNothingNode(), m_comp); + m_elseOperation.node->gtBashToNOP(); m_comp->gtSetEvalOrder(m_elseOperation.node); m_comp->fgSetStmtSeq(m_elseOperation.stmt); } diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index edfee1c4c2e644..33d31bb3db95c0 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -8264,8 +8264,6 @@ void Lowering::TryRetypingFloatingPointStoreToIntegerStore(GenTree* store) if (store->OperIs(GT_STORE_LCL_VAR)) { store->SetOper(GT_STORE_LCL_FLD); - store->AsLclFld()->SetLclOffs(0); - store->AsLclFld()->SetLayout(nullptr); } store->ChangeType(type); } diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index 4cfbfb2eeb63e5..8c1bec692f269f 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -7909,32 +7909,28 @@ GenTree* Compiler::fgMorphCall(GenTreeCall* call) // Either or both of the array and index arguments may have been spilled to temps by `fgMorphArgs`. Copy // the spill trees as well if necessary. - GenTreeOp* argSetup = nullptr; + GenTree* argSetup = nullptr; for (CallArg& arg : call->gtArgs.EarlyArgs()) { - GenTree* const argNode = arg.GetEarlyNode(); - if (argNode->OperGet() != GT_ASG) + if (arg.GetLateNode() == nullptr) { continue; } - assert(argNode != arr); - assert(argNode != index); + GenTree* const setupArgNode = arg.GetEarlyNode(); + assert((setupArgNode != arr) && (setupArgNode != index)); - GenTree* op1 = argSetup; - if (op1 == nullptr) + if (argSetup == nullptr) { - op1 = gtNewNothingNode(); -#if DEBUG - op1->gtDebugFlags |= GTF_DEBUG_NODE_MORPHED; -#endif // DEBUG + argSetup = setupArgNode; } - - argSetup = new (this, GT_COMMA) GenTreeOp(GT_COMMA, TYP_VOID, op1, argNode); - + else + { + argSetup = new (this, GT_COMMA) GenTreeOp(GT_COMMA, TYP_VOID, argSetup, setupArgNode); #if DEBUG - argSetup->gtDebugFlags |= GTF_DEBUG_NODE_MORPHED; + argSetup->gtDebugFlags |= GTF_DEBUG_NODE_MORPHED; #endif // DEBUG + } } #ifdef DEBUG @@ -7948,11 +7944,18 @@ GenTree* Compiler::fgMorphCall(GenTreeCall* call) fgWalkTreePost(&value, resetMorphedFlag); #endif // DEBUG - GenTree* const arrIndexAddr = gtNewArrayIndexAddr(arr, index, TYP_REF, NO_CLASS_HANDLE); - GenTree* const arrIndex = gtNewIndexIndir(arrIndexAddr->AsIndexAddr()); - GenTree* const arrStore = gtNewAssignNode(arrIndex, value); + GenTree* indexAddr = gtNewArrayIndexAddr(arr, index, TYP_REF, NO_CLASS_HANDLE); + GenTree* store; + if (compAssignmentRationalized) + { + store = gtNewStoreIndNode(TYP_REF, indexAddr, value); + } + else + { + store = gtNewAssignNode(gtNewIndir(TYP_REF, indexAddr), value); + } - GenTree* result = fgMorphTree(arrStore); + GenTree* result = fgMorphTree(store); if (argSetup != nullptr) { result = new (this, GT_COMMA) GenTreeOp(GT_COMMA, TYP_VOID, argSetup, result); @@ -11662,6 +11665,10 @@ GenTree* Compiler::fgMorphSmpOpOptional(GenTreeOp* tree, bool* optAssertionPropD switch (oper) { case GT_ASG: + case GT_STOREIND: + case GT_STORE_BLK: + case GT_STORE_LCL_VAR: + case GT_STORE_LCL_FLD: // Make sure we're allowed to do this. if (optValnumCSE_phase) { @@ -11689,6 +11696,11 @@ GenTree* Compiler::fgMorphSmpOpOptional(GenTreeOp* tree, bool* optAssertionPropD } } + if (!tree->OperIs(GT_ASG)) + { + break; + } + if (typ == TYP_LONG) { break; diff --git a/src/coreclr/jit/morphblock.cpp b/src/coreclr/jit/morphblock.cpp index 2048612180f881..54e144753138e4 100644 --- a/src/coreclr/jit/morphblock.cpp +++ b/src/coreclr/jit/morphblock.cpp @@ -9,7 +9,7 @@ class MorphInitBlockHelper static GenTree* MorphInitBlock(Compiler* comp, GenTree* tree); protected: - MorphInitBlockHelper(Compiler* comp, GenTree* asg, bool initBlock); + MorphInitBlockHelper(Compiler* comp, GenTree* store, bool initBlock); GenTree* Morph(); @@ -36,9 +36,9 @@ class MorphInitBlockHelper Compiler* m_comp; bool m_initBlock; - GenTreeOp* m_asg = nullptr; - GenTree* m_dst = nullptr; - GenTree* m_src = nullptr; + GenTree* m_asg = nullptr; + GenTree* m_dst = nullptr; + GenTree* m_src = nullptr; unsigned m_blockSize = 0; ClassLayout* m_blockLayout = nullptr; @@ -93,21 +93,12 @@ GenTree* MorphInitBlockHelper::MorphInitBlock(Compiler* comp, GenTree* tree) // Notes: // Most class members are initialized via in-class member initializers. // -MorphInitBlockHelper::MorphInitBlockHelper(Compiler* comp, GenTree* asg, bool initBlock = true) +MorphInitBlockHelper::MorphInitBlockHelper(Compiler* comp, GenTree* store, bool initBlock = true) : m_comp(comp), m_initBlock(initBlock) { - assert(asg->OperIs(GT_ASG)); -#if defined(DEBUG) - if (m_initBlock) - { - assert(asg->OperIsInitBlkOp()); - } - else - { - assert(asg->OperIsCopyBlkOp()); - } -#endif // DEBUG - m_asg = asg->AsOp(); + assert(store->OperIs(GT_ASG) || store->OperIsStore()); + assert((m_initBlock == store->OperIsInitBlkOp()) && (!m_initBlock == store->OperIsCopyBlkOp())); + m_asg = store; } //------------------------------------------------------------------------ @@ -188,15 +179,16 @@ GenTree* MorphInitBlockHelper::Morph() // void MorphInitBlockHelper::PrepareDst() { - m_dst = m_asg->gtGetOp1(); + m_dst = m_asg->GetStoreDestination(); // Commas cannot be destinations. assert(!m_dst->OperIs(GT_COMMA)); + // TODO-ASG: delete this retyping. if (m_asg->TypeGet() != m_dst->TypeGet()) { - assert(!m_initBlock && "the asg type should be final for an init block."); - JITDUMP("changing type of assignment from %-6s to %-6s\n", varTypeName(m_asg->TypeGet()), + assert(!m_initBlock && "the store type should be final for an init block."); + JITDUMP("changing type of store from %-6s to %-6s\n", varTypeName(m_asg->TypeGet()), varTypeName(m_dst->TypeGet())); m_asg->ChangeType(m_dst->TypeGet()); @@ -218,7 +210,7 @@ void MorphInitBlockHelper::PrepareDst() else { assert(m_dst == m_dst->gtEffectiveVal() && "the commas were skipped in MorphBlock"); - assert(m_dst->OperIs(GT_IND, GT_BLK) && (!m_dst->OperIs(GT_IND) || !m_dst->TypeIs(TYP_STRUCT))); + assert(m_dst->OperIsIndir() && (!m_dst->isIndir() || !m_dst->TypeIs(TYP_STRUCT))); } if (m_dst->TypeIs(TYP_STRUCT)) @@ -290,7 +282,7 @@ void MorphInitBlockHelper::PropagateExpansionAssertions() // void MorphInitBlockHelper::PrepareSrc() { - m_src = m_asg->gtGetOp2(); + m_src = m_asg->Data(); } //------------------------------------------------------------------------ @@ -330,7 +322,8 @@ void MorphInitBlockHelper::MorphStructCases() if (m_dstVarDsc != nullptr) { - if (m_dst->OperIs(GT_LCL_FLD)) + // TODO-ASG: delete the GT_LCL_FLD check on "m_dst". + if (m_dst->OperIs(GT_LCL_FLD, GT_STORE_LCL_FLD)) { m_comp->lvaSetVarDoNotEnregister(m_dstLclNum DEBUGARG(DoNotEnregisterReason::LocalField)); } @@ -454,7 +447,6 @@ void MorphInitBlockHelper::TryInitFieldByField() LclVarDsc* fieldDesc = m_comp->lvaGetDesc(fieldLclNum); var_types fieldType = fieldDesc->TypeGet(); - GenTree* dest = m_comp->gtNewLclvNode(fieldLclNum, fieldType); GenTree* src; switch (fieldType) @@ -470,7 +462,7 @@ void MorphInitBlockHelper::TryInitFieldByField() FALLTHROUGH; case TYP_INT: { - int64_t mask = (int64_t(1) << (genTypeSize(dest->TypeGet()) * 8)) - 1; + int64_t mask = (int64_t(1) << (genTypeSize(fieldType) * 8)) - 1; src = m_comp->gtNewIconNode(static_cast(initPattern & mask)); break; } @@ -508,20 +500,20 @@ void MorphInitBlockHelper::TryInitFieldByField() unreached(); } - GenTree* asg = m_comp->gtNewAssignNode(dest, src); + GenTree* store = m_comp->gtNewTempAssign(fieldLclNum, src); if (m_comp->optLocalAssertionProp) { - m_comp->optAssertionGen(asg); + m_comp->optAssertionGen(store); } if (tree != nullptr) { - tree = m_comp->gtNewOperNode(GT_COMMA, TYP_VOID, tree, asg); + tree = m_comp->gtNewOperNode(GT_COMMA, TYP_VOID, tree, store); } else { - tree = asg; + tree = store; } } @@ -556,14 +548,25 @@ void MorphInitBlockHelper::TryPrimitiveInit() m_src->BashToZeroConst(lclVarType); } - m_dst->ChangeType(m_dstVarDsc->lvNormalizeOnLoad() ? lclVarType : genActualType(lclVarType)); - m_dst->ChangeOper(GT_LCL_VAR); - m_dst->AsLclVar()->SetLclNum(m_dstLclNum); - m_dst->gtFlags |= GTF_VAR_DEF; + if (m_asg->OperIs(GT_ASG)) + { + m_dst->ChangeType(m_dstVarDsc->lvNormalizeOnLoad() ? lclVarType : genActualType(lclVarType)); + m_dst->ChangeOper(GT_LCL_VAR); + m_dst->AsLclVar()->SetLclNum(m_dstLclNum); + m_dst->gtFlags |= GTF_VAR_DEF; + + m_asg->ChangeType(m_dst->TypeGet()); + m_asg->AsOp()->gtOp1 = m_dst; + m_asg->AsOp()->gtOp2 = m_src; + } + else + { + m_asg->ChangeType(m_dstVarDsc->lvNormalizeOnLoad() ? lclVarType : genActualType(lclVarType)); + m_asg->ChangeOper(GT_STORE_LCL_VAR); + m_asg->AsLclVar()->SetLclNum(m_dstLclNum); + m_asg->gtFlags |= GTF_VAR_DEF; + } - m_asg->ChangeType(m_dst->TypeGet()); - m_asg->gtOp1 = m_dst; - m_asg->gtOp2 = m_src; m_result = m_asg; m_transformationDecision = BlockTransformation::OneAsgBlock; } @@ -586,41 +589,41 @@ void MorphInitBlockHelper::TryPrimitiveInit() // We have a tree like the following (note that location-valued commas are // illegal, so there cannot be a comma on the left): // -// ASG -// / \. -// IND COMMA -// | / \. -// B C D +// ASG STOREIND +// / \ / \. +// IND COMMA or B COMMA +// | / \ / \. +// B C D C D // -// We'd like downstream code to just see and be expand ASG(IND(B), D). +// We'd like downstream code to just see and expand ASG(IND(B), D). // We will produce: // -// COMMA -// / \. -// ASG COMMA -// / \ / \. -// tmp B C ASG -// / \. -// IND D +// COMMA COMMA +// / \. / \. +// ASG COMMA STORE_LCL_VAR COMMA +// / \ / \. or / / \. +// tmp B C ASG B C STOREIND +// / \. / \. +// IND D tmp D // | // tmp // -// If the ASG has GTF_REVERSE_OPS then we will produce: +// If the store has GTF_REVERSE_OPS then we will produce: // -// COMMA -// / \. -// C ASG -// / \. -// IND D +// COMMA COMMA +// / \. / \. +// C ASG or C STOREIND +// / \. / \. +// IND D B D // | // B // // While keeping the GTF_REVERSE_OPS. // // Note that the final resulting tree is created in the caller since it also -// needs to propagate side effect flags from the decomposed assignment to all -// the created commas. Therefore this function just returns a linked list of -// the side effects to be used for that purpose. +// needs to propagate side effect flags from the decomposed store to all the +// created commas. Therefore this function just returns a linked list of the +// side effects to be used for that purpose. // GenTree* MorphInitBlockHelper::EliminateCommas(GenTree** commaPool) { @@ -638,44 +641,43 @@ GenTree* MorphInitBlockHelper::EliminateCommas(GenTree** commaPool) *commaPool = comma; }; - GenTree* lhs = m_asg->gtGetOp1(); - assert(lhs->OperIsIndir() || lhs->OperIsLocal()); - - GenTree* rhs = m_asg->gtGetOp2(); + GenTree* dst = m_asg->GetStoreDestination(); + GenTree* src = m_asg->Data(); + assert(dst->OperIsIndir() || dst->OperIsLocal()); if (m_asg->IsReverseOp()) { - while (rhs->OperIs(GT_COMMA)) + while (src->OperIs(GT_COMMA)) { - addComma(rhs); - rhs = rhs->gtGetOp2(); + addComma(src); + src = src->gtGetOp2(); } } else { - if (lhs->OperIsIndir() && rhs->OperIs(GT_COMMA)) + if (dst->OperIsIndir() && src->OperIs(GT_COMMA)) { - GenTree* addr = lhs->gtGetOp1(); - if (((addr->gtFlags & GTF_ALL_EFFECT) != 0) || (((rhs->gtFlags & GTF_ASG) != 0) && !addr->IsInvariant())) + GenTree* addr = dst->AsIndir()->Addr(); + if (((addr->gtFlags & GTF_ALL_EFFECT) != 0) || (((src->gtFlags & GTF_ASG) != 0) && !addr->IsInvariant())) { unsigned lhsAddrLclNum = m_comp->lvaGrabTemp(true DEBUGARG("Block morph LHS addr")); addSideEffect(m_comp->gtNewTempAssign(lhsAddrLclNum, addr)); - lhs->AsUnOp()->gtOp1 = m_comp->gtNewLclvNode(lhsAddrLclNum, genActualType(addr)); - m_comp->gtUpdateNodeSideEffects(lhs); + dst->AsUnOp()->gtOp1 = m_comp->gtNewLclvNode(lhsAddrLclNum, genActualType(addr)); + m_comp->gtUpdateNodeSideEffects(dst); } } - while (rhs->OperIs(GT_COMMA)) + while (src->OperIs(GT_COMMA)) { - addComma(rhs); - rhs = rhs->gtGetOp2(); + addComma(src); + src = src->gtGetOp2(); } } if (sideEffects != nullptr) { - m_asg->gtOp2 = rhs; + m_asg->Data() = src; m_comp->gtUpdateNodeSideEffects(m_asg); } @@ -752,7 +754,7 @@ MorphCopyBlockHelper::MorphCopyBlockHelper(Compiler* comp, GenTree* asg) : Morph // void MorphCopyBlockHelper::PrepareSrc() { - m_src = m_asg->gtGetOp2(); + m_src = m_asg->Data(); if (m_src->IsLocal()) { @@ -762,7 +764,7 @@ void MorphCopyBlockHelper::PrepareSrc() m_srcVarDsc = m_comp->lvaGetDesc(m_srcLclNum); } - // Verify that the types on the LHS and RHS match. + // Verify that the types of the store and data match. assert(m_dst->TypeGet() == m_src->TypeGet()); if (m_dst->TypeIs(TYP_STRUCT)) { @@ -777,7 +779,7 @@ void MorphCopyBlockHelper::TrySpecialCases() { if (m_src->IsMultiRegNode()) { - assert(m_dst->OperIs(GT_LCL_VAR)); + assert(m_dst->OperIsScalarLocal()); m_dstVarDsc->lvIsMultiRegRet = true; @@ -785,7 +787,7 @@ void MorphCopyBlockHelper::TrySpecialCases() m_transformationDecision = BlockTransformation::SkipMultiRegSrc; m_result = m_asg; } - else if (m_src->IsCall() && m_dst->OperIs(GT_LCL_VAR) && m_dstVarDsc->CanBeReplacedWithItsField(m_comp)) + else if (m_src->IsCall() && m_dst->OperIsScalarLocal() && m_dstVarDsc->CanBeReplacedWithItsField(m_comp)) { JITDUMP("Not morphing a single reg call return\n"); m_transformationDecision = BlockTransformation::SkipSingleRegCallSrc; @@ -863,7 +865,8 @@ void MorphCopyBlockHelper::MorphStructCases() // If either src or dest is a reg-sized non-field-addressed struct, keep the copyBlock; // this will avoid having to DNER the enregisterable local when creating LCL_FLD nodes. - if ((m_dst->OperIs(GT_LCL_VAR) && m_dstVarDsc->lvRegStruct) || + // TODO-ASG: delete the GT_LCL_VAR check on "m_dst". + if ((m_dst->OperIs(GT_STORE_LCL_VAR, GT_LCL_VAR) && m_dstVarDsc->lvRegStruct) || (m_src->OperIs(GT_LCL_VAR) && m_srcVarDsc->lvRegStruct)) { requiresCopyBlock = true; @@ -1065,7 +1068,8 @@ void MorphCopyBlockHelper::MorphStructCases() // if (!m_dstDoFldAsg && (m_dstVarDsc != nullptr) && !m_dstSingleLclVarAsg) { - if (m_dst->OperIs(GT_LCL_FLD)) + // TODO-ASG: delete the GT_LCL_FLD check on "m_dst". + if (m_dst->OperIs(GT_LCL_FLD, GT_STORE_LCL_FLD)) { m_comp->lvaSetVarDoNotEnregister(m_dstLclNum DEBUGARG(DoNotEnregisterReason::LocalField)); } @@ -1107,10 +1111,10 @@ void MorphCopyBlockHelper::TryPrimitiveCopy() } var_types asgType = TYP_UNDEF; - assert((m_src == m_src->gtEffectiveVal()) && (m_dst == m_dst->gtEffectiveVal())); // Can we use the LHS local directly? - if (m_dst->OperIs(GT_LCL_FLD)) + // TODO-ASG: delete the GT_LCL_FLD check on "m_dst". + if (m_dst->OperIs(GT_LCL_FLD, GT_STORE_LCL_FLD)) { if (m_blockSize == genTypeSize(m_dstVarDsc)) { @@ -1139,30 +1143,31 @@ void MorphCopyBlockHelper::TryPrimitiveCopy() return; } - auto doRetypeNode = [asgType](GenTree* op, LclVarDsc* varDsc) { + auto doRetypeNode = [asgType](GenTree* op, LclVarDsc* varDsc, bool isUse) { if (op->OperIsIndir()) { - op->SetOper(GT_IND); + op->SetOper(isUse ? GT_IND : GT_STOREIND); op->ChangeType(asgType); } else if (varDsc->TypeGet() == asgType) { - op->SetOper(GT_LCL_VAR); + op->SetOper(isUse ? GT_LCL_VAR : GT_STORE_LCL_VAR); op->ChangeType(varDsc->lvNormalizeOnLoad() ? varDsc->TypeGet() : genActualType(varDsc)); op->gtFlags &= ~GTF_VAR_USEASG; } else { - if (op->OperIs(GT_LCL_VAR)) + if (op->OperIsScalarLocal()) { - op->SetOper(GT_LCL_FLD); + op->SetOper(isUse ? GT_LCL_FLD : GT_STORE_LCL_FLD); } op->ChangeType(asgType); } }; - doRetypeNode(m_dst, m_dstVarDsc); - doRetypeNode(m_src, m_srcVarDsc); + doRetypeNode(m_dst, m_dstVarDsc, /* isUse */ m_asg->OperIs(GT_ASG)); + doRetypeNode(m_src, m_srcVarDsc, /* isUse */ true); + // TODO-ASG: delete. m_asg->ChangeType(asgType); m_result = m_asg; @@ -1184,7 +1189,7 @@ GenTree* MorphCopyBlockHelper::CopyFieldByField() GenTree* addrSpill = nullptr; unsigned addrSpillTemp = BAD_VAR_NUM; - GenTree* addrSpillAsg = nullptr; + GenTree* addrSpillStore = nullptr; unsigned fieldCnt = 0; @@ -1290,113 +1295,20 @@ GenTree* MorphCopyBlockHelper::CopyFieldByField() addrSpillTemp = m_comp->lvaGrabTemp(true DEBUGARG("BlockOp address local")); LclVarDsc* addrSpillDsc = m_comp->lvaGetDesc(addrSpillTemp); - addrSpillDsc->lvType = TYP_BYREF; - - GenTreeLclVar* addrSpillNode = m_comp->gtNewLclvNode(addrSpillTemp, TYP_BYREF); - addrSpillAsg = m_comp->gtNewAssignNode(addrSpillNode, addrSpill); + addrSpillDsc->lvType = TYP_BYREF; // TODO-ASG: zero-diff quirk, delete. + addrSpillStore = m_comp->gtNewTempAssign(addrSpillTemp, addrSpill); } // We may have allocated a temp above, and that may have caused the lvaTable to be expanded. // So, beyond this point we cannot rely on the old values of 'm_srcVarDsc' and 'm_dstVarDsc'. - + bool useAsg = m_asg->OperIs(GT_ASG); for (unsigned i = 0; i < fieldCnt; ++i) { - GenTree* dstFld; - if (m_dstDoFldAsg) + if (m_dstDoFldAsg && m_comp->fgGlobalMorph && m_dstLclNode->IsLastUse(i)) { - noway_assert((m_dstLclNum != BAD_VAR_NUM) && (dstAddr == nullptr)); - - unsigned dstFieldLclNum = m_comp->lvaGetDesc(m_dstLclNum)->lvFieldLclStart + i; - if (m_comp->fgGlobalMorph && m_dstLclNode->IsLastUse(i)) - { - JITDUMP("Field-by-field copy skipping write to dead field V%02u\n", dstFieldLclNum); - continue; - } - - dstFld = m_comp->gtNewLclvNode(dstFieldLclNum, m_comp->lvaGetDesc(dstFieldLclNum)->TypeGet()); - - // If it had been labeled a "USEASG", assignments to the individual promoted fields are not. - dstFld->gtFlags |= m_dstLclNode->gtFlags & ~(GTF_NODE_MASK | GTF_VAR_USEASG | GTF_VAR_DEATH_MASK); - - // Don't CSE the lhs of an assignment. - dstFld->gtFlags |= GTF_DONT_CSE; - } - else - { - noway_assert(m_srcDoFldAsg); - - if (m_dstSingleLclVarAsg) - { - noway_assert(fieldCnt == 1); - noway_assert(m_dstVarDsc != nullptr); - noway_assert(addrSpill == nullptr); - - dstFld = m_comp->gtNewLclvNode(m_dstLclNum, m_dstVarDsc->TypeGet()); - } - else - { - GenTree* dstAddrClone = nullptr; - if (!m_dstUseLclFld) - { - // Need address of the destination. - if (addrSpill) - { - assert(addrSpillTemp != BAD_VAR_NUM); - dstAddrClone = m_comp->gtNewLclvNode(addrSpillTemp, TYP_BYREF); - } - else - { - if (result == nullptr) - { - // Reuse the original "dstAddr" tree for the first field. - dstAddrClone = dstAddr; - } - else - { - // We can't clone multiple copies of a tree with persistent side effects - noway_assert((dstAddr->gtFlags & GTF_PERSISTENT_SIDE_EFFECTS) == 0); - - dstAddrClone = m_comp->gtCloneExpr(dstAddr); - noway_assert(dstAddrClone != nullptr); - - JITDUMP("dstAddr - Multiple Fields Clone created:\n"); - DISPTREE(dstAddrClone); - - // Morph the newly created tree - dstAddrClone = m_comp->fgMorphTree(dstAddrClone); - } - } - } - - LclVarDsc* srcVarDsc = m_comp->lvaGetDesc(m_srcLclNum); - unsigned srcFieldLclNum = srcVarDsc->lvFieldLclStart + i; - LclVarDsc* srcFieldVarDsc = m_comp->lvaGetDesc(srcFieldLclNum); - unsigned srcFieldOffset = srcFieldVarDsc->lvFldOffset; - var_types srcType = srcFieldVarDsc->TypeGet(); - - if (!m_dstUseLclFld) - { - if (srcFieldOffset != 0) - { - GenTree* fieldOffsetNode = m_comp->gtNewIconNode(srcFieldVarDsc->lvFldOffset, TYP_I_IMPL); - dstAddrClone = m_comp->gtNewOperNode(GT_ADD, TYP_BYREF, dstAddrClone, fieldOffsetNode); - } - - dstFld = m_comp->gtNewIndir(srcType, dstAddrClone); - } - else - { - assert(dstAddrClone == nullptr); - - // If the dst was a struct type field "B" in a struct "A" then we add - // add offset of ("B" in "A") + current offset in "B". - unsigned totalOffset = m_dstLclOffset + srcFieldOffset; - dstFld = m_comp->gtNewLclFldNode(m_dstLclNum, srcType, totalOffset); - - // TODO-1stClassStructs: remove this and implement storing to a field in a struct in a reg. - m_comp->lvaSetVarDoNotEnregister(m_dstLclNum DEBUGARG(DoNotEnregisterReason::LocalField)); - } - } + INDEBUG(unsigned dstFieldLclNum = m_comp->lvaGetDesc(m_dstLclNum)->lvFieldLclStart + i); + JITDUMP("Field-by-field copy skipping write to dead field V%02u\n", dstFieldLclNum); + continue; } GenTree* srcFld = nullptr; @@ -1509,27 +1421,147 @@ GenTree* MorphCopyBlockHelper::CopyFieldByField() } } assert(srcFld != nullptr); + + GenTree* dstFld; + if (m_dstDoFldAsg) + { + noway_assert((m_dstLclNum != BAD_VAR_NUM) && (dstAddr == nullptr)); + + unsigned dstFieldLclNum = m_comp->lvaGetDesc(m_dstLclNum)->lvFieldLclStart + i; + if (useAsg) + { + dstFld = m_comp->gtNewLclvNode(dstFieldLclNum, m_comp->lvaGetDesc(dstFieldLclNum)->TypeGet()); + + // If it had been labeled a "USEASG", assignments to the individual promoted fields are not. + dstFld->gtFlags |= m_dstLclNode->gtFlags & ~(GTF_NODE_MASK | GTF_VAR_USEASG | GTF_VAR_DEATH_MASK); + + // Don't CSE the lhs of an assignment. + dstFld->gtFlags |= GTF_DONT_CSE; + } + else + { + dstFld = m_comp->gtNewStoreLclVarNode(dstFieldLclNum, srcFld); + } + } + else + { + noway_assert(m_srcDoFldAsg); + + if (m_dstSingleLclVarAsg) + { + noway_assert(fieldCnt == 1); + noway_assert(m_dstVarDsc != nullptr); + noway_assert(addrSpill == nullptr); + + if (useAsg) + { + dstFld = m_comp->gtNewLclvNode(m_dstLclNum, m_dstVarDsc->TypeGet()); + } + else + { + dstFld = m_comp->gtNewStoreLclVarNode(m_dstLclNum, srcFld); + } + } + else + { + GenTree* dstAddrClone = nullptr; + if (!m_dstUseLclFld) + { + // Need address of the destination. + if (addrSpill) + { + assert(addrSpillTemp != BAD_VAR_NUM); + dstAddrClone = m_comp->gtNewLclvNode(addrSpillTemp, TYP_BYREF); + } + else + { + if (result == nullptr) + { + // Reuse the original "dstAddr" tree for the first field. + dstAddrClone = dstAddr; + } + else + { + // We can't clone multiple copies of a tree with persistent side effects + noway_assert((dstAddr->gtFlags & GTF_PERSISTENT_SIDE_EFFECTS) == 0); + + dstAddrClone = m_comp->gtCloneExpr(dstAddr); + noway_assert(dstAddrClone != nullptr); + + JITDUMP("dstAddr - Multiple Fields Clone created:\n"); + DISPTREE(dstAddrClone); + + // Morph the newly created tree + dstAddrClone = m_comp->fgMorphTree(dstAddrClone); + } + } + } + + LclVarDsc* srcVarDsc = m_comp->lvaGetDesc(m_srcLclNum); + unsigned srcFieldLclNum = srcVarDsc->lvFieldLclStart + i; + LclVarDsc* srcFieldVarDsc = m_comp->lvaGetDesc(srcFieldLclNum); + unsigned srcFieldOffset = srcFieldVarDsc->lvFldOffset; + var_types srcType = srcFieldVarDsc->TypeGet(); + + if (!m_dstUseLclFld) + { + if (srcFieldOffset != 0) + { + GenTree* fieldOffsetNode = m_comp->gtNewIconNode(srcFieldOffset, TYP_I_IMPL); + dstAddrClone = m_comp->gtNewOperNode(GT_ADD, TYP_BYREF, dstAddrClone, fieldOffsetNode); + } + + if (useAsg) + { + dstFld = m_comp->gtNewIndir(srcType, dstAddrClone); + } + else + { + dstFld = m_comp->gtNewStoreIndNode(srcType, dstAddrClone, srcFld); + } + } + else + { + assert(dstAddrClone == nullptr); + + // If the dst was a struct type field "B" in a struct "A" then we add + // add offset of ("B" in "A") + current offset in "B". + unsigned totalOffset = m_dstLclOffset + srcFieldOffset; + if (useAsg) + { + dstFld = m_comp->gtNewLclFldNode(m_dstLclNum, srcType, totalOffset); + } + else + { + dstFld = m_comp->gtNewStoreLclFldNode(m_dstLclNum, srcType, totalOffset, srcFld); + } + + // TODO-1stClassStructs: remove this and implement storing to a field in a struct in a reg. + m_comp->lvaSetVarDoNotEnregister(m_dstLclNum DEBUGARG(DoNotEnregisterReason::LocalField)); + } + } + } noway_assert(dstFld->TypeGet() == srcFld->TypeGet()); - GenTreeOp* asgOneFld = m_comp->gtNewAssignNode(dstFld, srcFld); + GenTree* storeOneFld = useAsg ? m_comp->gtNewAssignNode(dstFld, srcFld) : dstFld; if (m_comp->optLocalAssertionProp) { - m_comp->optAssertionGen(asgOneFld); + m_comp->optAssertionGen(storeOneFld); } - if (addrSpillAsg != nullptr) + if (addrSpillStore != nullptr) { - result = m_comp->gtNewOperNode(GT_COMMA, TYP_VOID, addrSpillAsg, asgOneFld)->AsOp(); - addrSpillAsg = nullptr; + result = m_comp->gtNewOperNode(GT_COMMA, TYP_VOID, addrSpillStore, storeOneFld); + addrSpillStore = nullptr; } else if (result != nullptr) { - result = m_comp->gtNewOperNode(GT_COMMA, TYP_VOID, result, asgOneFld)->AsOp(); + result = m_comp->gtNewOperNode(GT_COMMA, TYP_VOID, result, storeOneFld); } else { - result = asgOneFld; + result = storeOneFld; } } @@ -1619,10 +1651,7 @@ GenTree* Compiler::fgMorphStoreDynBlock(GenTreeStoreDynBlk* tree) if ((size != 0) && FitsIn(size)) { ClassLayout* layout = typGetBlkLayout(static_cast(size)); - GenTree* dst = gtNewLoadValueNode(layout, tree->Addr(), tree->gtFlags & GTF_IND_FLAGS); - dst->gtFlags |= GTF_GLOB_REF; - - GenTree* src = tree->Data(); + GenTree* src = tree->Data(); if (src->OperIs(GT_IND)) { assert(src->TypeIs(TYP_STRUCT)); @@ -1630,15 +1659,25 @@ GenTree* Compiler::fgMorphStoreDynBlock(GenTreeStoreDynBlk* tree) src->AsBlk()->Initialize(layout); } - GenTree* asg = gtNewAssignNode(dst, src); - asg->AddAllEffectsFlags(tree); - INDEBUG(asg->gtDebugFlags |= GTF_DEBUG_NODE_MORPHED); + GenTree* store; + if (compAssignmentRationalized) + { + store = gtNewStoreValueNode(layout, tree->Addr(), src, tree->gtFlags & GTF_IND_FLAGS); + } + else + { + GenTree* dst = gtNewLoadValueNode(layout, tree->Addr(), tree->gtFlags & GTF_IND_FLAGS); + dst->gtFlags |= GTF_GLOB_REF; + store = gtNewAssignNode(dst, src); + } + store->AddAllEffectsFlags(tree); + INDEBUG(store->gtDebugFlags |= GTF_DEBUG_NODE_MORPHED); - fgAssignSetVarDef(asg); + fgAssignSetVarDef(store); JITDUMP("MorphStoreDynBlock: transformed STORE_DYN_BLK into ASG(BLK, Data())\n"); - return tree->OperIsCopyBlkOp() ? fgMorphCopyBlock(asg) : fgMorphInitBlock(asg); + return tree->OperIsCopyBlkOp() ? fgMorphCopyBlock(store) : fgMorphInitBlock(store); } }