Add emitLoopAlignAdjustments()

dotnet · kunalspathak · Jan 12, 2021 · Oct 8, 2020 · Oct 9, 2020 · Oct 9, 2020
commit 8036061a2d5d5a7218c27b1d9624f755d4bd2e81
diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp
@@ -2258,6 +2258,9 @@ void CodeGen::genGenerateMachineCode()
 
     GetEmitter()->emitJumpDistBind();
 
+    /* Perform alignment adjustments */
+    GetEmitter()->emitLoopAlignAdjustments();
+
     /* The code is now complete and final; it should not change after this. */
 }
 
@@ -2348,7 +2351,8 @@ void CodeGen::genEmitMachineCode()
         printf("\n; Total bytes of code %d, prolog size %d, PerfScore %.2f, instruction count %d, allocated bytes for "
                "code %d (MethodHash=%08x) for "
                "method %s\n",
-               codeSize, prologSize, compiler->info.compPerfScore, instrCount, GetEmitter()->emitTotalHotCodeSize,
+               codeSize, prologSize, compiler->info.compPerfScore, instrCount,
+               GetEmitter()->emitTotalHotCodeSize + GetEmitter()->emitTotalColdCodeSize,
                compiler->info.compMethodHash(), compiler->info.compFullName);
         printf("; ============================================================\n\n");
         printf(""); // in our logic this causes a flush

diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp
@@ -773,6 +773,7 @@ void CodeGen::genCodeForBBlist()
         if ((block->bbNext != nullptr) && (block->bbNext->bbFlags & BBF_FIRST_BLOCK_IN_INNERLOOP))
         {
             assert(ShouldAlignLoops());
+
 #ifndef ADAPTIVE_LOOP_ALIGNMENT
             if (verbose)
             {

diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp
@@ -3665,6 +3665,8 @@ size_t emitter::emitIssue1Instr(insGroup* ig, instrDesc* id, BYTE** dp)
         }
 #endif // DEBUG_EMIT
 
+        //assert(id->idCodeSize() == csz);
+
         /* The instruction size estimate wasn't accurate; remember this */
 
         ig->igFlags |= IGF_UPD_ISZ;
@@ -4487,6 +4489,139 @@ void emitter::emitJumpDistBind()
 #endif // DEBUG
 }
 
+void emitter::emitLoopAlignAdjustments()
+{
+#ifdef TARGET_XARCH
+
+#ifdef ADAPTIVE_LOOP_ALIGNMENT
+    unsigned alignmentBoundary = DEFAULT_ALIGN_LOOP_BOUNDARY;
+#else
+    unsigned alignmentBoundary = emitComp->opts.compJitAlignLoopBoundary;
+#endif
+    unsigned removeAlignment         = 0;
+    bool     skipPadding             = false;
+    int      maxBlocksAllowedForLoop = genLog2(alignmentBoundary) - 1;
+    unsigned maxLoopSize             = DEFAULT_ALIGN_LOOP_BOUNDARY * maxBlocksAllowedForLoop;
+    size_t   dst                     = 0, of = 0;
+    unsigned minBlocksNeededForLoop = 0, nMaxPaddingBytes = 0, nPaddingBytes = 0, loopSize = 0;
+    for (insGroup* ig = emitIGlist; ig != nullptr; ig = ig->igNext)
+    {
+        ig->igOffs -= removeAlignment;
+        dst += ig->igSize;
+
+        // Below is not needed because we just care about the igSize
+        // and that gets adjusted in emitJumpDst when we add IGF_UPD_ISZ
+        //// recalculate the size
+        //if ((ig->igFlags & IGF_UPD_ISZ) != 0)
+        //{
+        //    igSize = emitFindOffset(ig, ig->igInsCnt);
+        //    assert(igSize == ig->igSize);
+        //}
+        //else
+        //{
+        //    igSize = ig->igSize;
+        //}
+
+        //if (emitComp->compMethodID == 37683)
+        //{
+        //    unsigned       insNum = ig->igInsCnt;
+        //    instrDesc*     id     = (instrDesc*)ig->igData;
+
+        //    /* Walk the instruction list until all are counted */
+
+        //    while (insNum > 0)
+        //    {
+        //        unsigned currSize = id->idCodeSize();
+
+
+        //        printf("[%04XH] size = %d -- ", of, id->idCodeSize());
+        //        of += currSize;
+
+        //        emitDispIns(id, true, false, false);
+
+        //        castto(id, BYTE*) += emitSizeOfInsDsc(id);
+
+        //        insNum--;
+        //    }
+        //}
+
+        //dst += igSize;
+
+        if (!(ig->igFlags & IGF_ALIGN_LOOP))
+        {
+            continue;
+        }
+
+        // TODO: Add logging for Skip/Add?
+        // TODO: I am about to align so (dst -= 15)
+
+        if ((dst & (alignmentBoundary - 1)) == 0)
+        {
+            skipPadding = true;
+        }
+        else
+        {
+            loopSize               = 0;
+            insGroup* loopHeaderIg = ig->igNext;
+            for (insGroup* igInLoop = loopHeaderIg; igInLoop; igInLoop = igInLoop->igNext)
+            {
+                loopSize += igInLoop->igSize;
+                if (igInLoop->igLoopBackEdge == loopHeaderIg)
+                {
+                    break;
+                }
+            }
+
+            minBlocksNeededForLoop = (loopSize + alignmentBoundary - 1) / alignmentBoundary;
+            nMaxPaddingBytes       = (1 << (maxBlocksAllowedForLoop - minBlocksNeededForLoop + 1)) - 1;
+            nPaddingBytes          = (-(int)dst) & (alignmentBoundary - 1);
+
+            if (loopSize > maxLoopSize)
+            {
+                skipPadding = true;
+            }
+            else if (nPaddingBytes > nMaxPaddingBytes)
+            {
+                alignmentBoundary = 16;
+                nMaxPaddingBytes  = 1 << (maxBlocksAllowedForLoop - minBlocksNeededForLoop + 1);
+                nPaddingBytes     = (-(int)dst) & (alignmentBoundary - 1);
+
+                if (nPaddingBytes > nMaxPaddingBytes)
+                {
+                    //skipPadding = true;
+                }
+            }
+        }
+
+        if (!skipPadding && (nPaddingBytes > 0))
+        {
+            size_t extraBytesNotInLoop =
+                (32 * minBlocksNeededForLoop) - loopSize;   // Still have it at alignmentboundary=32
+            size_t currentOffset = dst % alignmentBoundary; // TODO: Change to & (boundary - 1)
+            if (currentOffset <= extraBytesNotInLoop)
+            {
+                //skipPadding = true;
+
+                // TODO: Detect actual no. of padding bytes.
+                // TODO: Figure out how to update size of align instructions so they just add padding in emitter.
+            }
+        }
+
+        if (skipPadding)
+        {
+            dst -= 15;
+            ig->igSize -= 15;
+            ig->igFlags |= IGF_UPD_ISZ;
+            removeAlignment += 15;
+            emitTotalCodeSize -= 15;
+
+            // remove the flag
+            ig->igFlags &= ~IGF_ALIGN_LOOP;
+        }
+    }
+#endif
+}
+
 void emitter::emitCheckFuncletBranch(instrDesc* jmp, insGroup* jmpIG)
 {
 #ifdef DEBUG

diff --git a/src/coreclr/jit/emit.h b/src/coreclr/jit/emit.h
@@ -1740,6 +1740,7 @@ class emitter
     instrDescJmp* emitJumpList;       // list of local jumps in method
     instrDescJmp* emitJumpLast;       // last of local jumps in method
     void          emitJumpDistBind(); // Bind all the local jumps in method
+    void          emitLoopAlignAdjustments(); // Predict if loop alignment is needed and make appropriate adjustments
 
     void emitCheckFuncletBranch(instrDesc* jmp, insGroup* jmpIG); // Check for illegal branches between funclets
 

diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp
@@ -12632,24 +12632,42 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp)
             {
                 // Candidate for loop alignment
                 assert(codeGen->ShouldAlignLoops());
-                assert(ig->igFlags & IGF_ALIGN_LOOP);
+
 #ifdef ADAPTIVE_LOOP_ALIGNMENT
                 unsigned alignmentBoundary = DEFAULT_ALIGN_LOOP_BOUNDARY;
 #else
                 unsigned alignmentBoundary = emitComp->opts.compJitAlignLoopBoundary;
 #endif
                 sz = SMALL_IDSC_SIZE;
 
+#if DEBUG
+                bool displayAlignmentDetails =
+                    (emitComp->opts.disAsm /*&& emitComp->opts.disAddr*/) || emitComp->verbose;
+#endif
+               if ((ig->igFlags & IGF_ALIGN_LOOP) == 0)
+                {
+                    id->idCodeSize(0);
+                    ig->igFlags |= IGF_UPD_ISZ;
+                    if (displayAlignmentDetails)
+                    {
+                        printf("\t\t;; Skip alignment: 'Big loop.' in (%s)\n", emitComp->info.compFullName);
+                    }
+                    break;
+                }
+
                 // If already at alignment boundary, no need to emit anything.
                 if (((size_t)dst & (alignmentBoundary - 1)) == 0)
                 {
+                    id->idCodeSize(0);
+                    ig->igFlags |= IGF_UPD_ISZ;
+                    if (displayAlignmentDetails)
+                    {
+                        printf("\t\t;; Skip alignment: 'Loop already aligned at boundary.' in (%s)\n",
+                               emitComp->info.compMethodName);
+                    }
                     break;
                 }
 
-#if DEBUG
-                bool displayAlignmentDetails = (emitComp->opts.disAsm && emitComp->opts.disAddr) || emitComp->verbose;
-#endif
-
 #ifndef ADAPTIVE_LOOP_ALIGNMENT
                 if (emitComp->opts.compJitAlignLoopAdaptive)
 #endif
@@ -12678,12 +12696,13 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp)
                     if (loopSize > maxLoopSize)
                     {
                         skipPadding = true;
+                        assert(!"This should have checked before!");
 #if DEBUG
-                        if (displayAlignmentDetails)
+                        /*if (displayAlignmentDetails)
                         {
-                            printf("\t\t;; Skip alignment: 'Loopsize= %d, MaxLoopSize= %d.' in (%s)\n", loopSize,
-                                   maxLoopSize, emitComp->info.compFullName);
-                        }
+                            printf("\t\t;; Skip alignment: 'Loopsize= %d, MaxLoopSize= %d, Estimated= %d.' in (%s)\n", loopSize,
+                                   maxLoopSize, ig->loopSize, emitComp->info.compFullName);
+                        }*/
 #endif
                     }
                     else if (nPaddingBytes > nMaxPaddingBytes)

diff --git a/src/coreclr/jit/optimizer.cpp b/src/coreclr/jit/optimizer.cpp
@@ -2546,14 +2546,15 @@ void Compiler::optFindNaturalLoops()
 #if defined(TARGET_XARCH)
         if (codeGen->ShouldAlignLoops())
         {
-            // An innerloop candidate that might need alignment
-            if ((optLoopTable[loopInd].lpChild == BasicBlock::NOT_IN_LOOP) && (
 #ifdef ADAPTIVE_LOOP_ALIGNMENT
-                                                                                  DEFAULT_ALIGN_LOOP_MIN_BLOCK_WEIGHT
+            unsigned minBlockWeight = DEFAULT_ALIGN_LOOP_MIN_BLOCK_WEIGHT;
 #else
-                                                                                  opts.compJitAlignLoopMinBlockWeight
+            unsigned minBlockWeight = opts.compJitAlignLoopMinBlockWeight;
 #endif
-                                                                                  <= first->getBBWeight(this)))
+
+            // An innerloop candidate that might need alignment
+            if ((optLoopTable[loopInd].lpChild == BasicBlock::NOT_IN_LOOP) &&
+                minBlockWeight <= first->getBBWeight(this))
             {
                 first->bbFlags |= BBF_FIRST_BLOCK_IN_INNERLOOP;
             }