From 7251c19632d5f8f5adb03e21c01903c4fd1dd59a Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 8 Dec 2023 16:54:59 -0800 Subject: [PATCH 01/37] Initial work --- src/coreclr/jit/codegenarm64.cpp | 12 ++- src/coreclr/jit/codegencommon.cpp | 2 +- src/coreclr/jit/emitarm64.cpp | 155 ++++++++++++++++++++++++++++++ src/coreclr/jit/emitarm64.h | 8 ++ 4 files changed, 175 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index a1133331cf9c97..2837d8b3ccca91 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -5436,7 +5436,7 @@ void CodeGen::genProfilingLeaveCallback(unsigned helper) #define ALL_ARM64_EMITTER_UNIT_TESTS // #define ALL_ARM64_EMITTER_UNIT_TESTS_GENERAL // #define ALL_ARM64_EMITTER_UNIT_TESTS_ADVSIMD -// #define ALL_ARM64_EMITTER_UNIT_TESTS_SVE +#define ALL_ARM64_EMITTER_UNIT_TESTS_SVE // #define ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED #if defined(DEBUG) @@ -10654,6 +10654,16 @@ void CodeGen::genArm64EmitterUnitTests() theEmitter->emitIns_R_R_R(INS_sve_fsqrt, EA_SCALABLE, REG_V6, REG_P6, REG_V6, INS_OPTS_SCALABLE_S); /* FSQRT ., /M, . */ + // IF_SVE_IL_3A + theEmitter->emitIns_R_R_R_I(INS_sve_ldnf1d, EA_SCALABLE, REG_V0, REG_P0, REG_R0, 0, + INS_OPTS_SCALABLE_D); // LDNF1D {.D }, /Z, [{, #, MUL VL}] + theEmitter->emitIns_R_R_R_I(INS_sve_ldnf1sw, EA_SCALABLE, REG_V0, REG_P0, REG_R0, 0, + INS_OPTS_SCALABLE_D); // LDNF1SW {.D }, /Z, [{, #, MUL VL}] + theEmitter->emitIns_R_R_R_I(INS_sve_ldnf1d, EA_SCALABLE, REG_V0, REG_P1, REG_R2, 5, + INS_OPTS_SCALABLE_D); // LDNF1D {.D }, /Z, [{, #, MUL VL}] + theEmitter->emitIns_R_R_R_I(INS_sve_ldnf1sw, EA_SCALABLE, REG_V0, REG_P1, REG_R2, 5, + INS_OPTS_SCALABLE_D); // LDNF1SW {.D }, /Z, [{, #, MUL VL}] + #endif // ALL_ARM64_EMITTER_UNIT_TESTS_SVE #ifdef ALL_ARM64_EMITTER_UNIT_TESTS diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 7a96748a50a2c6..95c965178aae6a 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -2096,7 +2096,7 @@ void CodeGen::genEmitUnwindDebugGCandEH() #endif // LATE_DISASM #ifdef DEBUG - if (JitConfig.JitRawHexCode().contains(compiler->info.compMethodHnd, compiler->info.compClassHnd, + if (compiler->opts.altJit && JitConfig.JitRawHexCode().contains(compiler->info.compMethodHnd, compiler->info.compClassHnd, &compiler->info.compMethodInfo->args)) { BYTE* addr = (BYTE*)*codePtr + compiler->GetEmitter()->writeableOffset; diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index aa81f83634ccf2..53b8c3b0c71da6 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -1193,6 +1193,47 @@ void emitter::emitInsSanityCheck(instrDesc* id) assert(isScalableVectorSize(elemsize)); break; + case IF_SVE_IH_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus + // immediate) + case IF_SVE_IH_3A_A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus + // immediate) + case IF_SVE_IH_3A_F: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus + // immediate) + case IF_SVE_IJ_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) + case IF_SVE_IJ_3A_D: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) + case IF_SVE_IJ_3A_E: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) + case IF_SVE_IJ_3A_F: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) + case IF_SVE_IJ_3A_G: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) + case IF_SVE_IL_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus immediate) + case IF_SVE_IL_3A_A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus + // immediate) + case IF_SVE_IL_3A_B: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus + // immediate) + case IF_SVE_IL_3A_C: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus + // immediate) + case IF_SVE_IM_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-temporal load (scalar plus + // immediate) + case IF_SVE_IO_3A: // ............iiii ...gggnnnnnttttt -- SVE load and broadcast quadword (scalar plus + // immediate) + case IF_SVE_IQ_3A: // ............iiii ...gggnnnnnttttt -- SVE load multiple structures (quadwords, scalar plus + // immediate) + case IF_SVE_IS_3A: // ............iiii ...gggnnnnnttttt -- SVE load multiple structures (scalar plus immediate) + case IF_SVE_JE_3A: // ............iiii ...gggnnnnnttttt -- SVE store multiple structures (quadwords, scalar plus + // immediate) + case IF_SVE_JM_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-temporal store (scalar plus + // immediate) + case IF_SVE_JN_3C: // ............iiii ...gggnnnnnttttt -- SVE contiguous store (scalar plus immediate) + case IF_SVE_JN_3C_D: // ............iiii ...gggnnnnnttttt -- SVE contiguous store (scalar plus immediate) + case IF_SVE_JO_3A: // ............iiii ...gggnnnnnttttt -- SVE store multiple structures (scalar plus immediate) + elemsize = id->idOpSize(); + assert(id->idInsOpt() == INS_OPTS_SCALABLE_D); + assert(isVectorRegister(id->idReg1())); // ttttt + assert(isPredicateRegister(id->idReg2())); // ggg + assert(isGeneralRegister(id->idReg3())); // nnnnn + assert(isValidSimm4(emitGetInsSC(id))); // iiii + assert(isScalableVectorSize(elemsize)); + break; + default: printf("unexpected format %s\n", emitIfName(id->idInsFmt())); assert(!"Unexpected format"); @@ -9367,6 +9408,16 @@ void emitter::emitIns_R_R_R_I(instruction ins, fmt = IF_DV_3AI; break; + case INS_sve_ldnf1sw: + case INS_sve_ldnf1d: + assert(insOptsScalable(opt)); + assert(isVectorRegister(reg1)); + assert(isPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isValidSimm4(imm)); + fmt = IF_SVE_IL_3A; + break; + default: unreached(); break; @@ -12609,6 +12660,17 @@ void emitter::emitIns_Call(EmitCallType callType, return 0; } +/***************************************************************************** + * + * TODO + */ + +/*static*/ emitter::code_t emitter::insEncodeSimm4_19_to_16(ssize_t imm) +{ + assert(isValidSimm4(imm)); + return (code_t)imm << 16; +} + BYTE* emitter::emitOutputLoadLabel(BYTE* dst, BYTE* srcAddr, BYTE* dstAddr, instrDescJmp* id) { instruction ins = id->idIns(); @@ -14640,6 +14702,7 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn code |= insEncodeSveElemsize_tszh_22_tszl_20_to_19(optGetSveElemsize(id->idInsOpt())); // xx // x + dst += emitOutput_Instr(dst, code); break; case IF_SVE_GK_2A: // ................ ......mmmmmddddd -- SVE2 crypto destructive binary operations @@ -14655,6 +14718,47 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) dst += emitOutput_Instr(dst, code); break; + case IF_SVE_IH_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus + // immediate) + case IF_SVE_IH_3A_A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus + // immediate) + case IF_SVE_IH_3A_F: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus + // immediate) + case IF_SVE_IJ_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) + case IF_SVE_IJ_3A_D: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) + case IF_SVE_IJ_3A_E: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) + case IF_SVE_IJ_3A_F: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) + case IF_SVE_IJ_3A_G: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) + case IF_SVE_IL_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus immediate) + case IF_SVE_IL_3A_A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus + // immediate) + case IF_SVE_IL_3A_B: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus + // immediate) + case IF_SVE_IL_3A_C: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus + // immediate) + case IF_SVE_IM_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-temporal load (scalar plus + // immediate) + case IF_SVE_IO_3A: // ............iiii ...gggnnnnnttttt -- SVE load and broadcast quadword (scalar plus + // immediate) + case IF_SVE_IQ_3A: // ............iiii ...gggnnnnnttttt -- SVE load multiple structures (quadwords, scalar plus + // immediate) + case IF_SVE_IS_3A: // ............iiii ...gggnnnnnttttt -- SVE load multiple structures (scalar plus immediate) + case IF_SVE_JE_3A: // ............iiii ...gggnnnnnttttt -- SVE store multiple structures (quadwords, scalar plus + // immediate) + case IF_SVE_JM_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-temporal store (scalar plus + // immediate) + case IF_SVE_JN_3C: // ............iiii ...gggnnnnnttttt -- SVE contiguous store (scalar plus immediate) + case IF_SVE_JN_3C_D: // ............iiii ...gggnnnnnttttt -- SVE contiguous store (scalar plus immediate) + case IF_SVE_JO_3A: // ............iiii ...gggnnnnnttttt -- SVE store multiple structures (scalar plus immediate) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_R_9_to_5(id->idReg3()); // ggg + code |= insEncodeReg_P_12_to_10(id->idReg2()); // nnnnn + code |= insEncodeSimm4_19_to_16(imm); // iiii + dst += emitOutput_Instr(dst, code); + break; + default: assert(!"Unexpected format"); break; @@ -17028,6 +17132,52 @@ void emitter::emitDispInsHelp( emitDispSveReg(id->idReg1(), id->idInsOpt(), false); // ddddd break; + case IF_SVE_IH_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus + // immediate) + case IF_SVE_IH_3A_A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus + // immediate) + case IF_SVE_IH_3A_F: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus + // immediate) + case IF_SVE_IJ_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) + case IF_SVE_IJ_3A_D: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) + case IF_SVE_IJ_3A_E: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) + case IF_SVE_IJ_3A_F: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) + case IF_SVE_IJ_3A_G: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) + // { .D }, /Z, [{, #, MUL VL}] + case IF_SVE_IL_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus immediate) + case IF_SVE_IL_3A_A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus + // immediate) + case IF_SVE_IL_3A_B: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus + // immediate) + case IF_SVE_IL_3A_C: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus + // immediate) + case IF_SVE_IM_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-temporal load (scalar plus + // immediate) + case IF_SVE_IO_3A: // ............iiii ...gggnnnnnttttt -- SVE load and broadcast quadword (scalar plus + // immediate) + case IF_SVE_IQ_3A: // ............iiii ...gggnnnnnttttt -- SVE load multiple structures (quadwords, scalar plus + // immediate) + case IF_SVE_IS_3A: // ............iiii ...gggnnnnnttttt -- SVE load multiple structures (scalar plus immediate) + case IF_SVE_JE_3A: // ............iiii ...gggnnnnnttttt -- SVE store multiple structures (quadwords, scalar plus + // immediate) + case IF_SVE_JM_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-temporal store (scalar plus + // immediate) + case IF_SVE_JN_3C: // ............iiii ...gggnnnnnttttt -- SVE contiguous store (scalar plus immediate) + case IF_SVE_JN_3C_D: // ............iiii ...gggnnnnnttttt -- SVE contiguous store (scalar plus immediate) + case IF_SVE_JO_3A: // ............iiii ...gggnnnnnttttt -- SVE store multiple structures (scalar plus immediate) + imm = emitGetInsSC(id); + emitDispSveRegList(id->idReg1(), 1, id->idInsOpt(), true); // ttttt + emitDispPredicateReg(id->idReg2(), PREDICATE_ZERO, true); // ggg + printf("["); + emitDispReg(id->idReg3(), optGetSveElemsize(id->idInsOpt()), imm != 0); // nnnnn + if (imm != 0) + { + emitDispImm(emitGetInsSC(id), true); // iiii + printf("mul vl"); + } + printf("]"); + break; + default: printf("unexpected format %s", emitIfName(id->idInsFmt())); assert(!"unexpectedFormat"); @@ -19515,6 +19665,11 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins result.insLatency = PERFSCORE_LATENCY_2C; break; + case IF_SVE_IL_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus immediate) + result.insThroughput = PERFSCORE_THROUGHPUT_3C; + result.insLatency = PERFSCORE_LATENCY_6C; + break; + default: // all other instructions perfScoreUnhandledInstruction(id, &result); diff --git a/src/coreclr/jit/emitarm64.h b/src/coreclr/jit/emitarm64.h index d70ea28472cac9..b68f4d4572486e 100644 --- a/src/coreclr/jit/emitarm64.h +++ b/src/coreclr/jit/emitarm64.h @@ -472,6 +472,8 @@ static code_t insEncodeSveElemsize(insOpts opt); // This specifically encodes the field 'tszh:tszl' at bit locations '22:20-19'. static code_t insEncodeSveElemsize_tszh_22_tszl_20_to_19(emitAttr size); +static code_t insEncodeSimm4_19_to_16(ssize_t imm); + // Returns true if 'reg' represents an integer register. static bool isIntegerRegister(regNumber reg) { @@ -484,6 +486,12 @@ static bool isStackRegister(regNumber reg) return (reg == REG_ZR) || (reg == REG_FP); } // ZR (R31) encodes the SP register +// Returns true if 'value' is a legal signed immediate 4 bit encoding (such as for LDNF1SW). +static bool isValidSimm4(ssize_t value) +{ + return (-8 <= value) && (value <= 7); +}; + // Returns true if 'value' is a legal unsigned immediate 5 bit encoding (such as for CCMP). static bool isValidUimm5(ssize_t value) { From bf05a314b22f4d03bf301ce4f52648d9c1bec545 Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 8 Dec 2023 17:04:47 -0800 Subject: [PATCH 02/37] Added another format --- src/coreclr/jit/codegenarm64.cpp | 6 ++++++ src/coreclr/jit/emitarm64.cpp | 16 ++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 2837d8b3ccca91..1050de5b4eb5da 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -10654,6 +10654,12 @@ void CodeGen::genArm64EmitterUnitTests() theEmitter->emitIns_R_R_R(INS_sve_fsqrt, EA_SCALABLE, REG_V6, REG_P6, REG_V6, INS_OPTS_SCALABLE_S); /* FSQRT ., /M, . */ + // IF_SVE_IH_3A + theEmitter->emitIns_R_R_R_I(INS_sve_ld1d, EA_SCALABLE, REG_V0, REG_P0, REG_R0, 0, + INS_OPTS_SCALABLE_D); // LD1D {.D }, /Z, [{, #, MUL VL}] + theEmitter->emitIns_R_R_R_I(INS_sve_ld1d, EA_SCALABLE, REG_V0, REG_P2, REG_R3, 5, + INS_OPTS_SCALABLE_D); // LD1D {.D }, /Z, [{, #, MUL VL}] + // IF_SVE_IL_3A theEmitter->emitIns_R_R_R_I(INS_sve_ldnf1d, EA_SCALABLE, REG_V0, REG_P0, REG_R0, 0, INS_OPTS_SCALABLE_D); // LDNF1D {.D }, /Z, [{, #, MUL VL}] diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 53b8c3b0c71da6..ff269f5c4b2c90 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -9408,6 +9408,16 @@ void emitter::emitIns_R_R_R_I(instruction ins, fmt = IF_DV_3AI; break; + case INS_sve_ld1d: + assert(insOptsScalable(opt)); + assert(isVectorRegister(reg1)); + assert(isPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isValidSimm4(imm)); + fmt = IF_SVE_IH_3A; + break; + + case INS_sve_ldnf1sw: case INS_sve_ldnf1d: assert(insOptsScalable(opt)); @@ -19665,6 +19675,12 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins result.insLatency = PERFSCORE_LATENCY_2C; break; + case IF_SVE_IH_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus + // immediate) + result.insThroughput = PERFSCORE_THROUGHPUT_1C; + result.insLatency = PERFSCORE_LATENCY_9C; + break; + case IF_SVE_IL_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus immediate) result.insThroughput = PERFSCORE_THROUGHPUT_3C; result.insLatency = PERFSCORE_LATENCY_6C; From 050ef3b5af45b512fbac0f303dc8318cd1c66d92 Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 8 Dec 2023 17:09:42 -0800 Subject: [PATCH 03/37] Fix-up comment --- src/coreclr/jit/emitarm64.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index ff269f5c4b2c90..7660764c4d5806 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -14763,8 +14763,8 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) imm = emitGetInsSC(id); code = emitInsCodeSve(ins, fmt); code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_R_9_to_5(id->idReg3()); // ggg - code |= insEncodeReg_P_12_to_10(id->idReg2()); // nnnnn + code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg code |= insEncodeSimm4_19_to_16(imm); // iiii dst += emitOutput_Instr(dst, code); break; From 8775968dd607058d67f0b38451e60b6c3477450b Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 8 Dec 2023 21:22:43 -0800 Subject: [PATCH 04/37] Added another format --- src/coreclr/jit/codegenarm64.cpp | 6 ++++-- src/coreclr/jit/emitarm64.cpp | 35 ++++++++++++++++++++++++++++---- src/coreclr/jit/emitarm64.h | 2 +- src/coreclr/jit/instr.h | 1 + 4 files changed, 37 insertions(+), 7 deletions(-) diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 1050de5b4eb5da..689e1fd7dd2fb3 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -10655,10 +10655,12 @@ void CodeGen::genArm64EmitterUnitTests() INS_OPTS_SCALABLE_S); /* FSQRT ., /M, . */ // IF_SVE_IH_3A - theEmitter->emitIns_R_R_R_I(INS_sve_ld1d, EA_SCALABLE, REG_V0, REG_P0, REG_R0, 0, + theEmitter->emitIns_R_R_R_I(INS_sve_ld1d, EA_SCALABLE, REG_V5, REG_P3, REG_R4, 0, INS_OPTS_SCALABLE_D); // LD1D {.D }, /Z, [{, #, MUL VL}] + + // IF_SVE_IH_3A_A theEmitter->emitIns_R_R_R_I(INS_sve_ld1d, EA_SCALABLE, REG_V0, REG_P2, REG_R3, 5, - INS_OPTS_SCALABLE_D); // LD1D {.D }, /Z, [{, #, MUL VL}] + INS_OPTS_SCALABLE_Q); // LD1D {.Q }, /Z, [{, #, MUL VL}] // IF_SVE_IL_3A theEmitter->emitIns_R_R_R_I(INS_sve_ldnf1d, EA_SCALABLE, REG_V0, REG_P0, REG_R0, 0, diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 7660764c4d5806..1cb193292c7de5 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -1226,7 +1226,16 @@ void emitter::emitInsSanityCheck(instrDesc* id) case IF_SVE_JN_3C_D: // ............iiii ...gggnnnnnttttt -- SVE contiguous store (scalar plus immediate) case IF_SVE_JO_3A: // ............iiii ...gggnnnnnttttt -- SVE store multiple structures (scalar plus immediate) elemsize = id->idOpSize(); - assert(id->idInsOpt() == INS_OPTS_SCALABLE_D); +#ifdef DEBUG + if (id->idInsFmt() == IF_SVE_IH_3A_A) + { + assert(id->idInsOpt() == INS_OPTS_SCALABLE_Q); + } + else + { + assert(id->idInsOpt() == INS_OPTS_SCALABLE_D); + } +#endif // DEBUG assert(isVectorRegister(id->idReg1())); // ttttt assert(isPredicateRegister(id->idReg2())); // ggg assert(isGeneralRegister(id->idReg3())); // nnnnn @@ -5342,6 +5351,9 @@ emitter::code_t emitter::emitInsCodeSve(instruction ins, insFormat fmt) case INS_OPTS_SCALABLE_D_WITH_SCALAR: return EA_8BYTE; + case INS_OPTS_SCALABLE_Q: + return EA_16BYTE; + default: assert(!"Invalid insOpt for vector register"); return EA_UNKNOWN; @@ -9414,10 +9426,16 @@ void emitter::emitIns_R_R_R_I(instruction ins, assert(isPredicateRegister(reg2)); assert(isGeneralRegister(reg3)); assert(isValidSimm4(imm)); - fmt = IF_SVE_IH_3A; + if (opt == INS_OPTS_SCALABLE_Q) + { + fmt = IF_SVE_IH_3A_A; + } + else + { + fmt = IF_SVE_IH_3A; + } break; - case INS_sve_ldnf1sw: case INS_sve_ldnf1d: assert(insOptsScalable(opt)); @@ -15436,6 +15454,9 @@ void emitter::emitDispArrangement(insOpts opt) case INS_OPTS_SCALABLE_D_WITH_SCALAR: str = "d"; break; + case INS_OPTS_SCALABLE_Q: + str = "q"; + break; default: assert(!"Invalid insOpt for vector register"); @@ -17179,7 +17200,7 @@ void emitter::emitDispInsHelp( emitDispSveRegList(id->idReg1(), 1, id->idInsOpt(), true); // ttttt emitDispPredicateReg(id->idReg2(), PREDICATE_ZERO, true); // ggg printf("["); - emitDispReg(id->idReg3(), optGetSveElemsize(id->idInsOpt()), imm != 0); // nnnnn + emitDispReg(id->idReg3(), EA_8BYTE, imm != 0); // nnnnn if (imm != 0) { emitDispImm(emitGetInsSC(id), true); // iiii @@ -19681,6 +19702,12 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins result.insLatency = PERFSCORE_LATENCY_9C; break; + case IF_SVE_IH_3A_A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus + // immediate) + result.insThroughput = PERFSCORE_THROUGHPUT_1C; + result.insLatency = PERFSCORE_LATENCY_9C; + break; + case IF_SVE_IL_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus immediate) result.insThroughput = PERFSCORE_THROUGHPUT_3C; result.insLatency = PERFSCORE_LATENCY_6C; diff --git a/src/coreclr/jit/emitarm64.h b/src/coreclr/jit/emitarm64.h index b68f4d4572486e..6df7efda88cbf3 100644 --- a/src/coreclr/jit/emitarm64.h +++ b/src/coreclr/jit/emitarm64.h @@ -894,7 +894,7 @@ inline static bool insOptsScalableSimple(insOpts opt) { // `opt` is any of the standard scalable types. return ((opt == INS_OPTS_SCALABLE_B) || (opt == INS_OPTS_SCALABLE_H) || (opt == INS_OPTS_SCALABLE_S) || - (opt == INS_OPTS_SCALABLE_D)); + (opt == INS_OPTS_SCALABLE_D) || (opt == INS_OPTS_SCALABLE_Q)); } inline static bool insOptsScalableWords(insOpts opt) diff --git a/src/coreclr/jit/instr.h b/src/coreclr/jit/instr.h index 08f28dfe74270b..9d72a9d1068c80 100644 --- a/src/coreclr/jit/instr.h +++ b/src/coreclr/jit/instr.h @@ -274,6 +274,7 @@ enum insOpts : unsigned INS_OPTS_SCALABLE_H, INS_OPTS_SCALABLE_S, INS_OPTS_SCALABLE_D, + INS_OPTS_SCALABLE_Q, INS_OPTS_SCALABLE_WIDE_B, INS_OPTS_SCALABLE_WIDE_H, From dc0ab4d9076c2e71724e201448772e731bdb8875 Mon Sep 17 00:00:00 2001 From: TIHan Date: Sat, 9 Dec 2023 13:43:49 -0800 Subject: [PATCH 05/37] Added more formats --- src/coreclr/jit/codegenarm64.cpp | 24 ++++++++++++ src/coreclr/jit/emitarm64.cpp | 65 ++++++++++++++++++++++++++++++-- 2 files changed, 85 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 689e1fd7dd2fb3..7a66bb93e34a49 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -10662,6 +10662,30 @@ void CodeGen::genArm64EmitterUnitTests() theEmitter->emitIns_R_R_R_I(INS_sve_ld1d, EA_SCALABLE, REG_V0, REG_P2, REG_R3, 5, INS_OPTS_SCALABLE_Q); // LD1D {.Q }, /Z, [{, #, MUL VL}] + // IF_SVE_IH_3A_F + theEmitter->emitIns_R_R_R_I(INS_sve_ld1w, EA_SCALABLE, REG_V0, REG_P2, REG_R3, 3, + INS_OPTS_SCALABLE_D); // LD1W {.D }, /Z, [{, #, MUL VL}] + + // IF_SVE_IJ_3A + theEmitter->emitIns_R_R_R_I(INS_sve_ld1sw, EA_SCALABLE, REG_V0, REG_P5, REG_R3, 4, + INS_OPTS_SCALABLE_D); // LD1SW {.D }, /Z, [{, #, MUL VL}] + + // IF_SVE_IJ_3A_D + theEmitter->emitIns_R_R_R_I(INS_sve_ld1sb, EA_SCALABLE, REG_V3, REG_P0, REG_R2, 6, + INS_OPTS_SCALABLE_D); // LD1SB {.D }, /Z, [{, #, MUL VL}] + + // IF_SVE_IJ_3A_E + theEmitter->emitIns_R_R_R_I(INS_sve_ld1b, EA_SCALABLE, REG_V5, REG_P1, REG_R3, 7, + INS_OPTS_SCALABLE_D); // LD1B {.D }, /Z, [{, #, MUL VL}] + + // IF_SVE_IJ_3A_F + theEmitter->emitIns_R_R_R_I(INS_sve_ld1sh, EA_SCALABLE, REG_V7, REG_P3, REG_R5, 2, + INS_OPTS_SCALABLE_D); // LD1SH {.D }, /Z, [{, #, MUL VL}] + + // IF_SVE_IJ_3A_G + theEmitter->emitIns_R_R_R_I(INS_sve_ld1h, EA_SCALABLE, REG_V2, REG_P1, REG_R6, 1, + INS_OPTS_SCALABLE_D); // LD1H {.D }, /Z, [{, #, MUL VL}] + // IF_SVE_IL_3A theEmitter->emitIns_R_R_R_I(INS_sve_ldnf1d, EA_SCALABLE, REG_V0, REG_P0, REG_R0, 0, INS_OPTS_SCALABLE_D); // LDNF1D {.D }, /Z, [{, #, MUL VL}] diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 1cb193292c7de5..763a996da14149 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -9436,6 +9436,60 @@ void emitter::emitIns_R_R_R_I(instruction ins, } break; + case INS_sve_ld1w: + assert(opt == INS_OPTS_SCALABLE_D); + assert(isVectorRegister(reg1)); + assert(isPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isValidSimm4(imm)); + fmt = IF_SVE_IH_3A_F; + break; + + case INS_sve_ld1sw: + assert(opt == INS_OPTS_SCALABLE_D); + assert(isVectorRegister(reg1)); + assert(isPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isValidSimm4(imm)); + fmt = IF_SVE_IJ_3A; + break; + + case INS_sve_ld1sb: + assert(opt == INS_OPTS_SCALABLE_D); + assert(isVectorRegister(reg1)); + assert(isPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isValidSimm4(imm)); + fmt = IF_SVE_IJ_3A_D; + break; + + case INS_sve_ld1b: + assert(opt == INS_OPTS_SCALABLE_D); + assert(isVectorRegister(reg1)); + assert(isPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isValidSimm4(imm)); + fmt = IF_SVE_IJ_3A_E; + break; + + case INS_sve_ld1sh: + assert(opt == INS_OPTS_SCALABLE_D); + assert(isVectorRegister(reg1)); + assert(isPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isValidSimm4(imm)); + fmt = IF_SVE_IJ_3A_F; + break; + + case INS_sve_ld1h: + assert(opt == INS_OPTS_SCALABLE_D); + assert(isVectorRegister(reg1)); + assert(isPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isValidSimm4(imm)); + fmt = IF_SVE_IJ_3A_G; + break; + case INS_sve_ldnf1sw: case INS_sve_ldnf1d: assert(insOptsScalable(opt)); @@ -19698,12 +19752,15 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins case IF_SVE_IH_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus // immediate) - result.insThroughput = PERFSCORE_THROUGHPUT_1C; - result.insLatency = PERFSCORE_LATENCY_9C; - break; - case IF_SVE_IH_3A_A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus // immediate) + case IF_SVE_IH_3A_F: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus + // immediate) + case IF_SVE_IJ_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) + case IF_SVE_IJ_3A_D: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) + case IF_SVE_IJ_3A_E: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) + case IF_SVE_IJ_3A_F: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) + case IF_SVE_IJ_3A_G: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) result.insThroughput = PERFSCORE_THROUGHPUT_1C; result.insLatency = PERFSCORE_LATENCY_9C; break; From 2318532f0ed45751756566cfcaea292c23bcaaf1 Mon Sep 17 00:00:00 2001 From: TIHan Date: Sat, 9 Dec 2023 14:08:34 -0800 Subject: [PATCH 06/37] Added more formats --- src/coreclr/jit/codegenarm64.cpp | 18 +++++++++++++- src/coreclr/jit/emitarm64.cpp | 40 ++++++++++++++++++++++++++++++-- src/coreclr/jit/emitarm64.h | 4 ++-- 3 files changed, 57 insertions(+), 5 deletions(-) diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 7a66bb93e34a49..efdb232ce0c7fd 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -10694,7 +10694,23 @@ void CodeGen::genArm64EmitterUnitTests() theEmitter->emitIns_R_R_R_I(INS_sve_ldnf1d, EA_SCALABLE, REG_V0, REG_P1, REG_R2, 5, INS_OPTS_SCALABLE_D); // LDNF1D {.D }, /Z, [{, #, MUL VL}] theEmitter->emitIns_R_R_R_I(INS_sve_ldnf1sw, EA_SCALABLE, REG_V0, REG_P1, REG_R2, 5, - INS_OPTS_SCALABLE_D); // LDNF1SW {.D }, /Z, [{, #, MUL VL}] + INS_OPTS_SCALABLE_D); // LDNF1SW {.D }, /Z, [{, #, MUL VL}] + + // IF_SVE_IL_3A_A + theEmitter->emitIns_R_R_R_I(INS_sve_ldnf1sh, EA_SCALABLE, REG_V0, REG_P1, REG_R5, 5, + INS_OPTS_SCALABLE_D); // LDNF1SH {.D }, /Z, [{, #, MUL VL}] + theEmitter->emitIns_R_R_R_I(INS_sve_ldnf1w, EA_SCALABLE, REG_V0, REG_P2, REG_R4, 5, + INS_OPTS_SCALABLE_D); // LDNF1W {.D }, /Z, [{, #, MUL VL}] + + // IF_SVE_IL_3A_B + theEmitter->emitIns_R_R_R_I(INS_sve_ldnf1h, EA_SCALABLE, REG_V1, REG_P3, REG_R2, 5, + INS_OPTS_SCALABLE_D); // LDNF1H {.D }, /Z, [{, #, MUL VL}] + theEmitter->emitIns_R_R_R_I(INS_sve_ldnf1sb, EA_SCALABLE, REG_V0, REG_P4, REG_R1, 5, + INS_OPTS_SCALABLE_D); // LDNF1SB {.D }, /Z, [{, #, MUL VL}] + + // IF_SVE_IL_3A_C + theEmitter->emitIns_R_R_R_I(INS_sve_ldnf1b, EA_SCALABLE, REG_V2, REG_P5, REG_R3, 5, + INS_OPTS_SCALABLE_D); // LDNF1B {.D }, /Z, [{, #, MUL VL}] #endif // ALL_ARM64_EMITTER_UNIT_TESTS_SVE diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 763a996da14149..949b1573faa054 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -1233,6 +1233,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) } else { + assert(id->idInsOpt() == INS_OPTS_SCALABLE_D); } #endif // DEBUG @@ -9500,6 +9501,35 @@ void emitter::emitIns_R_R_R_I(instruction ins, fmt = IF_SVE_IL_3A; break; + case INS_sve_ldnf1sh: + case INS_sve_ldnf1w: + assert(insOptsScalable(opt)); + assert(isVectorRegister(reg1)); + assert(isPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isValidSimm4(imm)); + fmt = IF_SVE_IL_3A_A; + break; + + case INS_sve_ldnf1h: + case INS_sve_ldnf1sb: + assert(insOptsScalable(opt)); + assert(isVectorRegister(reg1)); + assert(isPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isValidSimm4(imm)); + fmt = IF_SVE_IL_3A_B; + break; + + case INS_sve_ldnf1b: + assert(insOptsScalable(opt)); + assert(isVectorRegister(reg1)); + assert(isPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isValidSimm4(imm)); + fmt = IF_SVE_IL_3A_C; + break; + default: unreached(); break; @@ -19750,8 +19780,8 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins result.insLatency = PERFSCORE_LATENCY_2C; break; - case IF_SVE_IH_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus - // immediate) + case IF_SVE_IH_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus + // immediate) case IF_SVE_IH_3A_A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus // immediate) case IF_SVE_IH_3A_F: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus @@ -19766,6 +19796,12 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins break; case IF_SVE_IL_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus immediate) + case IF_SVE_IL_3A_A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus + // immediate) + case IF_SVE_IL_3A_B: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus + // immediate) + case IF_SVE_IL_3A_C: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus + // immediate) result.insThroughput = PERFSCORE_THROUGHPUT_3C; result.insLatency = PERFSCORE_LATENCY_6C; break; diff --git a/src/coreclr/jit/emitarm64.h b/src/coreclr/jit/emitarm64.h index 6df7efda88cbf3..d454b451ad77ff 100644 --- a/src/coreclr/jit/emitarm64.h +++ b/src/coreclr/jit/emitarm64.h @@ -887,14 +887,14 @@ inline static bool insOptsScalable(insOpts opt) { // Opt is any of the scalable types. return ((insOptsScalableSimple(opt)) || (insOptsScalableWide(opt)) || (insOptsScalableWithSimdScalar(opt)) || - (insOptsScalableWithScalar(opt)) || (insOptsScalableWithSimdVector(opt))); + (insOptsScalableWithScalar(opt)) || (insOptsScalableWithSimdVector(opt)) || (opt == INS_OPTS_SCALABLE_Q)); } inline static bool insOptsScalableSimple(insOpts opt) { // `opt` is any of the standard scalable types. return ((opt == INS_OPTS_SCALABLE_B) || (opt == INS_OPTS_SCALABLE_H) || (opt == INS_OPTS_SCALABLE_S) || - (opt == INS_OPTS_SCALABLE_D) || (opt == INS_OPTS_SCALABLE_Q)); + (opt == INS_OPTS_SCALABLE_D)); } inline static bool insOptsScalableWords(insOpts opt) From 3ebbb3d4b02ad7f7efb18ac17db46763ff4791d5 Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 13 Dec 2023 11:04:57 -0800 Subject: [PATCH 07/37] Properly encoding ld1w --- src/coreclr/jit/codegenarm64.cpp | 4 ++ src/coreclr/jit/emitarm64.cpp | 66 ++++++++++++++++++++++++++------ src/coreclr/jit/emitarm64.h | 10 +++++ src/coreclr/jit/instrsarm64sve.h | 2 +- 4 files changed, 69 insertions(+), 13 deletions(-) diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index efdb232ce0c7fd..7979aabbfa44cc 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -10663,8 +10663,12 @@ void CodeGen::genArm64EmitterUnitTests() INS_OPTS_SCALABLE_Q); // LD1D {.Q }, /Z, [{, #, MUL VL}] // IF_SVE_IH_3A_F + theEmitter->emitIns_R_R_R_I(INS_sve_ld1w, EA_SCALABLE, REG_V0, REG_P2, REG_R3, 3, + INS_OPTS_SCALABLE_S); // LD1W {.S }, /Z, [{, #, MUL VL}] theEmitter->emitIns_R_R_R_I(INS_sve_ld1w, EA_SCALABLE, REG_V0, REG_P2, REG_R3, 3, INS_OPTS_SCALABLE_D); // LD1W {.D }, /Z, [{, #, MUL VL}] + theEmitter->emitIns_R_R_R_I(INS_sve_ld1w, EA_SCALABLE, REG_V0, REG_P2, REG_R3, 3, + INS_OPTS_SCALABLE_Q); // LD1W {.Q }, /Z, [{, #, MUL VL}] // IF_SVE_IJ_3A theEmitter->emitIns_R_R_R_I(INS_sve_ld1sw, EA_SCALABLE, REG_V0, REG_P5, REG_R3, 4, diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 949b1573faa054..4e0915e13016ae 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -1226,17 +1226,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) case IF_SVE_JN_3C_D: // ............iiii ...gggnnnnnttttt -- SVE contiguous store (scalar plus immediate) case IF_SVE_JO_3A: // ............iiii ...gggnnnnnttttt -- SVE store multiple structures (scalar plus immediate) elemsize = id->idOpSize(); -#ifdef DEBUG - if (id->idInsFmt() == IF_SVE_IH_3A_A) - { - assert(id->idInsOpt() == INS_OPTS_SCALABLE_Q); - } - else - { - - assert(id->idInsOpt() == INS_OPTS_SCALABLE_D); - } -#endif // DEBUG + assert(insOptsScalable(id->idInsOpt())); assert(isVectorRegister(id->idReg1())); // ttttt assert(isPredicateRegister(id->idReg2())); // ggg assert(isGeneralRegister(id->idReg3())); // nnnnn @@ -9433,12 +9423,13 @@ void emitter::emitIns_R_R_R_I(instruction ins, } else { + assert(opt == INS_OPTS_SCALABLE_D); fmt = IF_SVE_IH_3A; } break; case INS_sve_ld1w: - assert(opt == INS_OPTS_SCALABLE_D); + assert(insOptsScalableWordsOrQuadwords(opt)); assert(isVectorRegister(reg1)); assert(isPredicateRegister(reg2)); assert(isGeneralRegister(reg3)); @@ -12772,6 +12763,51 @@ void emitter::emitIns_Call(EmitCallType callType, return 0; } +/*static*/ bool emitter::canEncodeSveElemsize_dtype(insFormat fmt) +{ + switch (fmt) + { + case IF_SVE_IH_3A_F: + return true; + + default: + return false; + } +} + +/***************************************************************************** + * + * Returns the encoding to select the 1/2/4/8 byte elemsize for an Arm64 Sve vector instruction + * This specifically encodes the field 'tszh:tszl' at bit locations '22:20-19'. + */ + +/*static*/ emitter::code_t emitter::insEncodeSveElemsize_dtype(insFormat fmt, emitAttr size, code_t code) +{ + assert(canEncodeSveElemsize_dtype(fmt)); + switch (fmt) + { + case IF_SVE_IH_3A_F: + switch (size) + { + case EA_4BYTE: + return code & ~(1 << 21); // Set bit '21' to 0. + + case EA_8BYTE: + return code; // By default, the instruction already encodes 64-bit. + + case EA_16BYTE: + return (code & ~((1 << 22) | (1 << 21) | (1 << 15))) | (1 << 20); // Set bits '22', '21' and '15' to 0. Set bit '20' to 1. + + default: + assert(!"Invalid size for encoding dtype."); + } + + default: + assert(!"Invalid format for encoding dtype."); + } + return code; +} + /***************************************************************************** * * TODO @@ -14868,6 +14904,12 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg code |= insEncodeSimm4_19_to_16(imm); // iiii + + if (canEncodeSveElemsize_dtype(fmt)) + { + code = insEncodeSveElemsize_dtype(fmt, optGetSveElemsize(id->idInsOpt()), code); + } + dst += emitOutput_Instr(dst, code); break; diff --git a/src/coreclr/jit/emitarm64.h b/src/coreclr/jit/emitarm64.h index d454b451ad77ff..d9f6dcf59addf2 100644 --- a/src/coreclr/jit/emitarm64.h +++ b/src/coreclr/jit/emitarm64.h @@ -472,6 +472,10 @@ static code_t insEncodeSveElemsize(insOpts opt); // This specifically encodes the field 'tszh:tszl' at bit locations '22:20-19'. static code_t insEncodeSveElemsize_tszh_22_tszl_20_to_19(emitAttr size); +static bool canEncodeSveElemsize_dtype(insFormat fmt); + +static code_t insEncodeSveElemsize_dtype(insFormat fmt, emitAttr size, code_t code); + static code_t insEncodeSimm4_19_to_16(ssize_t imm); // Returns true if 'reg' represents an integer register. @@ -903,6 +907,12 @@ inline static bool insOptsScalableWords(insOpts opt) return ((opt == INS_OPTS_SCALABLE_S) || (opt == INS_OPTS_SCALABLE_D)); } +inline static bool insOptsScalableWordsOrQuadwords(insOpts opt) +{ + // `opt` is any of the standard word, quadword and above scalable types. + return (insOptsScalableWords(opt) || (opt == INS_OPTS_SCALABLE_Q)); +} + inline static bool insOptsScalableAtLeastHalf(insOpts opt) { // `opt` is any of the standard half and above scalable types. diff --git a/src/coreclr/jit/instrsarm64sve.h b/src/coreclr/jit/instrsarm64sve.h index d10b075ad09467..e78399518ef4d5 100644 --- a/src/coreclr/jit/instrsarm64sve.h +++ b/src/coreclr/jit/instrsarm64sve.h @@ -132,7 +132,7 @@ INST9(ld1h, "ld1h", 0, IF_SV // enum name info SVE_HW_4A SVE_HW_4A_A SVE_HW_4A_B SVE_HW_4A_C SVE_HW_4B SVE_HW_4B_D SVE_HX_3A_E SVE_IH_3A_F SVE_II_4A_H -INST9(ld1w, "ld1w", 0, IF_SVE_9C, 0x85204000, 0xC5204000, 0xC5004000, 0x85004000, 0xC560C000, 0xC540C000, 0x8520C000, 0xA5002000, 0xA5000000 ) +INST9(ld1w, "ld1w", 0, IF_SVE_9C, 0x85204000, 0xC5204000, 0xC5004000, 0x85004000, 0xC560C000, 0xC540C000, 0x8520C000, 0xA560A000, 0xA5000000 ) // LD1W {.S }, /Z, [, .S, #2] SVE_HW_4A 100001010h1mmmmm 010gggnnnnnttttt 8520 4000 // LD1W {.D }, /Z, [, .D, #2] SVE_HW_4A_A 110001010h1mmmmm 010gggnnnnnttttt C520 4000 // LD1W {.D }, /Z, [, .D, ] SVE_HW_4A_B 110001010h0mmmmm 010gggnnnnnttttt C500 4000 From 0ef3323dada1a95e99c68620a1caac702330308e Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 13 Dec 2023 11:19:35 -0800 Subject: [PATCH 08/37] Encoding ld1sb properly --- src/coreclr/jit/codegenarm64.cpp | 4 ++++ src/coreclr/jit/emitarm64.cpp | 19 ++++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 7979aabbfa44cc..7268a10dee64c8 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -10675,6 +10675,10 @@ void CodeGen::genArm64EmitterUnitTests() INS_OPTS_SCALABLE_D); // LD1SW {.D }, /Z, [{, #, MUL VL}] // IF_SVE_IJ_3A_D + theEmitter->emitIns_R_R_R_I(INS_sve_ld1sb, EA_SCALABLE, REG_V3, REG_P0, REG_R2, 6, + INS_OPTS_SCALABLE_H); // LD1SB {.H }, /Z, [{, #, MUL VL}] + theEmitter->emitIns_R_R_R_I(INS_sve_ld1sb, EA_SCALABLE, REG_V3, REG_P0, REG_R2, 6, + INS_OPTS_SCALABLE_S); // LD1SB {.S }, /Z, [{, #, MUL VL}] theEmitter->emitIns_R_R_R_I(INS_sve_ld1sb, EA_SCALABLE, REG_V3, REG_P0, REG_R2, 6, INS_OPTS_SCALABLE_D); // LD1SB {.D }, /Z, [{, #, MUL VL}] diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 4e0915e13016ae..5b56ac082350b3 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -9447,7 +9447,7 @@ void emitter::emitIns_R_R_R_I(instruction ins, break; case INS_sve_ld1sb: - assert(opt == INS_OPTS_SCALABLE_D); + assert(insOptsScalableAtLeastHalf(opt)); assert(isVectorRegister(reg1)); assert(isPredicateRegister(reg2)); assert(isGeneralRegister(reg3)); @@ -12768,6 +12768,7 @@ void emitter::emitIns_Call(EmitCallType callType, switch (fmt) { case IF_SVE_IH_3A_F: + case IF_SVE_IJ_3A_D: return true; default: @@ -12802,6 +12803,22 @@ void emitter::emitIns_Call(EmitCallType callType, assert(!"Invalid size for encoding dtype."); } + case IF_SVE_IJ_3A_D: + switch (size) + { + case EA_2BYTE: + return code | (1 << 22); // Set bit '22' to 1. + + case EA_4BYTE: + return code | (1 << 21); // Set bit '21' to 1. + + case EA_8BYTE: + return code; // By default, the instruction already encodes 64-bit. + + default: + assert(!"Invalid size for encoding dtype."); + } + default: assert(!"Invalid format for encoding dtype."); } From 523a544000fecd24361f38dc2121dd9a2f8516e1 Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 13 Dec 2023 11:41:39 -0800 Subject: [PATCH 09/37] Properly encoding ld1b --- src/coreclr/jit/codegenarm64.cpp | 6 ++++++ src/coreclr/jit/emitarm64.cpp | 19 ++++++++++++++++++- src/coreclr/jit/instrsarm64sve.h | 6 +++--- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 7268a10dee64c8..5fe8a23d089375 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -10683,6 +10683,12 @@ void CodeGen::genArm64EmitterUnitTests() INS_OPTS_SCALABLE_D); // LD1SB {.D }, /Z, [{, #, MUL VL}] // IF_SVE_IJ_3A_E + theEmitter->emitIns_R_R_R_I(INS_sve_ld1b, EA_SCALABLE, REG_V5, REG_P1, REG_R3, 7, + INS_OPTS_SCALABLE_B); // LD1B {.B }, /Z, [{, #, MUL VL}] + theEmitter->emitIns_R_R_R_I(INS_sve_ld1b, EA_SCALABLE, REG_V5, REG_P1, REG_R3, 7, + INS_OPTS_SCALABLE_H); // LD1B {.H }, /Z, [{, #, MUL VL}] + theEmitter->emitIns_R_R_R_I(INS_sve_ld1b, EA_SCALABLE, REG_V5, REG_P1, REG_R3, 7, + INS_OPTS_SCALABLE_S); // LD1B {.S }, /Z, [{, #, MUL VL}] theEmitter->emitIns_R_R_R_I(INS_sve_ld1b, EA_SCALABLE, REG_V5, REG_P1, REG_R3, 7, INS_OPTS_SCALABLE_D); // LD1B {.D }, /Z, [{, #, MUL VL}] diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 5b56ac082350b3..79743adc9127e5 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -9456,7 +9456,7 @@ void emitter::emitIns_R_R_R_I(instruction ins, break; case INS_sve_ld1b: - assert(opt == INS_OPTS_SCALABLE_D); + assert(insOptsScalableSimple(opt)); assert(isVectorRegister(reg1)); assert(isPredicateRegister(reg2)); assert(isGeneralRegister(reg3)); @@ -12769,6 +12769,7 @@ void emitter::emitIns_Call(EmitCallType callType, { case IF_SVE_IH_3A_F: case IF_SVE_IJ_3A_D: + case IF_SVE_IJ_3A_E: return true; default: @@ -12819,6 +12820,22 @@ void emitter::emitIns_Call(EmitCallType callType, assert(!"Invalid size for encoding dtype."); } + case IF_SVE_IJ_3A_E: + switch (size) + { + case EA_1BYTE: + return code & ~((1 << 22) | (1 << 21)); // Set bit '22' and '21' to 0. + + case EA_2BYTE: + return code & ~(1 << 22); // Set bit '22' to 0. + + case EA_4BYTE: + return code & ~(1 << 21); // Set bit '21' to 0. + + case EA_8BYTE: + return code; // By default, the instruction already encodes 64-bit. + } + default: assert(!"Invalid format for encoding dtype."); } diff --git a/src/coreclr/jit/instrsarm64sve.h b/src/coreclr/jit/instrsarm64sve.h index e78399518ef4d5..01ecba92d33417 100644 --- a/src/coreclr/jit/instrsarm64sve.h +++ b/src/coreclr/jit/instrsarm64sve.h @@ -140,7 +140,7 @@ INST9(ld1w, "ld1w", 0, IF_SV // LD1W {.D }, /Z, [, .D, LSL #2] SVE_HW_4B 11000101011mmmmm 110gggnnnnnttttt C560 C000 // LD1W {.D }, /Z, [, .D] SVE_HW_4B_D 11000101010mmmmm 110gggnnnnnttttt C540 C000 // LD1W {.D }, /Z, [.D{, #}] SVE_HX_3A_E 10000101001iiiii 110gggnnnnnttttt 8520 C000 - // LD1W {.D }, /Z, [{, #, MUL VL}] SVE_IH_3A_F 101001010000iiii 001gggnnnnnttttt A500 2000 + // LD1W {.D }, /Z, [{, #, MUL VL}] SVE_IH_3A_F 101001010110iiii 101gggnnnnnttttt A560 A000 // LD1W {.D }, /Z, [, , LSL #2] SVE_II_4A_H 10100101000mmmmm 000gggnnnnnttttt A500 0000 @@ -271,12 +271,12 @@ INST6(ld1sb, "ld1sb", 0, IF_SV // enum name info SVE_HW_4A SVE_HW_4A_A SVE_HW_4B SVE_HX_3A_B SVE_IJ_3A_E SVE_IK_4A_H -INST6(ld1b, "ld1b", 0, IF_SVE_6D, 0xC4004000, 0x84004000, 0xC440C000, 0x8420C000, 0xA400A000, 0xA4004000 ) +INST6(ld1b, "ld1b", 0, IF_SVE_6D, 0xC4004000, 0x84004000, 0xC440C000, 0x8420C000, 0xA460A000, 0xA4004000 ) // LD1B {.D }, /Z, [, .D, ] SVE_HW_4A 110001000h0mmmmm 010gggnnnnnttttt C400 4000 // LD1B {.S }, /Z, [, .S, ] SVE_HW_4A_A 100001000h0mmmmm 010gggnnnnnttttt 8400 4000 // LD1B {.D }, /Z, [, .D] SVE_HW_4B 11000100010mmmmm 110gggnnnnnttttt C440 C000 // LD1B {.D }, /Z, [.D{, #}] SVE_HX_3A_B 10000100001iiiii 110gggnnnnnttttt 8420 C000 - // LD1B {.D }, /Z, [{, #, MUL VL}] SVE_IJ_3A_E 101001000000iiii 101gggnnnnnttttt A400 A000 + // LD1B {.D }, /Z, [{, #, MUL VL}] SVE_IJ_3A_E 101001000110iiii 101gggnnnnnttttt A460 A000 // LD1B {.D }, /Z, [, ] SVE_IK_4A_H 10100100000mmmmm 010gggnnnnnttttt A400 4000 From 08ca147f3496d1aeaee7aaaf672d2ef3152475eb Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 13 Dec 2023 11:49:32 -0800 Subject: [PATCH 10/37] Properly encoding ld1sh --- src/coreclr/jit/codegenarm64.cpp | 2 ++ src/coreclr/jit/emitarm64.cpp | 19 ++++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 5fe8a23d089375..1335da349d0130 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -10693,6 +10693,8 @@ void CodeGen::genArm64EmitterUnitTests() INS_OPTS_SCALABLE_D); // LD1B {.D }, /Z, [{, #, MUL VL}] // IF_SVE_IJ_3A_F + theEmitter->emitIns_R_R_R_I(INS_sve_ld1sh, EA_SCALABLE, REG_V7, REG_P3, REG_R5, 2, + INS_OPTS_SCALABLE_S); // LD1SH {.S }, /Z, [{, #, MUL VL}] theEmitter->emitIns_R_R_R_I(INS_sve_ld1sh, EA_SCALABLE, REG_V7, REG_P3, REG_R5, 2, INS_OPTS_SCALABLE_D); // LD1SH {.D }, /Z, [{, #, MUL VL}] diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 79743adc9127e5..72651b06ce324c 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -9465,7 +9465,7 @@ void emitter::emitIns_R_R_R_I(instruction ins, break; case INS_sve_ld1sh: - assert(opt == INS_OPTS_SCALABLE_D); + assert(insOptsScalableWords(opt)); assert(isVectorRegister(reg1)); assert(isPredicateRegister(reg2)); assert(isGeneralRegister(reg3)); @@ -12770,6 +12770,7 @@ void emitter::emitIns_Call(EmitCallType callType, case IF_SVE_IH_3A_F: case IF_SVE_IJ_3A_D: case IF_SVE_IJ_3A_E: + case IF_SVE_IJ_3A_F: return true; default: @@ -12834,6 +12835,22 @@ void emitter::emitIns_Call(EmitCallType callType, case EA_8BYTE: return code; // By default, the instruction already encodes 64-bit. + + default: + assert(!"Invalid size for encoding dtype."); + } + + case IF_SVE_IJ_3A_F: + switch (size) + { + case EA_4BYTE: + return code | (1 << 21); // Set bit '21' to 1. + + case EA_8BYTE: + return code; // By default, the instruction already encodes 64-bit. + + default: + assert(!"Invalid size for encoding dtype."); } default: From e8aaf876cc98ae39a80fde37153fc5f723d3a836 Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 13 Dec 2023 12:08:25 -0800 Subject: [PATCH 11/37] Properly encoding ld1h --- src/coreclr/jit/codegenarm64.cpp | 4 ++++ src/coreclr/jit/emitarm64.cpp | 27 +++++++++++++++++++++++++-- src/coreclr/jit/instrsarm64sve.h | 4 ++-- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 1335da349d0130..bd83b51a479084 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -10699,6 +10699,10 @@ void CodeGen::genArm64EmitterUnitTests() INS_OPTS_SCALABLE_D); // LD1SH {.D }, /Z, [{, #, MUL VL}] // IF_SVE_IJ_3A_G + theEmitter->emitIns_R_R_R_I(INS_sve_ld1h, EA_SCALABLE, REG_V2, REG_P1, REG_R6, 1, + INS_OPTS_SCALABLE_H); // LD1H {.H }, /Z, [{, #, MUL VL}] + theEmitter->emitIns_R_R_R_I(INS_sve_ld1h, EA_SCALABLE, REG_V2, REG_P1, REG_R6, 1, + INS_OPTS_SCALABLE_S); // LD1H {.S }, /Z, [{, #, MUL VL}] theEmitter->emitIns_R_R_R_I(INS_sve_ld1h, EA_SCALABLE, REG_V2, REG_P1, REG_R6, 1, INS_OPTS_SCALABLE_D); // LD1H {.D }, /Z, [{, #, MUL VL}] diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 72651b06ce324c..c35aec3198bfac 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -9474,7 +9474,7 @@ void emitter::emitIns_R_R_R_I(instruction ins, break; case INS_sve_ld1h: - assert(opt == INS_OPTS_SCALABLE_D); + assert(insOptsScalableAtLeastHalf(opt)); assert(isVectorRegister(reg1)); assert(isPredicateRegister(reg2)); assert(isGeneralRegister(reg3)); @@ -12763,6 +12763,11 @@ void emitter::emitIns_Call(EmitCallType callType, return 0; } +/***************************************************************************** + * + * Returns true if the specified format can encode the 'dtype' field. + */ + /*static*/ bool emitter::canEncodeSveElemsize_dtype(insFormat fmt) { switch (fmt) @@ -12771,6 +12776,7 @@ void emitter::emitIns_Call(EmitCallType callType, case IF_SVE_IJ_3A_D: case IF_SVE_IJ_3A_E: case IF_SVE_IJ_3A_F: + case IF_SVE_IJ_3A_G: return true; default: @@ -12781,7 +12787,7 @@ void emitter::emitIns_Call(EmitCallType callType, /***************************************************************************** * * Returns the encoding to select the 1/2/4/8 byte elemsize for an Arm64 Sve vector instruction - * This specifically encodes the field 'tszh:tszl' at bit locations '22:20-19'. + * based on the format for the 'dtype' field. */ /*static*/ emitter::code_t emitter::insEncodeSveElemsize_dtype(insFormat fmt, emitAttr size, code_t code) @@ -12799,6 +12805,7 @@ void emitter::emitIns_Call(EmitCallType callType, return code; // By default, the instruction already encodes 64-bit. case EA_16BYTE: + // Note: Bit '15' is not actually part of 'dtype', but it is necessary to set to '0' to get the proper encoding for Q. return (code & ~((1 << 22) | (1 << 21) | (1 << 15))) | (1 << 20); // Set bits '22', '21' and '15' to 0. Set bit '20' to 1. default: @@ -12853,6 +12860,22 @@ void emitter::emitIns_Call(EmitCallType callType, assert(!"Invalid size for encoding dtype."); } + case IF_SVE_IJ_3A_G: + switch (size) + { + case EA_2BYTE: + return code & ~(1 << 22); // Set bit '22' to 0. + + case EA_4BYTE: + return code & ~(1 << 21); // Set bit '21' to 0. + + case EA_8BYTE: + return code; // By default, the instruction already encodes 64-bit. + + default: + assert(!"Invalid size for encoding dtype."); + } + default: assert(!"Invalid format for encoding dtype."); } diff --git a/src/coreclr/jit/instrsarm64sve.h b/src/coreclr/jit/instrsarm64sve.h index 01ecba92d33417..b2640aa65e967b 100644 --- a/src/coreclr/jit/instrsarm64sve.h +++ b/src/coreclr/jit/instrsarm64sve.h @@ -119,7 +119,7 @@ INST9(ld1sh, "ld1sh", 0, IF_SV // enum name info SVE_HW_4A SVE_HW_4A_A SVE_HW_4A_B SVE_HW_4A_C SVE_HW_4B SVE_HW_4B_D SVE_HX_3A_E SVE_IJ_3A_G SVE_IK_4A_I -INST9(ld1h, "ld1h", 0, IF_SVE_9B, 0x84A04000, 0xC4A04000, 0xC4804000, 0x84804000, 0xC4E0C000, 0xC4C0C000, 0x84A0C000, 0xA480A000, 0xA4804000 ) +INST9(ld1h, "ld1h", 0, IF_SVE_9B, 0x84A04000, 0xC4A04000, 0xC4804000, 0x84804000, 0xC4E0C000, 0xC4C0C000, 0x84A0C000, 0xA4E0A000, 0xA4804000 ) // LD1H {.S }, /Z, [, .S, #1] SVE_HW_4A 100001001h1mmmmm 010gggnnnnnttttt 84A0 4000 // LD1H {.D }, /Z, [, .D, #1] SVE_HW_4A_A 110001001h1mmmmm 010gggnnnnnttttt C4A0 4000 // LD1H {.D }, /Z, [, .D, ] SVE_HW_4A_B 110001001h0mmmmm 010gggnnnnnttttt C480 4000 @@ -127,7 +127,7 @@ INST9(ld1h, "ld1h", 0, IF_SV // LD1H {.D }, /Z, [, .D, LSL #1] SVE_HW_4B 11000100111mmmmm 110gggnnnnnttttt C4E0 C000 // LD1H {.D }, /Z, [, .D] SVE_HW_4B_D 11000100110mmmmm 110gggnnnnnttttt C4C0 C000 // LD1H {.D }, /Z, [.D{, #}] SVE_HX_3A_E 10000100101iiiii 110gggnnnnnttttt 84A0 C000 - // LD1H {.D }, /Z, [{, #, MUL VL}] SVE_IJ_3A_G 101001001000iiii 101gggnnnnnttttt A480 A000 + // LD1H {.D }, /Z, [{, #, MUL VL}] SVE_IJ_3A_G 101001001110iiii 101gggnnnnnttttt A4E0 A000 // LD1H {.D }, /Z, [, , LSL #1] SVE_IK_4A_I 10100100100mmmmm 010gggnnnnnttttt A480 4000 From 9300110a2bf5fe5cb0fa2f7df6be021b8596da22 Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 13 Dec 2023 12:10:57 -0800 Subject: [PATCH 12/37] Fixing up comments --- src/coreclr/jit/emitarm64.cpp | 2 +- src/coreclr/jit/emitarm64.h | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index c35aec3198bfac..6751baa780906f 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -12786,7 +12786,7 @@ void emitter::emitIns_Call(EmitCallType callType, /***************************************************************************** * - * Returns the encoding to select the 1/2/4/8 byte elemsize for an Arm64 Sve vector instruction + * Returns the encoding to select the 1/2/4/8/16 byte elemsize for an Arm64 Sve vector instruction * based on the format for the 'dtype' field. */ diff --git a/src/coreclr/jit/emitarm64.h b/src/coreclr/jit/emitarm64.h index d9f6dcf59addf2..1ef35458100911 100644 --- a/src/coreclr/jit/emitarm64.h +++ b/src/coreclr/jit/emitarm64.h @@ -472,8 +472,11 @@ static code_t insEncodeSveElemsize(insOpts opt); // This specifically encodes the field 'tszh:tszl' at bit locations '22:20-19'. static code_t insEncodeSveElemsize_tszh_22_tszl_20_to_19(emitAttr size); +// Returns true if the specified format can encode the 'dtype' field. static bool canEncodeSveElemsize_dtype(insFormat fmt); +// Returns the encoding to select the 1/2/4/8/16 byte elemsize for an Arm64 Sve vector instruction +// based on the format for the 'dtype' field. static code_t insEncodeSveElemsize_dtype(insFormat fmt, emitAttr size, code_t code); static code_t insEncodeSimm4_19_to_16(ssize_t imm); From 8233187ef6ac418e5298fe25e8bc1868af972741 Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 13 Dec 2023 12:26:20 -0800 Subject: [PATCH 13/37] Simplifying insEncodeSveElemsize_dtype --- src/coreclr/jit/emitarm64.cpp | 99 ++++++++++++++--------------------- 1 file changed, 40 insertions(+), 59 deletions(-) diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 6751baa780906f..950f2f2dce8292 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -12793,92 +12793,73 @@ void emitter::emitIns_Call(EmitCallType callType, /*static*/ emitter::code_t emitter::insEncodeSveElemsize_dtype(insFormat fmt, emitAttr size, code_t code) { assert(canEncodeSveElemsize_dtype(fmt)); - switch (fmt) + + switch (size) { - case IF_SVE_IH_3A_F: - switch (size) + case EA_1BYTE: + switch (fmt) { - case EA_4BYTE: - return code & ~(1 << 21); // Set bit '21' to 0. - - case EA_8BYTE: - return code; // By default, the instruction already encodes 64-bit. - - case EA_16BYTE: - // Note: Bit '15' is not actually part of 'dtype', but it is necessary to set to '0' to get the proper encoding for Q. - return (code & ~((1 << 22) | (1 << 21) | (1 << 15))) | (1 << 20); // Set bits '22', '21' and '15' to 0. Set bit '20' to 1. + case IF_SVE_IJ_3A_E: + return code & ~((1 << 22) | (1 << 21)); // Set bit '22' and '21' to 0. default: - assert(!"Invalid size for encoding dtype."); + assert(!"Invalid format for encoding dtype."); } + return code; - case IF_SVE_IJ_3A_D: - switch (size) + case EA_2BYTE: + switch (fmt) { - case EA_2BYTE: - return code | (1 << 22); // Set bit '22' to 1. - - case EA_4BYTE: - return code | (1 << 21); // Set bit '21' to 1. + case IF_SVE_IJ_3A_E: + case IF_SVE_IJ_3A_G: + return code & ~(1 << 22); // Set bit '22' to 0. - case EA_8BYTE: - return code; // By default, the instruction already encodes 64-bit. + case IF_SVE_IJ_3A_D: + return code | (1 << 22); // Set bit '22' to 1. default: - assert(!"Invalid size for encoding dtype."); + assert(!"Invalid format for encoding dtype."); } + return code; - case IF_SVE_IJ_3A_E: - switch (size) + case EA_4BYTE: + switch (fmt) { - case EA_1BYTE: - return code & ~((1 << 22) | (1 << 21)); // Set bit '22' and '21' to 0. - - case EA_2BYTE: - return code & ~(1 << 22); // Set bit '22' to 0. - - case EA_4BYTE: + case IF_SVE_IH_3A_F: + case IF_SVE_IJ_3A_E: + case IF_SVE_IJ_3A_G: return code & ~(1 << 21); // Set bit '21' to 0. - case EA_8BYTE: - return code; // By default, the instruction already encodes 64-bit. - - default: - assert(!"Invalid size for encoding dtype."); - } - - case IF_SVE_IJ_3A_F: - switch (size) - { - case EA_4BYTE: + case IF_SVE_IJ_3A_D: + case IF_SVE_IJ_3A_F: return code | (1 << 21); // Set bit '21' to 1. - case EA_8BYTE: - return code; // By default, the instruction already encodes 64-bit. - default: - assert(!"Invalid size for encoding dtype."); + assert(!"Invalid format for encoding dtype."); } + return code; - case IF_SVE_IJ_3A_G: - switch (size) - { - case EA_2BYTE: - return code & ~(1 << 22); // Set bit '22' to 0. - - case EA_4BYTE: - return code & ~(1 << 21); // Set bit '21' to 0. + case EA_8BYTE: + return code; // By default, the instruction already encodes 64-bit. - case EA_8BYTE: - return code; // By default, the instruction already encodes 64-bit. + case EA_16BYTE: + switch (fmt) + { + case IF_SVE_IH_3A_F: + // Note: Bit '15' is not actually part of 'dtype', but it is necessary to set to '0' to get the + // proper encoding for Q. + return (code & ~((1 << 22) | (1 << 21) | (1 << 15))) | + (1 << 20); // Set bits '22', '21' and '15' to 0. Set bit '20' to 1. default: - assert(!"Invalid size for encoding dtype."); + assert(!"Invalid format for encoding dtype."); } + return code; default: - assert(!"Invalid format for encoding dtype."); + assert(!"Invalid size for encoding dtype."); } + return code; } From 224a8961281bdaf3910d90a60697d7b0d4fc56b9 Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 13 Dec 2023 13:06:20 -0800 Subject: [PATCH 14/37] Instruction determines how to encode the dtype --- src/coreclr/jit/codegenarm64.cpp | 4 ++ src/coreclr/jit/emitarm64.cpp | 70 +++++++++++++++++--------------- src/coreclr/jit/emitarm64.h | 8 ++-- src/coreclr/jit/instrsarm64sve.h | 4 +- 4 files changed, 47 insertions(+), 39 deletions(-) diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index bd83b51a479084..e776502734946a 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -10717,6 +10717,10 @@ void CodeGen::genArm64EmitterUnitTests() INS_OPTS_SCALABLE_D); // LDNF1SW {.D }, /Z, [{, #, MUL VL}] // IF_SVE_IL_3A_A + theEmitter->emitIns_R_R_R_I(INS_sve_ldnf1sh, EA_SCALABLE, REG_V0, REG_P1, REG_R5, 5, + INS_OPTS_SCALABLE_S); // LDNF1SH {.S }, /Z, [{, #, MUL VL}] + theEmitter->emitIns_R_R_R_I(INS_sve_ldnf1w, EA_SCALABLE, REG_V0, REG_P2, REG_R4, 5, + INS_OPTS_SCALABLE_S); // LDNF1W {.S }, /Z, [{, #, MUL VL}] theEmitter->emitIns_R_R_R_I(INS_sve_ldnf1sh, EA_SCALABLE, REG_V0, REG_P1, REG_R5, 5, INS_OPTS_SCALABLE_D); // LDNF1SH {.D }, /Z, [{, #, MUL VL}] theEmitter->emitIns_R_R_R_I(INS_sve_ldnf1w, EA_SCALABLE, REG_V0, REG_P2, REG_R4, 5, diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 950f2f2dce8292..571dfa3dedd503 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -9484,7 +9484,7 @@ void emitter::emitIns_R_R_R_I(instruction ins, case INS_sve_ldnf1sw: case INS_sve_ldnf1d: - assert(insOptsScalable(opt)); + assert(opt == INS_OPTS_SCALABLE_D); assert(isVectorRegister(reg1)); assert(isPredicateRegister(reg2)); assert(isGeneralRegister(reg3)); @@ -9494,7 +9494,7 @@ void emitter::emitIns_R_R_R_I(instruction ins, case INS_sve_ldnf1sh: case INS_sve_ldnf1w: - assert(insOptsScalable(opt)); + assert(insOptsScalableWords(opt)); assert(isVectorRegister(reg1)); assert(isPredicateRegister(reg2)); assert(isGeneralRegister(reg3)); @@ -12765,18 +12765,20 @@ void emitter::emitIns_Call(EmitCallType callType, /***************************************************************************** * - * Returns true if the specified format can encode the 'dtype' field. + * Returns true if the specified instruction can encode the 'dtype' field. */ -/*static*/ bool emitter::canEncodeSveElemsize_dtype(insFormat fmt) +/*static*/ bool emitter::canEncodeSveElemsize_dtype(instruction ins) { - switch (fmt) + switch (ins) { - case IF_SVE_IH_3A_F: - case IF_SVE_IJ_3A_D: - case IF_SVE_IJ_3A_E: - case IF_SVE_IJ_3A_F: - case IF_SVE_IJ_3A_G: + case INS_sve_ld1w: + case INS_sve_ld1sb: + case INS_sve_ld1b: + case INS_sve_ld1sh: + case INS_sve_ld1h: + case INS_sve_ldnf1sh: + case INS_sve_ldnf1w: return true; default: @@ -12787,55 +12789,57 @@ void emitter::emitIns_Call(EmitCallType callType, /***************************************************************************** * * Returns the encoding to select the 1/2/4/8/16 byte elemsize for an Arm64 Sve vector instruction - * based on the format for the 'dtype' field. + * for the 'dtype' field. */ -/*static*/ emitter::code_t emitter::insEncodeSveElemsize_dtype(insFormat fmt, emitAttr size, code_t code) +/*static*/ emitter::code_t emitter::insEncodeSveElemsize_dtype(instruction ins, emitAttr size, code_t code) { - assert(canEncodeSveElemsize_dtype(fmt)); + assert(canEncodeSveElemsize_dtype(ins)); switch (size) { case EA_1BYTE: - switch (fmt) + switch (ins) { - case IF_SVE_IJ_3A_E: + case INS_sve_ld1b: return code & ~((1 << 22) | (1 << 21)); // Set bit '22' and '21' to 0. default: - assert(!"Invalid format for encoding dtype."); + assert(!"Invalid instruction for encoding dtype."); } return code; case EA_2BYTE: - switch (fmt) + switch (ins) { - case IF_SVE_IJ_3A_E: - case IF_SVE_IJ_3A_G: + case INS_sve_ld1b: + case INS_sve_ld1h: return code & ~(1 << 22); // Set bit '22' to 0. - case IF_SVE_IJ_3A_D: + case INS_sve_ld1sb: return code | (1 << 22); // Set bit '22' to 1. default: - assert(!"Invalid format for encoding dtype."); + assert(!"Invalid instruction for encoding dtype."); } return code; case EA_4BYTE: - switch (fmt) + switch (ins) { - case IF_SVE_IH_3A_F: - case IF_SVE_IJ_3A_E: - case IF_SVE_IJ_3A_G: + case INS_sve_ld1w: + case INS_sve_ld1b: + case INS_sve_ld1h: + case INS_sve_ldnf1w: return code & ~(1 << 21); // Set bit '21' to 0. - case IF_SVE_IJ_3A_D: - case IF_SVE_IJ_3A_F: + case INS_sve_ld1sb: + case INS_sve_ld1sh: + case INS_sve_ldnf1sh: return code | (1 << 21); // Set bit '21' to 1. default: - assert(!"Invalid format for encoding dtype."); + assert(!"Invalid instruction for encoding dtype."); } return code; @@ -12843,16 +12847,16 @@ void emitter::emitIns_Call(EmitCallType callType, return code; // By default, the instruction already encodes 64-bit. case EA_16BYTE: - switch (fmt) + switch (ins) { - case IF_SVE_IH_3A_F: + case INS_sve_ld1w: // Note: Bit '15' is not actually part of 'dtype', but it is necessary to set to '0' to get the // proper encoding for Q. return (code & ~((1 << 22) | (1 << 21) | (1 << 15))) | (1 << 20); // Set bits '22', '21' and '15' to 0. Set bit '20' to 1. default: - assert(!"Invalid format for encoding dtype."); + assert(!"Invalid instruction for encoding dtype."); } return code; @@ -14960,9 +14964,9 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg code |= insEncodeSimm4_19_to_16(imm); // iiii - if (canEncodeSveElemsize_dtype(fmt)) + if (canEncodeSveElemsize_dtype(ins)) { - code = insEncodeSveElemsize_dtype(fmt, optGetSveElemsize(id->idInsOpt()), code); + code = insEncodeSveElemsize_dtype(ins, optGetSveElemsize(id->idInsOpt()), code); } dst += emitOutput_Instr(dst, code); diff --git a/src/coreclr/jit/emitarm64.h b/src/coreclr/jit/emitarm64.h index 1ef35458100911..a4bfab49ac35de 100644 --- a/src/coreclr/jit/emitarm64.h +++ b/src/coreclr/jit/emitarm64.h @@ -472,12 +472,12 @@ static code_t insEncodeSveElemsize(insOpts opt); // This specifically encodes the field 'tszh:tszl' at bit locations '22:20-19'. static code_t insEncodeSveElemsize_tszh_22_tszl_20_to_19(emitAttr size); -// Returns true if the specified format can encode the 'dtype' field. -static bool canEncodeSveElemsize_dtype(insFormat fmt); +// Returns true if the specified instruction can encode the 'dtype' field. +static bool canEncodeSveElemsize_dtype(instruction ins); // Returns the encoding to select the 1/2/4/8/16 byte elemsize for an Arm64 Sve vector instruction -// based on the format for the 'dtype' field. -static code_t insEncodeSveElemsize_dtype(insFormat fmt, emitAttr size, code_t code); +// for the 'dtype' field. +static code_t insEncodeSveElemsize_dtype(instruction ins, emitAttr size, code_t code); static code_t insEncodeSimm4_19_to_16(ssize_t imm); diff --git a/src/coreclr/jit/instrsarm64sve.h b/src/coreclr/jit/instrsarm64sve.h index b2640aa65e967b..6108ac2385bc74 100644 --- a/src/coreclr/jit/instrsarm64sve.h +++ b/src/coreclr/jit/instrsarm64sve.h @@ -2839,8 +2839,8 @@ INST1(ldnf1sb, "ldnf1sb", 0, IF_SV INST1(ldnf1sh, "ldnf1sh", 0, IF_SVE_IL_3A_A, 0xA510A000 ) // LDNF1SH {.D }, /Z, [{, #, MUL VL}] SVE_IL_3A_A 101001010001iiii 101gggnnnnnttttt A510 A000 -INST1(ldnf1w, "ldnf1w", 0, IF_SVE_IL_3A_A, 0xA550A000 ) - // LDNF1W {.D }, /Z, [{, #, MUL VL}] SVE_IL_3A_A 101001010101iiii 101gggnnnnnttttt A550 A000 +INST1(ldnf1w, "ldnf1w", 0, IF_SVE_IL_3A_A, 0xA570A000 ) + // LDNF1W {.D }, /Z, [{, #, MUL VL}] SVE_IL_3A_A 101001010111iiii 101gggnnnnnttttt A570 A000 // enum name info SVE_IW_4A From 1a4a5456aff47365ba788693178e2ba284168293 Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 13 Dec 2023 13:34:27 -0800 Subject: [PATCH 15/37] More encodings --- src/coreclr/jit/codegenarm64.cpp | 14 ++++++++++++++ src/coreclr/jit/emitarm64.cpp | 14 ++++++++++++-- src/coreclr/jit/instrsarm64sve.h | 8 ++++---- 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index e776502734946a..902a230529b968 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -10727,12 +10727,26 @@ void CodeGen::genArm64EmitterUnitTests() INS_OPTS_SCALABLE_D); // LDNF1W {.D }, /Z, [{, #, MUL VL}] // IF_SVE_IL_3A_B + theEmitter->emitIns_R_R_R_I(INS_sve_ldnf1h, EA_SCALABLE, REG_V1, REG_P3, REG_R2, 5, + INS_OPTS_SCALABLE_H); // LDNF1H {.H }, /Z, [{, #, MUL VL}] + theEmitter->emitIns_R_R_R_I(INS_sve_ldnf1sb, EA_SCALABLE, REG_V0, REG_P4, REG_R1, 5, + INS_OPTS_SCALABLE_H); // LDNF1SB {.H }, /Z, [{, #, MUL VL}] + theEmitter->emitIns_R_R_R_I(INS_sve_ldnf1h, EA_SCALABLE, REG_V1, REG_P3, REG_R2, 5, + INS_OPTS_SCALABLE_S); // LDNF1H {.S }, /Z, [{, #, MUL VL}] + theEmitter->emitIns_R_R_R_I(INS_sve_ldnf1sb, EA_SCALABLE, REG_V0, REG_P4, REG_R1, 5, + INS_OPTS_SCALABLE_S); // LDNF1SB {.S }, /Z, [{, #, MUL VL}] theEmitter->emitIns_R_R_R_I(INS_sve_ldnf1h, EA_SCALABLE, REG_V1, REG_P3, REG_R2, 5, INS_OPTS_SCALABLE_D); // LDNF1H {.D }, /Z, [{, #, MUL VL}] theEmitter->emitIns_R_R_R_I(INS_sve_ldnf1sb, EA_SCALABLE, REG_V0, REG_P4, REG_R1, 5, INS_OPTS_SCALABLE_D); // LDNF1SB {.D }, /Z, [{, #, MUL VL}] // IF_SVE_IL_3A_C + theEmitter->emitIns_R_R_R_I(INS_sve_ldnf1b, EA_SCALABLE, REG_V2, REG_P5, REG_R3, 5, + INS_OPTS_SCALABLE_B); // LDNF1B {.B }, /Z, [{, #, MUL VL}] + theEmitter->emitIns_R_R_R_I(INS_sve_ldnf1b, EA_SCALABLE, REG_V2, REG_P5, REG_R3, 5, + INS_OPTS_SCALABLE_H); // LDNF1B {.H }, /Z, [{, #, MUL VL}] + theEmitter->emitIns_R_R_R_I(INS_sve_ldnf1b, EA_SCALABLE, REG_V2, REG_P5, REG_R3, 5, + INS_OPTS_SCALABLE_S); // LDNF1B {.S }, /Z, [{, #, MUL VL}] theEmitter->emitIns_R_R_R_I(INS_sve_ldnf1b, EA_SCALABLE, REG_V2, REG_P5, REG_R3, 5, INS_OPTS_SCALABLE_D); // LDNF1B {.D }, /Z, [{, #, MUL VL}] diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 571dfa3dedd503..1b463893b45521 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -9504,7 +9504,7 @@ void emitter::emitIns_R_R_R_I(instruction ins, case INS_sve_ldnf1h: case INS_sve_ldnf1sb: - assert(insOptsScalable(opt)); + assert(insOptsScalableAtLeastHalf(opt)); assert(isVectorRegister(reg1)); assert(isPredicateRegister(reg2)); assert(isGeneralRegister(reg3)); @@ -9513,7 +9513,7 @@ void emitter::emitIns_R_R_R_I(instruction ins, break; case INS_sve_ldnf1b: - assert(insOptsScalable(opt)); + assert(insOptsScalableSimple(opt)); assert(isVectorRegister(reg1)); assert(isPredicateRegister(reg2)); assert(isGeneralRegister(reg3)); @@ -12779,6 +12779,9 @@ void emitter::emitIns_Call(EmitCallType callType, case INS_sve_ld1h: case INS_sve_ldnf1sh: case INS_sve_ldnf1w: + case INS_sve_ldnf1h: + case INS_sve_ldnf1sb: + case INS_sve_ldnf1b: return true; default: @@ -12802,6 +12805,7 @@ void emitter::emitIns_Call(EmitCallType callType, switch (ins) { case INS_sve_ld1b: + case INS_sve_ldnf1b: return code & ~((1 << 22) | (1 << 21)); // Set bit '22' and '21' to 0. default: @@ -12814,9 +12818,12 @@ void emitter::emitIns_Call(EmitCallType callType, { case INS_sve_ld1b: case INS_sve_ld1h: + case INS_sve_ldnf1h: + case INS_sve_ldnf1b: return code & ~(1 << 22); // Set bit '22' to 0. case INS_sve_ld1sb: + case INS_sve_ldnf1sb: return code | (1 << 22); // Set bit '22' to 1. default: @@ -12831,11 +12838,14 @@ void emitter::emitIns_Call(EmitCallType callType, case INS_sve_ld1b: case INS_sve_ld1h: case INS_sve_ldnf1w: + case INS_sve_ldnf1h: + case INS_sve_ldnf1b: return code & ~(1 << 21); // Set bit '21' to 0. case INS_sve_ld1sb: case INS_sve_ld1sh: case INS_sve_ldnf1sh: + case INS_sve_ldnf1sb: return code | (1 << 21); // Set bit '21' to 1. default: diff --git a/src/coreclr/jit/instrsarm64sve.h b/src/coreclr/jit/instrsarm64sve.h index 6108ac2385bc74..d270b5f54709e6 100644 --- a/src/coreclr/jit/instrsarm64sve.h +++ b/src/coreclr/jit/instrsarm64sve.h @@ -2815,8 +2815,8 @@ INST1(ld1rw, "ld1rw", 0, IF_SV // enum name info SVE_IL_3A_C -INST1(ldnf1b, "ldnf1b", 0, IF_SVE_IL_3A_C, 0xA410A000 ) - // LDNF1B {.D }, /Z, [