51#define DEBUG_TYPE "x86-instr-info"
53#define GET_INSTRINFO_CTOR_DTOR
54#include "X86GenInstrInfo.inc"
60 cl::desc(
"Disable fusing of spill code into instructions"),
64 cl::desc(
"Print instructions that the allocator wants to"
65 " fuse, but the X86 backend currently can't"),
69 cl::desc(
"Re-materialize load from stub in PIC mode"),
73 cl::desc(
"Clearance between two register writes "
74 "for inserting XOR to avoid partial "
78 "undef-reg-clearance",
79 cl::desc(
"How many idle instructions we would like before "
80 "certain undef register reads"),
84void X86InstrInfo::anchor() {}
88 (STI.isTarget64BitLP64() ?
X86::ADJCALLSTACKDOWN64
89 :
X86::ADJCALLSTACKDOWN32),
90 (STI.isTarget64BitLP64() ?
X86::ADJCALLSTACKUP64
91 :
X86::ADJCALLSTACKUP32),
93 Subtarget(STI), RI(STI.getTargetTriple()) {}
101 if (!RC || !Subtarget.hasEGPR())
113 unsigned &SubIdx)
const {
114 switch (
MI.getOpcode()) {
117 case X86::MOVSX16rr8:
118 case X86::MOVZX16rr8:
119 case X86::MOVSX32rr8:
120 case X86::MOVZX32rr8:
121 case X86::MOVSX64rr8:
122 if (!Subtarget.is64Bit())
127 case X86::MOVSX32rr16:
128 case X86::MOVZX32rr16:
129 case X86::MOVSX64rr16:
130 case X86::MOVSX64rr32: {
131 if (
MI.getOperand(0).getSubReg() ||
MI.getOperand(1).getSubReg())
134 SrcReg =
MI.getOperand(1).getReg();
135 DstReg =
MI.getOperand(0).getReg();
136 switch (
MI.getOpcode()) {
139 case X86::MOVSX16rr8:
140 case X86::MOVZX16rr8:
141 case X86::MOVSX32rr8:
142 case X86::MOVZX32rr8:
143 case X86::MOVSX64rr8:
144 SubIdx = X86::sub_8bit;
146 case X86::MOVSX32rr16:
147 case X86::MOVZX32rr16:
148 case X86::MOVSX64rr16:
149 SubIdx = X86::sub_16bit;
151 case X86::MOVSX64rr32:
152 SubIdx = X86::sub_32bit;
162 if (
MI.mayLoad() ||
MI.mayStore())
167 if (
MI.isCopyLike() ||
MI.isInsertSubreg())
170 unsigned Opcode =
MI.getOpcode();
181 if (isBSF(Opcode) || isBSR(Opcode) || isLZCNT(Opcode) || isPOPCNT(Opcode) ||
187 if (isBLCFILL(Opcode) || isBLCI(Opcode) || isBLCIC(Opcode) ||
188 isBLCMSK(Opcode) || isBLCS(Opcode) || isBLSFILL(Opcode) ||
189 isBLSI(Opcode) || isBLSIC(Opcode) || isBLSMSK(Opcode) || isBLSR(Opcode) ||
194 if (isBEXTR(Opcode) || isBZHI(Opcode))
197 if (isROL(Opcode) || isROR(Opcode) || isSAR(Opcode) || isSHL(Opcode) ||
198 isSHR(Opcode) || isSHLD(Opcode) || isSHRD(Opcode))
201 if (isADC(Opcode) || isADD(Opcode) || isAND(Opcode) || isOR(Opcode) ||
202 isSBB(Opcode) || isSUB(Opcode) || isXOR(Opcode))
208 if (isDEC(Opcode) || isINC(Opcode) || isNEG(Opcode))
216 if (isMOVSX(Opcode) || isMOVZX(Opcode) || isMOVSXD(Opcode) || isMOV(Opcode))
219 if (isRORX(Opcode) || isSARX(Opcode) || isSHLX(Opcode) || isSHRX(Opcode))
229 switch (
MI.getOpcode()) {
242 case X86::IMUL64rmi32:
257 case X86::POPCNT16rm:
258 case X86::POPCNT32rm:
259 case X86::POPCNT64rm:
267 case X86::BLCFILL32rm:
268 case X86::BLCFILL64rm:
273 case X86::BLCMSK32rm:
274 case X86::BLCMSK64rm:
277 case X86::BLSFILL32rm:
278 case X86::BLSFILL64rm:
283 case X86::BLSMSK32rm:
284 case X86::BLSMSK64rm:
294 case X86::BEXTRI32mi:
295 case X86::BEXTRI64mi:
348 case X86::CVTTSD2SI64rm:
349 case X86::VCVTTSD2SI64rm:
350 case X86::VCVTTSD2SI64Zrm:
351 case X86::CVTTSD2SIrm:
352 case X86::VCVTTSD2SIrm:
353 case X86::VCVTTSD2SIZrm:
354 case X86::CVTTSS2SI64rm:
355 case X86::VCVTTSS2SI64rm:
356 case X86::VCVTTSS2SI64Zrm:
357 case X86::CVTTSS2SIrm:
358 case X86::VCVTTSS2SIrm:
359 case X86::VCVTTSS2SIZrm:
360 case X86::CVTSI2SDrm:
361 case X86::VCVTSI2SDrm:
362 case X86::VCVTSI2SDZrm:
363 case X86::CVTSI2SSrm:
364 case X86::VCVTSI2SSrm:
365 case X86::VCVTSI2SSZrm:
366 case X86::CVTSI642SDrm:
367 case X86::VCVTSI642SDrm:
368 case X86::VCVTSI642SDZrm:
369 case X86::CVTSI642SSrm:
370 case X86::VCVTSI642SSrm:
371 case X86::VCVTSI642SSZrm:
372 case X86::CVTSS2SDrm:
373 case X86::VCVTSS2SDrm:
374 case X86::VCVTSS2SDZrm:
375 case X86::CVTSD2SSrm:
376 case X86::VCVTSD2SSrm:
377 case X86::VCVTSD2SSZrm:
379 case X86::VCVTTSD2USI64Zrm:
380 case X86::VCVTTSD2USIZrm:
381 case X86::VCVTTSS2USI64Zrm:
382 case X86::VCVTTSS2USIZrm:
383 case X86::VCVTUSI2SDZrm:
384 case X86::VCVTUSI642SDZrm:
385 case X86::VCVTUSI2SSZrm:
386 case X86::VCVTUSI642SSZrm:
390 case X86::MOV8rm_NOREX:
394 case X86::MOVSX16rm8:
395 case X86::MOVSX32rm16:
396 case X86::MOVSX32rm8:
397 case X86::MOVSX32rm8_NOREX:
398 case X86::MOVSX64rm16:
399 case X86::MOVSX64rm32:
400 case X86::MOVSX64rm8:
401 case X86::MOVZX16rm8:
402 case X86::MOVZX32rm16:
403 case X86::MOVZX32rm8:
404 case X86::MOVZX32rm8_NOREX:
405 case X86::MOVZX64rm16:
406 case X86::MOVZX64rm8:
415 if (isFrameInstr(
MI)) {
418 if (!isFrameSetup(
MI))
429 for (
auto E =
MBB->end();
I != E; ++
I) {
430 if (
I->getOpcode() == getCallFrameDestroyOpcode() ||
I->isCall())
436 if (
I->getOpcode() != getCallFrameDestroyOpcode())
439 return -(
I->getOperand(1).
getImm());
444 switch (
MI.getOpcode()) {
463 int &FrameIndex)
const {
483 case X86::KMOVBkm_EVEX:
488 case X86::KMOVWkm_EVEX:
490 case X86::VMOVSHZrm_alt:
495 case X86::MOVSSrm_alt:
497 case X86::VMOVSSrm_alt:
499 case X86::VMOVSSZrm_alt:
501 case X86::KMOVDkm_EVEX:
507 case X86::MOVSDrm_alt:
509 case X86::VMOVSDrm_alt:
511 case X86::VMOVSDZrm_alt:
512 case X86::MMX_MOVD64rm:
513 case X86::MMX_MOVQ64rm:
515 case X86::KMOVQkm_EVEX:
530 case X86::VMOVAPSZ128rm:
531 case X86::VMOVUPSZ128rm:
532 case X86::VMOVAPSZ128rm_NOVLX:
533 case X86::VMOVUPSZ128rm_NOVLX:
534 case X86::VMOVAPDZ128rm:
535 case X86::VMOVUPDZ128rm:
536 case X86::VMOVDQU8Z128rm:
537 case X86::VMOVDQU16Z128rm:
538 case X86::VMOVDQA32Z128rm:
539 case X86::VMOVDQU32Z128rm:
540 case X86::VMOVDQA64Z128rm:
541 case X86::VMOVDQU64Z128rm:
544 case X86::VMOVAPSYrm:
545 case X86::VMOVUPSYrm:
546 case X86::VMOVAPDYrm:
547 case X86::VMOVUPDYrm:
548 case X86::VMOVDQAYrm:
549 case X86::VMOVDQUYrm:
550 case X86::VMOVAPSZ256rm:
551 case X86::VMOVUPSZ256rm:
552 case X86::VMOVAPSZ256rm_NOVLX:
553 case X86::VMOVUPSZ256rm_NOVLX:
554 case X86::VMOVAPDZ256rm:
555 case X86::VMOVUPDZ256rm:
556 case X86::VMOVDQU8Z256rm:
557 case X86::VMOVDQU16Z256rm:
558 case X86::VMOVDQA32Z256rm:
559 case X86::VMOVDQU32Z256rm:
560 case X86::VMOVDQA64Z256rm:
561 case X86::VMOVDQU64Z256rm:
564 case X86::VMOVAPSZrm:
565 case X86::VMOVUPSZrm:
566 case X86::VMOVAPDZrm:
567 case X86::VMOVUPDZrm:
568 case X86::VMOVDQU8Zrm:
569 case X86::VMOVDQU16Zrm:
570 case X86::VMOVDQA32Zrm:
571 case X86::VMOVDQU32Zrm:
572 case X86::VMOVDQA64Zrm:
573 case X86::VMOVDQU64Zrm:
585 case X86::KMOVBmk_EVEX:
590 case X86::KMOVWmk_EVEX:
599 case X86::KMOVDmk_EVEX:
607 case X86::MMX_MOVD64mr:
608 case X86::MMX_MOVQ64mr:
609 case X86::MMX_MOVNTQmr:
611 case X86::KMOVQmk_EVEX:
626 case X86::VMOVUPSZ128mr:
627 case X86::VMOVAPSZ128mr:
628 case X86::VMOVUPSZ128mr_NOVLX:
629 case X86::VMOVAPSZ128mr_NOVLX:
630 case X86::VMOVUPDZ128mr:
631 case X86::VMOVAPDZ128mr:
632 case X86::VMOVDQA32Z128mr:
633 case X86::VMOVDQU32Z128mr:
634 case X86::VMOVDQA64Z128mr:
635 case X86::VMOVDQU64Z128mr:
636 case X86::VMOVDQU8Z128mr:
637 case X86::VMOVDQU16Z128mr:
640 case X86::VMOVUPSYmr:
641 case X86::VMOVAPSYmr:
642 case X86::VMOVUPDYmr:
643 case X86::VMOVAPDYmr:
644 case X86::VMOVDQUYmr:
645 case X86::VMOVDQAYmr:
646 case X86::VMOVUPSZ256mr:
647 case X86::VMOVAPSZ256mr:
648 case X86::VMOVUPSZ256mr_NOVLX:
649 case X86::VMOVAPSZ256mr_NOVLX:
650 case X86::VMOVUPDZ256mr:
651 case X86::VMOVAPDZ256mr:
652 case X86::VMOVDQU8Z256mr:
653 case X86::VMOVDQU16Z256mr:
654 case X86::VMOVDQA32Z256mr:
655 case X86::VMOVDQU32Z256mr:
656 case X86::VMOVDQA64Z256mr:
657 case X86::VMOVDQU64Z256mr:
660 case X86::VMOVUPSZmr:
661 case X86::VMOVAPSZmr:
662 case X86::VMOVUPDZmr:
663 case X86::VMOVAPDZmr:
664 case X86::VMOVDQU8Zmr:
665 case X86::VMOVDQU16Zmr:
666 case X86::VMOVDQA32Zmr:
667 case X86::VMOVDQU32Zmr:
668 case X86::VMOVDQA64Zmr:
669 case X86::VMOVDQU64Zmr:
677 int &FrameIndex)
const {
686 if (
MI.getOperand(0).getSubReg() == 0 && isFrameOperand(
MI, 1, FrameIndex))
687 return MI.getOperand(0).getReg();
692 int &FrameIndex)
const {
703 return MI.getOperand(0).getReg();
710 int &FrameIndex)
const {
720 isFrameOperand(
MI, 0, FrameIndex))
726 int &FrameIndex)
const {
746 if (!BaseReg.isVirtual())
748 bool isPICBase =
false;
750 if (
DefMI.getOpcode() != X86::MOVPC32r)
752 assert(!isPICBase &&
"More than one PIC base?");
760 switch (
MI.getOpcode()) {
766 case X86::IMPLICIT_DEF:
769 case X86::LOAD_STACK_GUARD:
776 case X86::AVX1_SETALLONES:
777 case X86::AVX2_SETALLONES:
778 case X86::AVX512_128_SET0:
779 case X86::AVX512_256_SET0:
780 case X86::AVX512_512_SET0:
781 case X86::AVX512_512_SETALLONES:
782 case X86::AVX512_FsFLD0SD:
783 case X86::AVX512_FsFLD0SH:
784 case X86::AVX512_FsFLD0SS:
785 case X86::AVX512_FsFLD0F128:
790 case X86::FsFLD0F128:
798 case X86::MOV32ImmSExti8:
803 case X86::MOV64ImmSExti8:
805 case X86::V_SETALLONES:
811 case X86::PTILEZEROV:
815 case X86::MOV8rm_NOREX:
820 case X86::MOVSSrm_alt:
822 case X86::MOVSDrm_alt:
830 case X86::VMOVSSrm_alt:
832 case X86::VMOVSDrm_alt:
839 case X86::VMOVAPSYrm:
840 case X86::VMOVUPSYrm:
841 case X86::VMOVAPDYrm:
842 case X86::VMOVUPDYrm:
843 case X86::VMOVDQAYrm:
844 case X86::VMOVDQUYrm:
845 case X86::MMX_MOVD64rm:
846 case X86::MMX_MOVQ64rm:
847 case X86::VBROADCASTSSrm:
848 case X86::VBROADCASTSSYrm:
849 case X86::VBROADCASTSDYrm:
851 case X86::VPBROADCASTBZ128rm:
852 case X86::VPBROADCASTBZ256rm:
853 case X86::VPBROADCASTBZrm:
854 case X86::VBROADCASTF32X2Z256rm:
855 case X86::VBROADCASTF32X2Zrm:
856 case X86::VBROADCASTI32X2Z128rm:
857 case X86::VBROADCASTI32X2Z256rm:
858 case X86::VBROADCASTI32X2Zrm:
859 case X86::VPBROADCASTWZ128rm:
860 case X86::VPBROADCASTWZ256rm:
861 case X86::VPBROADCASTWZrm:
862 case X86::VPBROADCASTDZ128rm:
863 case X86::VPBROADCASTDZ256rm:
864 case X86::VPBROADCASTDZrm:
865 case X86::VBROADCASTSSZ128rm:
866 case X86::VBROADCASTSSZ256rm:
867 case X86::VBROADCASTSSZrm:
868 case X86::VPBROADCASTQZ128rm:
869 case X86::VPBROADCASTQZ256rm:
870 case X86::VPBROADCASTQZrm:
871 case X86::VBROADCASTSDZ256rm:
872 case X86::VBROADCASTSDZrm:
874 case X86::VMOVSSZrm_alt:
876 case X86::VMOVSDZrm_alt:
878 case X86::VMOVSHZrm_alt:
879 case X86::VMOVAPDZ128rm:
880 case X86::VMOVAPDZ256rm:
881 case X86::VMOVAPDZrm:
882 case X86::VMOVAPSZ128rm:
883 case X86::VMOVAPSZ256rm:
884 case X86::VMOVAPSZ128rm_NOVLX:
885 case X86::VMOVAPSZ256rm_NOVLX:
886 case X86::VMOVAPSZrm:
887 case X86::VMOVDQA32Z128rm:
888 case X86::VMOVDQA32Z256rm:
889 case X86::VMOVDQA32Zrm:
890 case X86::VMOVDQA64Z128rm:
891 case X86::VMOVDQA64Z256rm:
892 case X86::VMOVDQA64Zrm:
893 case X86::VMOVDQU16Z128rm:
894 case X86::VMOVDQU16Z256rm:
895 case X86::VMOVDQU16Zrm:
896 case X86::VMOVDQU32Z128rm:
897 case X86::VMOVDQU32Z256rm:
898 case X86::VMOVDQU32Zrm:
899 case X86::VMOVDQU64Z128rm:
900 case X86::VMOVDQU64Z256rm:
901 case X86::VMOVDQU64Zrm:
902 case X86::VMOVDQU8Z128rm:
903 case X86::VMOVDQU8Z256rm:
904 case X86::VMOVDQU8Zrm:
905 case X86::VMOVUPDZ128rm:
906 case X86::VMOVUPDZ256rm:
907 case X86::VMOVUPDZrm:
908 case X86::VMOVUPSZ128rm:
909 case X86::VMOVUPSZ256rm:
910 case X86::VMOVUPSZ128rm_NOVLX:
911 case X86::VMOVUPSZ256rm_NOVLX:
912 case X86::VMOVUPSZrm: {
918 MI.isDereferenceableInvariantLoad()) {
920 if (BaseReg == 0 || BaseReg == X86::RIP)
963 if (ClobbersEFLAGS &&
MBB.computeRegisterLiveness(&
TRI, X86::EFLAGS,
I) !=
998 if (MO.isReg() && MO.isDef() && MO.getReg() == X86::EFLAGS &&
1008 unsigned ShiftAmtOperandIdx) {
1010 unsigned ShiftCountMask = (
MI.getDesc().TSFlags &
X86II::REX_W) ? 63 : 31;
1011 unsigned Imm =
MI.getOperand(ShiftAmtOperandIdx).getImm();
1012 return Imm & ShiftCountMask;
1023 return ShAmt < 4 && ShAmt > 0;
1030 bool &NoSignFlag,
bool &ClearsOverflowFlag) {
1031 if (!(CmpValDefInstr.
getOpcode() == X86::SUBREG_TO_REG &&
1032 CmpInstr.
getOpcode() == X86::TEST64rr) &&
1033 !(CmpValDefInstr.
getOpcode() == X86::COPY &&
1041 "CmpInstr is an analyzable TEST16rr/TEST64rr, and "
1042 "`X86InstrInfo::analyzeCompare` requires two reg operands are the"
1051 "Caller guarantees that TEST64rr is a user of SUBREG_TO_REG or TEST16rr "
1052 "is a user of COPY sub16bit.");
1054 if (CmpInstr.
getOpcode() == X86::TEST16rr) {
1063 if (!((VregDefInstr->
getOpcode() == X86::AND32ri ||
1064 VregDefInstr->
getOpcode() == X86::AND64ri32) &&
1069 if (CmpInstr.
getOpcode() == X86::TEST64rr) {
1083 assert(VregDefInstr &&
"Must have a definition (SSA)");
1093 if (X86::isAND(VregDefInstr->
getOpcode()) &&
1114 if (Instr.modifiesRegister(X86::EFLAGS,
TRI))
1118 *AndInstr = VregDefInstr;
1139 ClearsOverflowFlag =
true;
1147 unsigned &NewSrcSubReg,
bool &isKill,
1153 RC =
Opc != X86::LEA32r ? &X86::GR64RegClass : &X86::GR32RegClass;
1155 RC =
Opc != X86::LEA32r ? &X86::GR64_NOSPRegClass : &X86::GR32_NOSPRegClass;
1158 unsigned SubReg = Src.getSubReg();
1159 isKill =
MI.killsRegister(SrcReg,
nullptr);
1161 NewSrcSubReg = X86::NoSubRegister;
1165 if (
Opc != X86::LEA64_32r) {
1168 assert(!Src.isUndef() &&
"Undef op doesn't need optimization");
1185 assert(!Src.isUndef() &&
"Undef op doesn't need optimization");
1190 NewSrcSubReg = X86::NoSubRegister;
1216MachineInstr *X86InstrInfo::convertToThreeAddressWithLEA(
unsigned MIOpc,
1220 bool Is8BitOp)
const {
1225 RegInfo.getTargetRegisterInfo()->getRegSizeInBits(
1226 *RegInfo.getRegClass(
MI.getOperand(0).getReg())) == 16) &&
1227 "Unexpected type for LEA transform");
1236 if (!Subtarget.is64Bit())
1239 unsigned Opcode = X86::LEA64_32r;
1240 Register InRegLEA = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass);
1241 Register OutRegLEA = RegInfo.createVirtualRegister(&X86::GR32RegClass);
1254 unsigned SrcSubReg =
MI.getOperand(1).getSubReg();
1256 unsigned Src2SubReg;
1257 bool IsDead =
MI.getOperand(0).isDead();
1258 bool IsKill =
MI.getOperand(1).isKill();
1259 unsigned SubReg = Is8BitOp ? X86::sub_8bit : X86::sub_16bit;
1260 assert(!
MI.getOperand(1).isUndef() &&
"Undef op doesn't need optimization");
1272#define CASE_NF(OP) \
1280 unsigned ShAmt =
MI.getOperand(2).getImm();
1298 case X86::ADD8ri_DB:
1299 case X86::ADD16ri_DB:
1304 case X86::ADD8rr_DB:
1305 case X86::ADD16rr_DB: {
1306 Src2 =
MI.getOperand(2).getReg();
1307 Src2SubReg =
MI.getOperand(2).getSubReg();
1308 bool IsKill2 =
MI.getOperand(2).isKill();
1309 assert(!
MI.getOperand(2).isUndef() &&
"Undef op doesn't need optimization");
1313 addRegReg(MIB, InRegLEA,
true, X86::NoSubRegister, InRegLEA,
false,
1314 X86::NoSubRegister);
1316 if (Subtarget.is64Bit())
1322 ImpDef2 =
BuildMI(
MBB, &*MIB,
MI.getDebugLoc(),
get(X86::IMPLICIT_DEF),
1324 InsMI2 =
BuildMI(
MBB, &*MIB,
MI.getDebugLoc(),
get(TargetOpcode::COPY))
1327 addRegReg(MIB, InRegLEA,
true, X86::NoSubRegister, InRegLEA2,
true,
1328 X86::NoSubRegister);
1330 if (LV && IsKill2 && InsMI2)
1336 MachineInstr *NewMI = MIB;
1337 MachineInstr *ExtMI =
1385 LiveRange::Segment *DestSeg =
1426 if (
MI.getNumOperands() > 2)
1427 if (
MI.getOperand(2).isReg() &&
MI.getOperand(2).isUndef())
1432 unsigned SrcSubReg, SrcSubReg2;
1433 bool Is64Bit = Subtarget.is64Bit();
1435 bool Is8BitOp =
false;
1436 unsigned NumRegOperands = 2;
1437 unsigned MIOpc =
MI.getOpcode();
1442 assert(
MI.getNumOperands() >= 3 &&
"Unknown shift instruction!");
1449 Src.getReg(), &X86::GR64_NOSPRegClass))
1452 NewMI =
BuildMI(MF,
MI.getDebugLoc(),
get(X86::LEA64r))
1462 assert(
MI.getNumOperands() >= 3 &&
"Unknown shift instruction!");
1467 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1473 isKill, ImplicitOp, LV, LIS))
1484 if (ImplicitOp.
getReg() != 0)
1485 MIB.
add(ImplicitOp);
1489 if (LV && SrcReg != Src.getReg())
1497 assert(
MI.getNumOperands() >= 3 &&
"Unknown shift instruction!");
1501 return convertToThreeAddressWithLEA(MIOpc,
MI, LV, LIS, Is8BitOp);
1505 assert(
MI.getNumOperands() >= 2 &&
"Unknown inc instruction!");
1506 unsigned Opc = (MIOpc == X86::INC64r || MIOpc == X86::INC64r_NF)
1508 : (Is64Bit ? X86::LEA64_32r : X86::LEA32r);
1512 isKill, ImplicitOp, LV, LIS))
1518 if (ImplicitOp.
getReg() != 0)
1519 MIB.
add(ImplicitOp);
1524 if (LV && SrcReg != Src.getReg())
1530 assert(
MI.getNumOperands() >= 2 &&
"Unknown dec instruction!");
1531 unsigned Opc = (MIOpc == X86::DEC64r || MIOpc == X86::DEC64r_NF)
1533 : (Is64Bit ? X86::LEA64_32r : X86::LEA32r);
1538 isKill, ImplicitOp, LV, LIS))
1544 if (ImplicitOp.
getReg() != 0)
1545 MIB.
add(ImplicitOp);
1550 if (LV && SrcReg != Src.getReg())
1560 return convertToThreeAddressWithLEA(MIOpc,
MI, LV, LIS, Is8BitOp);
1563 case X86::ADD64rr_DB:
1564 case X86::ADD32rr_DB: {
1565 assert(
MI.getNumOperands() >= 3 &&
"Unknown add instruction!");
1567 if (MIOpc == X86::ADD64rr || MIOpc == X86::ADD64rr_NF ||
1568 MIOpc == X86::ADD64rr_DB)
1571 Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1577 isKill2, ImplicitOp2, LV, LIS))
1582 if (Src.getReg() == Src2.
getReg()) {
1587 SrcSubReg = SrcSubReg2;
1590 isKill, ImplicitOp, LV, LIS))
1595 if (ImplicitOp.
getReg() != 0)
1596 MIB.
add(ImplicitOp);
1597 if (ImplicitOp2.
getReg() != 0)
1598 MIB.
add(ImplicitOp2);
1601 addRegReg(MIB, SrcReg, isKill, SrcSubReg, SrcReg2, isKill2, SrcSubReg2);
1605 if (SrcReg2 != Src2.
getReg())
1607 if (SrcReg != SrcReg2 && SrcReg != Src.getReg())
1614 case X86::ADD8rr_DB:
1618 case X86::ADD16rr_DB:
1619 return convertToThreeAddressWithLEA(MIOpc,
MI, LV, LIS, Is8BitOp);
1621 case X86::ADD64ri32_DB:
1622 assert(
MI.getNumOperands() >= 3 &&
"Unknown add instruction!");
1624 BuildMI(MF,
MI.getDebugLoc(),
get(X86::LEA64r)).add(Dest).add(Src),
1628 case X86::ADD32ri_DB: {
1629 assert(
MI.getNumOperands() >= 3 &&
"Unknown add instruction!");
1630 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1635 isKill, ImplicitOp, LV, LIS))
1642 if (ImplicitOp.
getReg() != 0)
1643 MIB.
add(ImplicitOp);
1648 if (LV && SrcReg != Src.getReg())
1653 case X86::ADD8ri_DB:
1657 case X86::ADD16ri_DB:
1658 return convertToThreeAddressWithLEA(MIOpc,
MI, LV, LIS, Is8BitOp);
1664 if (!
MI.getOperand(2).isImm())
1666 int64_t Imm =
MI.getOperand(2).getImm();
1670 assert(
MI.getNumOperands() >= 3 &&
"Unknown add instruction!");
1671 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1676 isKill, ImplicitOp, LV, LIS))
1683 if (ImplicitOp.
getReg() != 0)
1684 MIB.
add(ImplicitOp);
1689 if (LV && SrcReg != Src.getReg())
1695 if (!
MI.getOperand(2).isImm())
1697 int64_t Imm =
MI.getOperand(2).getImm();
1701 assert(
MI.getNumOperands() >= 3 &&
"Unknown sub instruction!");
1709 case X86::VMOVDQU8Z128rmk:
1710 case X86::VMOVDQU8Z256rmk:
1711 case X86::VMOVDQU8Zrmk:
1712 case X86::VMOVDQU16Z128rmk:
1713 case X86::VMOVDQU16Z256rmk:
1714 case X86::VMOVDQU16Zrmk:
1715 case X86::VMOVDQU32Z128rmk:
1716 case X86::VMOVDQA32Z128rmk:
1717 case X86::VMOVDQU32Z256rmk:
1718 case X86::VMOVDQA32Z256rmk:
1719 case X86::VMOVDQU32Zrmk:
1720 case X86::VMOVDQA32Zrmk:
1721 case X86::VMOVDQU64Z128rmk:
1722 case X86::VMOVDQA64Z128rmk:
1723 case X86::VMOVDQU64Z256rmk:
1724 case X86::VMOVDQA64Z256rmk:
1725 case X86::VMOVDQU64Zrmk:
1726 case X86::VMOVDQA64Zrmk:
1727 case X86::VMOVUPDZ128rmk:
1728 case X86::VMOVAPDZ128rmk:
1729 case X86::VMOVUPDZ256rmk:
1730 case X86::VMOVAPDZ256rmk:
1731 case X86::VMOVUPDZrmk:
1732 case X86::VMOVAPDZrmk:
1733 case X86::VMOVUPSZ128rmk:
1734 case X86::VMOVAPSZ128rmk:
1735 case X86::VMOVUPSZ256rmk:
1736 case X86::VMOVAPSZ256rmk:
1737 case X86::VMOVUPSZrmk:
1738 case X86::VMOVAPSZrmk:
1739 case X86::VBROADCASTSDZ256rmk:
1740 case X86::VBROADCASTSDZrmk:
1741 case X86::VBROADCASTSSZ128rmk:
1742 case X86::VBROADCASTSSZ256rmk:
1743 case X86::VBROADCASTSSZrmk:
1744 case X86::VPBROADCASTDZ128rmk:
1745 case X86::VPBROADCASTDZ256rmk:
1746 case X86::VPBROADCASTDZrmk:
1747 case X86::VPBROADCASTQZ128rmk:
1748 case X86::VPBROADCASTQZ256rmk:
1749 case X86::VPBROADCASTQZrmk: {
1754 case X86::VMOVDQU8Z128rmk:
1755 Opc = X86::VPBLENDMBZ128rmk;
1757 case X86::VMOVDQU8Z256rmk:
1758 Opc = X86::VPBLENDMBZ256rmk;
1760 case X86::VMOVDQU8Zrmk:
1761 Opc = X86::VPBLENDMBZrmk;
1763 case X86::VMOVDQU16Z128rmk:
1764 Opc = X86::VPBLENDMWZ128rmk;
1766 case X86::VMOVDQU16Z256rmk:
1767 Opc = X86::VPBLENDMWZ256rmk;
1769 case X86::VMOVDQU16Zrmk:
1770 Opc = X86::VPBLENDMWZrmk;
1772 case X86::VMOVDQU32Z128rmk:
1773 Opc = X86::VPBLENDMDZ128rmk;
1775 case X86::VMOVDQU32Z256rmk:
1776 Opc = X86::VPBLENDMDZ256rmk;
1778 case X86::VMOVDQU32Zrmk:
1779 Opc = X86::VPBLENDMDZrmk;
1781 case X86::VMOVDQU64Z128rmk:
1782 Opc = X86::VPBLENDMQZ128rmk;
1784 case X86::VMOVDQU64Z256rmk:
1785 Opc = X86::VPBLENDMQZ256rmk;
1787 case X86::VMOVDQU64Zrmk:
1788 Opc = X86::VPBLENDMQZrmk;
1790 case X86::VMOVUPDZ128rmk:
1791 Opc = X86::VBLENDMPDZ128rmk;
1793 case X86::VMOVUPDZ256rmk:
1794 Opc = X86::VBLENDMPDZ256rmk;
1796 case X86::VMOVUPDZrmk:
1797 Opc = X86::VBLENDMPDZrmk;
1799 case X86::VMOVUPSZ128rmk:
1800 Opc = X86::VBLENDMPSZ128rmk;
1802 case X86::VMOVUPSZ256rmk:
1803 Opc = X86::VBLENDMPSZ256rmk;
1805 case X86::VMOVUPSZrmk:
1806 Opc = X86::VBLENDMPSZrmk;
1808 case X86::VMOVDQA32Z128rmk:
1809 Opc = X86::VPBLENDMDZ128rmk;
1811 case X86::VMOVDQA32Z256rmk:
1812 Opc = X86::VPBLENDMDZ256rmk;
1814 case X86::VMOVDQA32Zrmk:
1815 Opc = X86::VPBLENDMDZrmk;
1817 case X86::VMOVDQA64Z128rmk:
1818 Opc = X86::VPBLENDMQZ128rmk;
1820 case X86::VMOVDQA64Z256rmk:
1821 Opc = X86::VPBLENDMQZ256rmk;
1823 case X86::VMOVDQA64Zrmk:
1824 Opc = X86::VPBLENDMQZrmk;
1826 case X86::VMOVAPDZ128rmk:
1827 Opc = X86::VBLENDMPDZ128rmk;
1829 case X86::VMOVAPDZ256rmk:
1830 Opc = X86::VBLENDMPDZ256rmk;
1832 case X86::VMOVAPDZrmk:
1833 Opc = X86::VBLENDMPDZrmk;
1835 case X86::VMOVAPSZ128rmk:
1836 Opc = X86::VBLENDMPSZ128rmk;
1838 case X86::VMOVAPSZ256rmk:
1839 Opc = X86::VBLENDMPSZ256rmk;
1841 case X86::VMOVAPSZrmk:
1842 Opc = X86::VBLENDMPSZrmk;
1844 case X86::VBROADCASTSDZ256rmk:
1845 Opc = X86::VBLENDMPDZ256rmbk;
1847 case X86::VBROADCASTSDZrmk:
1848 Opc = X86::VBLENDMPDZrmbk;
1850 case X86::VBROADCASTSSZ128rmk:
1851 Opc = X86::VBLENDMPSZ128rmbk;
1853 case X86::VBROADCASTSSZ256rmk:
1854 Opc = X86::VBLENDMPSZ256rmbk;
1856 case X86::VBROADCASTSSZrmk:
1857 Opc = X86::VBLENDMPSZrmbk;
1859 case X86::VPBROADCASTDZ128rmk:
1860 Opc = X86::VPBLENDMDZ128rmbk;
1862 case X86::VPBROADCASTDZ256rmk:
1863 Opc = X86::VPBLENDMDZ256rmbk;
1865 case X86::VPBROADCASTDZrmk:
1866 Opc = X86::VPBLENDMDZrmbk;
1868 case X86::VPBROADCASTQZ128rmk:
1869 Opc = X86::VPBLENDMQZ128rmbk;
1871 case X86::VPBROADCASTQZ256rmk:
1872 Opc = X86::VPBLENDMQZ256rmbk;
1874 case X86::VPBROADCASTQZrmk:
1875 Opc = X86::VPBLENDMQZrmbk;
1881 .
add(
MI.getOperand(2))
1883 .
add(
MI.getOperand(3))
1884 .
add(
MI.getOperand(4))
1885 .
add(
MI.getOperand(5))
1886 .
add(
MI.getOperand(6))
1887 .
add(
MI.getOperand(7));
1892 case X86::VMOVDQU8Z128rrk:
1893 case X86::VMOVDQU8Z256rrk:
1894 case X86::VMOVDQU8Zrrk:
1895 case X86::VMOVDQU16Z128rrk:
1896 case X86::VMOVDQU16Z256rrk:
1897 case X86::VMOVDQU16Zrrk:
1898 case X86::VMOVDQU32Z128rrk:
1899 case X86::VMOVDQA32Z128rrk:
1900 case X86::VMOVDQU32Z256rrk:
1901 case X86::VMOVDQA32Z256rrk:
1902 case X86::VMOVDQU32Zrrk:
1903 case X86::VMOVDQA32Zrrk:
1904 case X86::VMOVDQU64Z128rrk:
1905 case X86::VMOVDQA64Z128rrk:
1906 case X86::VMOVDQU64Z256rrk:
1907 case X86::VMOVDQA64Z256rrk:
1908 case X86::VMOVDQU64Zrrk:
1909 case X86::VMOVDQA64Zrrk:
1910 case X86::VMOVUPDZ128rrk:
1911 case X86::VMOVAPDZ128rrk:
1912 case X86::VMOVUPDZ256rrk:
1913 case X86::VMOVAPDZ256rrk:
1914 case X86::VMOVUPDZrrk:
1915 case X86::VMOVAPDZrrk:
1916 case X86::VMOVUPSZ128rrk:
1917 case X86::VMOVAPSZ128rrk:
1918 case X86::VMOVUPSZ256rrk:
1919 case X86::VMOVAPSZ256rrk:
1920 case X86::VMOVUPSZrrk:
1921 case X86::VMOVAPSZrrk: {
1926 case X86::VMOVDQU8Z128rrk:
1927 Opc = X86::VPBLENDMBZ128rrk;
1929 case X86::VMOVDQU8Z256rrk:
1930 Opc = X86::VPBLENDMBZ256rrk;
1932 case X86::VMOVDQU8Zrrk:
1933 Opc = X86::VPBLENDMBZrrk;
1935 case X86::VMOVDQU16Z128rrk:
1936 Opc = X86::VPBLENDMWZ128rrk;
1938 case X86::VMOVDQU16Z256rrk:
1939 Opc = X86::VPBLENDMWZ256rrk;
1941 case X86::VMOVDQU16Zrrk:
1942 Opc = X86::VPBLENDMWZrrk;
1944 case X86::VMOVDQU32Z128rrk:
1945 Opc = X86::VPBLENDMDZ128rrk;
1947 case X86::VMOVDQU32Z256rrk:
1948 Opc = X86::VPBLENDMDZ256rrk;
1950 case X86::VMOVDQU32Zrrk:
1951 Opc = X86::VPBLENDMDZrrk;
1953 case X86::VMOVDQU64Z128rrk:
1954 Opc = X86::VPBLENDMQZ128rrk;
1956 case X86::VMOVDQU64Z256rrk:
1957 Opc = X86::VPBLENDMQZ256rrk;
1959 case X86::VMOVDQU64Zrrk:
1960 Opc = X86::VPBLENDMQZrrk;
1962 case X86::VMOVUPDZ128rrk:
1963 Opc = X86::VBLENDMPDZ128rrk;
1965 case X86::VMOVUPDZ256rrk:
1966 Opc = X86::VBLENDMPDZ256rrk;
1968 case X86::VMOVUPDZrrk:
1969 Opc = X86::VBLENDMPDZrrk;
1971 case X86::VMOVUPSZ128rrk:
1972 Opc = X86::VBLENDMPSZ128rrk;
1974 case X86::VMOVUPSZ256rrk:
1975 Opc = X86::VBLENDMPSZ256rrk;
1977 case X86::VMOVUPSZrrk:
1978 Opc = X86::VBLENDMPSZrrk;
1980 case X86::VMOVDQA32Z128rrk:
1981 Opc = X86::VPBLENDMDZ128rrk;
1983 case X86::VMOVDQA32Z256rrk:
1984 Opc = X86::VPBLENDMDZ256rrk;
1986 case X86::VMOVDQA32Zrrk:
1987 Opc = X86::VPBLENDMDZrrk;
1989 case X86::VMOVDQA64Z128rrk:
1990 Opc = X86::VPBLENDMQZ128rrk;
1992 case X86::VMOVDQA64Z256rrk:
1993 Opc = X86::VPBLENDMQZ256rrk;
1995 case X86::VMOVDQA64Zrrk:
1996 Opc = X86::VPBLENDMQZrrk;
1998 case X86::VMOVAPDZ128rrk:
1999 Opc = X86::VBLENDMPDZ128rrk;
2001 case X86::VMOVAPDZ256rrk:
2002 Opc = X86::VBLENDMPDZ256rrk;
2004 case X86::VMOVAPDZrrk:
2005 Opc = X86::VBLENDMPDZrrk;
2007 case X86::VMOVAPSZ128rrk:
2008 Opc = X86::VBLENDMPSZ128rrk;
2010 case X86::VMOVAPSZ256rrk:
2011 Opc = X86::VBLENDMPSZ256rrk;
2013 case X86::VMOVAPSZrrk:
2014 Opc = X86::VBLENDMPSZrrk;
2020 .
add(
MI.getOperand(2))
2022 .
add(
MI.getOperand(3));
2033 for (
unsigned I = 0;
I < NumRegOperands; ++
I) {
2035 if (
Op.isReg() && (
Op.isDead() ||
Op.isKill()))
2041 MBB.insert(
MI.getIterator(), NewMI);
2062 unsigned SrcOpIdx2) {
2064 if (SrcOpIdx1 > SrcOpIdx2)
2067 unsigned Op1 = 1, Op2 = 2, Op3 = 3;
2073 if (SrcOpIdx1 == Op1 && SrcOpIdx2 == Op2)
2075 if (SrcOpIdx1 == Op1 && SrcOpIdx2 == Op3)
2077 if (SrcOpIdx1 == Op2 && SrcOpIdx2 == Op3)
2086 unsigned Opc =
MI.getOpcode();
2095 "Intrinsic instructions can't commute operand 1");
2100 assert(Case < 3 &&
"Unexpected case number!");
2105 const unsigned Form132Index = 0;
2106 const unsigned Form213Index = 1;
2107 const unsigned Form231Index = 2;
2108 static const unsigned FormMapping[][3] = {
2113 {Form231Index, Form213Index, Form132Index},
2118 {Form132Index, Form231Index, Form213Index},
2123 {Form213Index, Form132Index, Form231Index}};
2125 unsigned FMAForms[3];
2131 for (
unsigned FormIndex = 0; FormIndex < 3; FormIndex++)
2132 if (
Opc == FMAForms[FormIndex])
2133 return FMAForms[FormMapping[Case][FormIndex]];
2139 unsigned SrcOpIdx2) {
2143 assert(Case < 3 &&
"Unexpected case value!");
2146 static const uint8_t SwapMasks[3][4] = {
2147 {0x04, 0x10, 0x08, 0x20},
2148 {0x02, 0x10, 0x08, 0x40},
2149 {0x02, 0x04, 0x20, 0x40},
2152 uint8_t Imm =
MI.getOperand(
MI.getNumOperands() - 1).getImm();
2154 uint8_t NewImm = Imm & ~(SwapMasks[Case][0] | SwapMasks[Case][1] |
2155 SwapMasks[Case][2] | SwapMasks[Case][3]);
2157 if (Imm & SwapMasks[Case][0])
2158 NewImm |= SwapMasks[Case][1];
2159 if (Imm & SwapMasks[Case][1])
2160 NewImm |= SwapMasks[Case][0];
2161 if (Imm & SwapMasks[Case][2])
2162 NewImm |= SwapMasks[Case][3];
2163 if (Imm & SwapMasks[Case][3])
2164 NewImm |= SwapMasks[Case][2];
2165 MI.getOperand(
MI.getNumOperands() - 1).setImm(NewImm);
2171#define VPERM_CASES(Suffix) \
2172 case X86::VPERMI2##Suffix##Z128rr: \
2173 case X86::VPERMT2##Suffix##Z128rr: \
2174 case X86::VPERMI2##Suffix##Z256rr: \
2175 case X86::VPERMT2##Suffix##Z256rr: \
2176 case X86::VPERMI2##Suffix##Zrr: \
2177 case X86::VPERMT2##Suffix##Zrr: \
2178 case X86::VPERMI2##Suffix##Z128rm: \
2179 case X86::VPERMT2##Suffix##Z128rm: \
2180 case X86::VPERMI2##Suffix##Z256rm: \
2181 case X86::VPERMT2##Suffix##Z256rm: \
2182 case X86::VPERMI2##Suffix##Zrm: \
2183 case X86::VPERMT2##Suffix##Zrm: \
2184 case X86::VPERMI2##Suffix##Z128rrkz: \
2185 case X86::VPERMT2##Suffix##Z128rrkz: \
2186 case X86::VPERMI2##Suffix##Z256rrkz: \
2187 case X86::VPERMT2##Suffix##Z256rrkz: \
2188 case X86::VPERMI2##Suffix##Zrrkz: \
2189 case X86::VPERMT2##Suffix##Zrrkz: \
2190 case X86::VPERMI2##Suffix##Z128rmkz: \
2191 case X86::VPERMT2##Suffix##Z128rmkz: \
2192 case X86::VPERMI2##Suffix##Z256rmkz: \
2193 case X86::VPERMT2##Suffix##Z256rmkz: \
2194 case X86::VPERMI2##Suffix##Zrmkz: \
2195 case X86::VPERMT2##Suffix##Zrmkz:
2197#define VPERM_CASES_BROADCAST(Suffix) \
2198 VPERM_CASES(Suffix) \
2199 case X86::VPERMI2##Suffix##Z128rmb: \
2200 case X86::VPERMT2##Suffix##Z128rmb: \
2201 case X86::VPERMI2##Suffix##Z256rmb: \
2202 case X86::VPERMT2##Suffix##Z256rmb: \
2203 case X86::VPERMI2##Suffix##Zrmb: \
2204 case X86::VPERMT2##Suffix##Zrmb: \
2205 case X86::VPERMI2##Suffix##Z128rmbkz: \
2206 case X86::VPERMT2##Suffix##Z128rmbkz: \
2207 case X86::VPERMI2##Suffix##Z256rmbkz: \
2208 case X86::VPERMT2##Suffix##Z256rmbkz: \
2209 case X86::VPERMI2##Suffix##Zrmbkz: \
2210 case X86::VPERMT2##Suffix##Zrmbkz:
2223#undef VPERM_CASES_BROADCAST
2230#define VPERM_CASES(Orig, New) \
2231 case X86::Orig##Z128rr: \
2232 return X86::New##Z128rr; \
2233 case X86::Orig##Z128rrkz: \
2234 return X86::New##Z128rrkz; \
2235 case X86::Orig##Z128rm: \
2236 return X86::New##Z128rm; \
2237 case X86::Orig##Z128rmkz: \
2238 return X86::New##Z128rmkz; \
2239 case X86::Orig##Z256rr: \
2240 return X86::New##Z256rr; \
2241 case X86::Orig##Z256rrkz: \
2242 return X86::New##Z256rrkz; \
2243 case X86::Orig##Z256rm: \
2244 return X86::New##Z256rm; \
2245 case X86::Orig##Z256rmkz: \
2246 return X86::New##Z256rmkz; \
2247 case X86::Orig##Zrr: \
2248 return X86::New##Zrr; \
2249 case X86::Orig##Zrrkz: \
2250 return X86::New##Zrrkz; \
2251 case X86::Orig##Zrm: \
2252 return X86::New##Zrm; \
2253 case X86::Orig##Zrmkz: \
2254 return X86::New##Zrmkz;
2256#define VPERM_CASES_BROADCAST(Orig, New) \
2257 VPERM_CASES(Orig, New) \
2258 case X86::Orig##Z128rmb: \
2259 return X86::New##Z128rmb; \
2260 case X86::Orig##Z128rmbkz: \
2261 return X86::New##Z128rmbkz; \
2262 case X86::Orig##Z256rmb: \
2263 return X86::New##Z256rmb; \
2264 case X86::Orig##Z256rmbkz: \
2265 return X86::New##Z256rmbkz; \
2266 case X86::Orig##Zrmb: \
2267 return X86::New##Zrmb; \
2268 case X86::Orig##Zrmbkz: \
2269 return X86::New##Zrmbkz;
2287#undef VPERM_CASES_BROADCAST
2293 unsigned OpIdx2)
const {
2295 return std::exchange(NewMI,
false)
2296 ?
MI.getParent()->getParent()->CloneMachineInstr(&
MI)
2300 unsigned Opc =
MI.getOpcode();
2302#define CASE_ND(OP) \
2318#define FROM_TO_SIZE(A, B, S) \
2324 Opc = X86::B##_ND; \
2332 Opc = X86::A##_ND; \
2341 WorkingMI = CloneIfNew(
MI);
2350 WorkingMI = CloneIfNew(
MI);
2352 get(X86::PFSUBRrr ==
Opc ? X86::PFSUBrr : X86::PFSUBRrr));
2354 case X86::BLENDPDrri:
2355 case X86::BLENDPSrri:
2356 case X86::PBLENDWrri:
2357 case X86::VBLENDPDrri:
2358 case X86::VBLENDPSrri:
2359 case X86::VBLENDPDYrri:
2360 case X86::VBLENDPSYrri:
2361 case X86::VPBLENDDrri:
2362 case X86::VPBLENDWrri:
2363 case X86::VPBLENDDYrri:
2364 case X86::VPBLENDWYrri: {
2369 case X86::BLENDPDrri:
2370 Mask = (int8_t)0x03;
2372 case X86::BLENDPSrri:
2373 Mask = (int8_t)0x0F;
2375 case X86::PBLENDWrri:
2376 Mask = (int8_t)0xFF;
2378 case X86::VBLENDPDrri:
2379 Mask = (int8_t)0x03;
2381 case X86::VBLENDPSrri:
2382 Mask = (int8_t)0x0F;
2384 case X86::VBLENDPDYrri:
2385 Mask = (int8_t)0x0F;
2387 case X86::VBLENDPSYrri:
2388 Mask = (int8_t)0xFF;
2390 case X86::VPBLENDDrri:
2391 Mask = (int8_t)0x0F;
2393 case X86::VPBLENDWrri:
2394 Mask = (int8_t)0xFF;
2396 case X86::VPBLENDDYrri:
2397 Mask = (int8_t)0xFF;
2399 case X86::VPBLENDWYrri:
2400 Mask = (int8_t)0xFF;
2406 int8_t Imm =
MI.getOperand(3).getImm() & Mask;
2407 WorkingMI = CloneIfNew(
MI);
2411 case X86::INSERTPSrri:
2412 case X86::VINSERTPSrri:
2413 case X86::VINSERTPSZrri: {
2414 unsigned Imm =
MI.getOperand(
MI.getNumOperands() - 1).getImm();
2415 unsigned ZMask = Imm & 15;
2416 unsigned DstIdx = (Imm >> 4) & 3;
2417 unsigned SrcIdx = (Imm >> 6) & 3;
2421 if (DstIdx == SrcIdx && (ZMask & (1 << DstIdx)) == 0 &&
2424 assert(AltIdx < 4 &&
"Illegal insertion index");
2425 unsigned AltImm = (AltIdx << 6) | (AltIdx << 4) | ZMask;
2426 WorkingMI = CloneIfNew(
MI);
2435 case X86::VMOVSSrr: {
2437 if (Subtarget.hasSSE41()) {
2443 Opc = X86::BLENDPDrri;
2447 Opc = X86::BLENDPSrri;
2451 Opc = X86::VBLENDPDrri;
2455 Opc = X86::VBLENDPSrri;
2460 WorkingMI = CloneIfNew(
MI);
2466 assert(
Opc == X86::MOVSDrr &&
"Only MOVSD can commute to SHUFPD");
2467 WorkingMI = CloneIfNew(
MI);
2472 case X86::SHUFPDrri: {
2474 assert(
MI.getOperand(3).getImm() == 0x02 &&
"Unexpected immediate!");
2475 WorkingMI = CloneIfNew(
MI);
2480 case X86::PCLMULQDQrri:
2481 case X86::VPCLMULQDQrri:
2482 case X86::VPCLMULQDQYrri:
2483 case X86::VPCLMULQDQZrri:
2484 case X86::VPCLMULQDQZ128rri:
2485 case X86::VPCLMULQDQZ256rri: {
2488 unsigned Imm =
MI.getOperand(3).getImm();
2489 unsigned Src1Hi = Imm & 0x01;
2490 unsigned Src2Hi = Imm & 0x10;
2491 WorkingMI = CloneIfNew(
MI);
2495 case X86::VPCMPBZ128rri:
2496 case X86::VPCMPUBZ128rri:
2497 case X86::VPCMPBZ256rri:
2498 case X86::VPCMPUBZ256rri:
2499 case X86::VPCMPBZrri:
2500 case X86::VPCMPUBZrri:
2501 case X86::VPCMPDZ128rri:
2502 case X86::VPCMPUDZ128rri:
2503 case X86::VPCMPDZ256rri:
2504 case X86::VPCMPUDZ256rri:
2505 case X86::VPCMPDZrri:
2506 case X86::VPCMPUDZrri:
2507 case X86::VPCMPQZ128rri:
2508 case X86::VPCMPUQZ128rri:
2509 case X86::VPCMPQZ256rri:
2510 case X86::VPCMPUQZ256rri:
2511 case X86::VPCMPQZrri:
2512 case X86::VPCMPUQZrri:
2513 case X86::VPCMPWZ128rri:
2514 case X86::VPCMPUWZ128rri:
2515 case X86::VPCMPWZ256rri:
2516 case X86::VPCMPUWZ256rri:
2517 case X86::VPCMPWZrri:
2518 case X86::VPCMPUWZrri:
2519 case X86::VPCMPBZ128rrik:
2520 case X86::VPCMPUBZ128rrik:
2521 case X86::VPCMPBZ256rrik:
2522 case X86::VPCMPUBZ256rrik:
2523 case X86::VPCMPBZrrik:
2524 case X86::VPCMPUBZrrik:
2525 case X86::VPCMPDZ128rrik:
2526 case X86::VPCMPUDZ128rrik:
2527 case X86::VPCMPDZ256rrik:
2528 case X86::VPCMPUDZ256rrik:
2529 case X86::VPCMPDZrrik:
2530 case X86::VPCMPUDZrrik:
2531 case X86::VPCMPQZ128rrik:
2532 case X86::VPCMPUQZ128rrik:
2533 case X86::VPCMPQZ256rrik:
2534 case X86::VPCMPUQZ256rrik:
2535 case X86::VPCMPQZrrik:
2536 case X86::VPCMPUQZrrik:
2537 case X86::VPCMPWZ128rrik:
2538 case X86::VPCMPUWZ128rrik:
2539 case X86::VPCMPWZ256rrik:
2540 case X86::VPCMPUWZ256rrik:
2541 case X86::VPCMPWZrrik:
2542 case X86::VPCMPUWZrrik:
2543 WorkingMI = CloneIfNew(
MI);
2547 MI.getOperand(
MI.getNumOperands() - 1).getImm() & 0x7));
2550 case X86::VPCOMUBri:
2552 case X86::VPCOMUDri:
2554 case X86::VPCOMUQri:
2556 case X86::VPCOMUWri:
2557 WorkingMI = CloneIfNew(
MI);
2562 case X86::VCMPSDZrri:
2563 case X86::VCMPSSZrri:
2564 case X86::VCMPPDZrri:
2565 case X86::VCMPPSZrri:
2566 case X86::VCMPSHZrri:
2567 case X86::VCMPPHZrri:
2568 case X86::VCMPPHZ128rri:
2569 case X86::VCMPPHZ256rri:
2570 case X86::VCMPPDZ128rri:
2571 case X86::VCMPPSZ128rri:
2572 case X86::VCMPPDZ256rri:
2573 case X86::VCMPPSZ256rri:
2574 case X86::VCMPPDZrrik:
2575 case X86::VCMPPSZrrik:
2576 case X86::VCMPPHZrrik:
2577 case X86::VCMPPDZ128rrik:
2578 case X86::VCMPPSZ128rrik:
2579 case X86::VCMPPHZ128rrik:
2580 case X86::VCMPPDZ256rrik:
2581 case X86::VCMPPSZ256rrik:
2582 case X86::VCMPPHZ256rrik:
2583 WorkingMI = CloneIfNew(
MI);
2586 MI.getOperand(
MI.getNumExplicitOperands() - 1).getImm() & 0x1f));
2588 case X86::VPERM2F128rri:
2589 case X86::VPERM2I128rri:
2593 WorkingMI = CloneIfNew(
MI);
2596 case X86::MOVHLPSrr:
2597 case X86::UNPCKHPDrr:
2598 case X86::VMOVHLPSrr:
2599 case X86::VUNPCKHPDrr:
2600 case X86::VMOVHLPSZrr:
2601 case X86::VUNPCKHPDZ128rr:
2602 assert(Subtarget.hasSSE2() &&
"Commuting MOVHLP/UNPCKHPD requires SSE2!");
2607 case X86::MOVHLPSrr:
2608 Opc = X86::UNPCKHPDrr;
2610 case X86::UNPCKHPDrr:
2611 Opc = X86::MOVHLPSrr;
2613 case X86::VMOVHLPSrr:
2614 Opc = X86::VUNPCKHPDrr;
2616 case X86::VUNPCKHPDrr:
2617 Opc = X86::VMOVHLPSrr;
2619 case X86::VMOVHLPSZrr:
2620 Opc = X86::VUNPCKHPDZ128rr;
2622 case X86::VUNPCKHPDZ128rr:
2623 Opc = X86::VMOVHLPSZrr;
2626 WorkingMI = CloneIfNew(
MI);
2632 WorkingMI = CloneIfNew(
MI);
2633 unsigned OpNo =
MI.getDesc().getNumOperands() - 1;
2638 case X86::VPTERNLOGDZrri:
2639 case X86::VPTERNLOGDZrmi:
2640 case X86::VPTERNLOGDZ128rri:
2641 case X86::VPTERNLOGDZ128rmi:
2642 case X86::VPTERNLOGDZ256rri:
2643 case X86::VPTERNLOGDZ256rmi:
2644 case X86::VPTERNLOGQZrri:
2645 case X86::VPTERNLOGQZrmi:
2646 case X86::VPTERNLOGQZ128rri:
2647 case X86::VPTERNLOGQZ128rmi:
2648 case X86::VPTERNLOGQZ256rri:
2649 case X86::VPTERNLOGQZ256rmi:
2650 case X86::VPTERNLOGDZrrik:
2651 case X86::VPTERNLOGDZ128rrik:
2652 case X86::VPTERNLOGDZ256rrik:
2653 case X86::VPTERNLOGQZrrik:
2654 case X86::VPTERNLOGQZ128rrik:
2655 case X86::VPTERNLOGQZ256rrik:
2656 case X86::VPTERNLOGDZrrikz:
2657 case X86::VPTERNLOGDZrmikz:
2658 case X86::VPTERNLOGDZ128rrikz:
2659 case X86::VPTERNLOGDZ128rmikz:
2660 case X86::VPTERNLOGDZ256rrikz:
2661 case X86::VPTERNLOGDZ256rmikz:
2662 case X86::VPTERNLOGQZrrikz:
2663 case X86::VPTERNLOGQZrmikz:
2664 case X86::VPTERNLOGQZ128rrikz:
2665 case X86::VPTERNLOGQZ128rmikz:
2666 case X86::VPTERNLOGQZ256rrikz:
2667 case X86::VPTERNLOGQZ256rmikz:
2668 case X86::VPTERNLOGDZ128rmbi:
2669 case X86::VPTERNLOGDZ256rmbi:
2670 case X86::VPTERNLOGDZrmbi:
2671 case X86::VPTERNLOGQZ128rmbi:
2672 case X86::VPTERNLOGQZ256rmbi:
2673 case X86::VPTERNLOGQZrmbi:
2674 case X86::VPTERNLOGDZ128rmbikz:
2675 case X86::VPTERNLOGDZ256rmbikz:
2676 case X86::VPTERNLOGDZrmbikz:
2677 case X86::VPTERNLOGQZ128rmbikz:
2678 case X86::VPTERNLOGQZ256rmbikz:
2679 case X86::VPTERNLOGQZrmbikz: {
2680 WorkingMI = CloneIfNew(
MI);
2686 WorkingMI = CloneIfNew(
MI);
2692 WorkingMI = CloneIfNew(
MI);
2701bool X86InstrInfo::findThreeSrcCommutedOpIndices(
const MachineInstr &
MI,
2702 unsigned &SrcOpIdx1,
2703 unsigned &SrcOpIdx2,
2704 bool IsIntrinsic)
const {
2707 unsigned FirstCommutableVecOp = 1;
2708 unsigned LastCommutableVecOp = 3;
2709 unsigned KMaskOp = -1U;
2732 FirstCommutableVecOp = 3;
2734 LastCommutableVecOp++;
2735 }
else if (IsIntrinsic) {
2738 FirstCommutableVecOp = 2;
2741 if (
isMem(
MI, LastCommutableVecOp))
2742 LastCommutableVecOp--;
2747 if (SrcOpIdx1 != CommuteAnyOperandIndex &&
2748 (SrcOpIdx1 < FirstCommutableVecOp || SrcOpIdx1 > LastCommutableVecOp ||
2749 SrcOpIdx1 == KMaskOp))
2751 if (SrcOpIdx2 != CommuteAnyOperandIndex &&
2752 (SrcOpIdx2 < FirstCommutableVecOp || SrcOpIdx2 > LastCommutableVecOp ||
2753 SrcOpIdx2 == KMaskOp))
2758 if (SrcOpIdx1 == CommuteAnyOperandIndex ||
2759 SrcOpIdx2 == CommuteAnyOperandIndex) {
2760 unsigned CommutableOpIdx2 = SrcOpIdx2;
2764 if (SrcOpIdx1 == SrcOpIdx2)
2767 CommutableOpIdx2 = LastCommutableVecOp;
2768 else if (SrcOpIdx2 == CommuteAnyOperandIndex)
2770 CommutableOpIdx2 = SrcOpIdx1;
2774 Register Op2Reg =
MI.getOperand(CommutableOpIdx2).getReg();
2776 unsigned CommutableOpIdx1;
2777 for (CommutableOpIdx1 = LastCommutableVecOp;
2778 CommutableOpIdx1 >= FirstCommutableVecOp; CommutableOpIdx1--) {
2780 if (CommutableOpIdx1 == KMaskOp)
2786 if (Op2Reg !=
MI.getOperand(CommutableOpIdx1).getReg())
2791 if (CommutableOpIdx1 < FirstCommutableVecOp)
2796 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,
2805 unsigned &SrcOpIdx1,
2806 unsigned &SrcOpIdx2)
const {
2808 if (!
Desc.isCommutable())
2811 switch (
MI.getOpcode()) {
2816 case X86::VCMPSDrri:
2817 case X86::VCMPSSrri:
2818 case X86::VCMPPDrri:
2819 case X86::VCMPPSrri:
2820 case X86::VCMPPDYrri:
2821 case X86::VCMPPSYrri:
2822 case X86::VCMPSDZrri:
2823 case X86::VCMPSSZrri:
2824 case X86::VCMPPDZrri:
2825 case X86::VCMPPSZrri:
2826 case X86::VCMPSHZrri:
2827 case X86::VCMPPHZrri:
2828 case X86::VCMPPHZ128rri:
2829 case X86::VCMPPHZ256rri:
2830 case X86::VCMPPDZ128rri:
2831 case X86::VCMPPSZ128rri:
2832 case X86::VCMPPDZ256rri:
2833 case X86::VCMPPSZ256rri:
2834 case X86::VCMPPDZrrik:
2835 case X86::VCMPPSZrrik:
2836 case X86::VCMPPHZrrik:
2837 case X86::VCMPPDZ128rrik:
2838 case X86::VCMPPSZ128rrik:
2839 case X86::VCMPPHZ128rrik:
2840 case X86::VCMPPDZ256rrik:
2841 case X86::VCMPPSZ256rrik:
2842 case X86::VCMPPHZ256rrik: {
2847 unsigned Imm =
MI.getOperand(3 + OpOffset).getImm() & 0x7;
2864 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 1 + OpOffset,
2871 if (Subtarget.hasSSE41())
2874 case X86::SHUFPDrri:
2876 if (
MI.getOperand(3).getImm() == 0x02)
2879 case X86::MOVHLPSrr:
2880 case X86::UNPCKHPDrr:
2881 case X86::VMOVHLPSrr:
2882 case X86::VUNPCKHPDrr:
2883 case X86::VMOVHLPSZrr:
2884 case X86::VUNPCKHPDZ128rr:
2885 if (Subtarget.hasSSE2())
2888 case X86::VPTERNLOGDZrri:
2889 case X86::VPTERNLOGDZrmi:
2890 case X86::VPTERNLOGDZ128rri:
2891 case X86::VPTERNLOGDZ128rmi:
2892 case X86::VPTERNLOGDZ256rri:
2893 case X86::VPTERNLOGDZ256rmi:
2894 case X86::VPTERNLOGQZrri:
2895 case X86::VPTERNLOGQZrmi:
2896 case X86::VPTERNLOGQZ128rri:
2897 case X86::VPTERNLOGQZ128rmi:
2898 case X86::VPTERNLOGQZ256rri:
2899 case X86::VPTERNLOGQZ256rmi:
2900 case X86::VPTERNLOGDZrrik:
2901 case X86::VPTERNLOGDZ128rrik:
2902 case X86::VPTERNLOGDZ256rrik:
2903 case X86::VPTERNLOGQZrrik:
2904 case X86::VPTERNLOGQZ128rrik:
2905 case X86::VPTERNLOGQZ256rrik:
2906 case X86::VPTERNLOGDZrrikz:
2907 case X86::VPTERNLOGDZrmikz:
2908 case X86::VPTERNLOGDZ128rrikz:
2909 case X86::VPTERNLOGDZ128rmikz:
2910 case X86::VPTERNLOGDZ256rrikz:
2911 case X86::VPTERNLOGDZ256rmikz:
2912 case X86::VPTERNLOGQZrrikz:
2913 case X86::VPTERNLOGQZrmikz:
2914 case X86::VPTERNLOGQZ128rrikz:
2915 case X86::VPTERNLOGQZ128rmikz:
2916 case X86::VPTERNLOGQZ256rrikz:
2917 case X86::VPTERNLOGQZ256rmikz:
2918 case X86::VPTERNLOGDZ128rmbi:
2919 case X86::VPTERNLOGDZ256rmbi:
2920 case X86::VPTERNLOGDZrmbi:
2921 case X86::VPTERNLOGQZ128rmbi:
2922 case X86::VPTERNLOGQZ256rmbi:
2923 case X86::VPTERNLOGQZrmbi:
2924 case X86::VPTERNLOGDZ128rmbikz:
2925 case X86::VPTERNLOGDZ256rmbikz:
2926 case X86::VPTERNLOGDZrmbikz:
2927 case X86::VPTERNLOGQZ128rmbikz:
2928 case X86::VPTERNLOGQZ256rmbikz:
2929 case X86::VPTERNLOGQZrmbikz:
2930 return findThreeSrcCommutedOpIndices(
MI, SrcOpIdx1, SrcOpIdx2);
2931 case X86::VPDPWSSDYrr:
2932 case X86::VPDPWSSDrr:
2933 case X86::VPDPWSSDSYrr:
2934 case X86::VPDPWSSDSrr:
2935 case X86::VPDPWUUDrr:
2936 case X86::VPDPWUUDYrr:
2937 case X86::VPDPWUUDSrr:
2938 case X86::VPDPWUUDSYrr:
2939 case X86::VPDPBSSDSrr:
2940 case X86::VPDPBSSDSYrr:
2941 case X86::VPDPBSSDrr:
2942 case X86::VPDPBSSDYrr:
2943 case X86::VPDPBUUDSrr:
2944 case X86::VPDPBUUDSYrr:
2945 case X86::VPDPBUUDrr:
2946 case X86::VPDPBUUDYrr:
2947 case X86::VPDPBSSDSZ128rr:
2948 case X86::VPDPBSSDSZ128rrk:
2949 case X86::VPDPBSSDSZ128rrkz:
2950 case X86::VPDPBSSDSZ256rr:
2951 case X86::VPDPBSSDSZ256rrk:
2952 case X86::VPDPBSSDSZ256rrkz:
2953 case X86::VPDPBSSDSZrr:
2954 case X86::VPDPBSSDSZrrk:
2955 case X86::VPDPBSSDSZrrkz:
2956 case X86::VPDPBSSDZ128rr:
2957 case X86::VPDPBSSDZ128rrk:
2958 case X86::VPDPBSSDZ128rrkz:
2959 case X86::VPDPBSSDZ256rr:
2960 case X86::VPDPBSSDZ256rrk:
2961 case X86::VPDPBSSDZ256rrkz:
2962 case X86::VPDPBSSDZrr:
2963 case X86::VPDPBSSDZrrk:
2964 case X86::VPDPBSSDZrrkz:
2965 case X86::VPDPBUUDSZ128rr:
2966 case X86::VPDPBUUDSZ128rrk:
2967 case X86::VPDPBUUDSZ128rrkz:
2968 case X86::VPDPBUUDSZ256rr:
2969 case X86::VPDPBUUDSZ256rrk:
2970 case X86::VPDPBUUDSZ256rrkz:
2971 case X86::VPDPBUUDSZrr:
2972 case X86::VPDPBUUDSZrrk:
2973 case X86::VPDPBUUDSZrrkz:
2974 case X86::VPDPBUUDZ128rr:
2975 case X86::VPDPBUUDZ128rrk:
2976 case X86::VPDPBUUDZ128rrkz:
2977 case X86::VPDPBUUDZ256rr:
2978 case X86::VPDPBUUDZ256rrk:
2979 case X86::VPDPBUUDZ256rrkz:
2980 case X86::VPDPBUUDZrr:
2981 case X86::VPDPBUUDZrrk:
2982 case X86::VPDPBUUDZrrkz:
2983 case X86::VPDPWSSDZ128rr:
2984 case X86::VPDPWSSDZ128rrk:
2985 case X86::VPDPWSSDZ128rrkz:
2986 case X86::VPDPWSSDZ256rr:
2987 case X86::VPDPWSSDZ256rrk:
2988 case X86::VPDPWSSDZ256rrkz:
2989 case X86::VPDPWSSDZrr:
2990 case X86::VPDPWSSDZrrk:
2991 case X86::VPDPWSSDZrrkz:
2992 case X86::VPDPWSSDSZ128rr:
2993 case X86::VPDPWSSDSZ128rrk:
2994 case X86::VPDPWSSDSZ128rrkz:
2995 case X86::VPDPWSSDSZ256rr:
2996 case X86::VPDPWSSDSZ256rrk:
2997 case X86::VPDPWSSDSZ256rrkz:
2998 case X86::VPDPWSSDSZrr:
2999 case X86::VPDPWSSDSZrrk:
3000 case X86::VPDPWSSDSZrrkz:
3001 case X86::VPDPWUUDZ128rr:
3002 case X86::VPDPWUUDZ128rrk:
3003 case X86::VPDPWUUDZ128rrkz:
3004 case X86::VPDPWUUDZ256rr:
3005 case X86::VPDPWUUDZ256rrk:
3006 case X86::VPDPWUUDZ256rrkz:
3007 case X86::VPDPWUUDZrr:
3008 case X86::VPDPWUUDZrrk:
3009 case X86::VPDPWUUDZrrkz:
3010 case X86::VPDPWUUDSZ128rr:
3011 case X86::VPDPWUUDSZ128rrk:
3012 case X86::VPDPWUUDSZ128rrkz:
3013 case X86::VPDPWUUDSZ256rr:
3014 case X86::VPDPWUUDSZ256rrk:
3015 case X86::VPDPWUUDSZ256rrkz:
3016 case X86::VPDPWUUDSZrr:
3017 case X86::VPDPWUUDSZrrk:
3018 case X86::VPDPWUUDSZrrkz:
3019 case X86::VPMADD52HUQrr:
3020 case X86::VPMADD52HUQYrr:
3021 case X86::VPMADD52HUQZ128r:
3022 case X86::VPMADD52HUQZ128rk:
3023 case X86::VPMADD52HUQZ128rkz:
3024 case X86::VPMADD52HUQZ256r:
3025 case X86::VPMADD52HUQZ256rk:
3026 case X86::VPMADD52HUQZ256rkz:
3027 case X86::VPMADD52HUQZr:
3028 case X86::VPMADD52HUQZrk:
3029 case X86::VPMADD52HUQZrkz:
3030 case X86::VPMADD52LUQrr:
3031 case X86::VPMADD52LUQYrr:
3032 case X86::VPMADD52LUQZ128r:
3033 case X86::VPMADD52LUQZ128rk:
3034 case X86::VPMADD52LUQZ128rkz:
3035 case X86::VPMADD52LUQZ256r:
3036 case X86::VPMADD52LUQZ256rk:
3037 case X86::VPMADD52LUQZ256rkz:
3038 case X86::VPMADD52LUQZr:
3039 case X86::VPMADD52LUQZrk:
3040 case X86::VPMADD52LUQZrkz:
3041 case X86::VFMADDCPHZr:
3042 case X86::VFMADDCPHZrk:
3043 case X86::VFMADDCPHZrkz:
3044 case X86::VFMADDCPHZ128r:
3045 case X86::VFMADDCPHZ128rk:
3046 case X86::VFMADDCPHZ128rkz:
3047 case X86::VFMADDCPHZ256r:
3048 case X86::VFMADDCPHZ256rk:
3049 case X86::VFMADDCPHZ256rkz:
3050 case X86::VFMADDCSHZr:
3051 case X86::VFMADDCSHZrk:
3052 case X86::VFMADDCSHZrkz: {
3053 unsigned CommutableOpIdx1 = 2;
3054 unsigned CommutableOpIdx2 = 3;
3060 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,
3063 if (!
MI.getOperand(SrcOpIdx1).isReg() || !
MI.getOperand(SrcOpIdx2).isReg())
3073 return findThreeSrcCommutedOpIndices(
MI, SrcOpIdx1, SrcOpIdx2,
3080 unsigned CommutableOpIdx1 =
Desc.getNumDefs() + 1;
3081 unsigned CommutableOpIdx2 =
Desc.getNumDefs() + 2;
3084 if ((
MI.getDesc().getOperandConstraint(
Desc.getNumDefs(),
3099 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,
3103 if (!
MI.getOperand(SrcOpIdx1).isReg() ||
3104 !
MI.getOperand(SrcOpIdx2).isReg())
3116 unsigned Opcode =
MI->getOpcode();
3117 if (Opcode != X86::LEA32r && Opcode != X86::LEA64r &&
3118 Opcode != X86::LEA64_32r)
3140 unsigned Opcode =
MI.getOpcode();
3141 if (Opcode != X86::ADD32rr && Opcode != X86::ADD64rr)
3168 unsigned Opcode =
MCID.getOpcode();
3169 if (!(X86::isJCC(Opcode) || X86::isSETCC(Opcode) || X86::isSETZUCC(Opcode) ||
3170 X86::isCMOVCC(Opcode) || X86::isCFCMOVCC(Opcode) ||
3171 X86::isCCMPCC(Opcode) || X86::isCTESTCC(Opcode)))
3174 unsigned NumUses =
MCID.getNumOperands() -
MCID.getNumDefs();
3183 CondNo +=
MCID.getNumDefs();
3193 return X86::isSETCC(
MI.getOpcode()) || X86::isSETZUCC(
MI.getOpcode())
3209 return X86::isCCMPCC(
MI.getOpcode()) || X86::isCTESTCC(
MI.getOpcode())
3240 enum { CF = 1, ZF = 2, SF = 4, OF = 8, PF = CF };
3271#define GET_X86_NF_TRANSFORM_TABLE
3272#define GET_X86_ND2NONND_TABLE
3273#include "X86GenInstrMapping.inc"
3278 return (
I == Table.
end() ||
I->OldOpc !=
Opc) ? 0U :
I->NewOpc;
3281#if defined(EXPENSIVE_CHECKS) && !defined(NDEBUG)
3283 static std::atomic<bool> NFTableChecked(
false);
3284 if (!NFTableChecked.load(std::memory_order_relaxed)) {
3286 "X86NFTransformTable is not sorted!");
3287 NFTableChecked.store(
true, std::memory_order_relaxed);
3294#if defined(EXPENSIVE_CHECKS) && !defined(NDEBUG)
3296 static std::atomic<bool> NDTableChecked(
false);
3297 if (!NDTableChecked.load(std::memory_order_relaxed)) {
3299 "X86ND2NonNDTableis not sorted!");
3300 NDTableChecked.store(
true, std::memory_order_relaxed);
3380std::pair<X86::CondCode, bool>
3383 bool NeedSwap =
false;
3384 switch (Predicate) {
3463 return std::make_pair(CC, NeedSwap);
3472#define GET_ND_IF_ENABLED(OPC) (HasNDD ? OPC##_ND : OPC)
3566 switch (Imm & 0x3) {
3584 if (Info.RegClass == X86::VR128RegClassID ||
3585 Info.RegClass == X86::VR128XRegClassID)
3587 if (Info.RegClass == X86::VR256RegClassID ||
3588 Info.RegClass == X86::VR256XRegClassID)
3590 if (Info.RegClass == X86::VR512RegClassID)
3597 return (
Reg == X86::FPCW ||
Reg == X86::FPSW ||
3598 (
Reg >= X86::ST0 &&
Reg <= X86::ST7));
3606 if (
MI.isCall() ||
MI.isInlineAsm())
3630#ifdef EXPENSIVE_CHECKS
3632 "Got false negative from X86II::getMemoryOperandNo()!");
3642#ifdef EXPENSIVE_CHECKS
3644 "Expected no operands to have OPERAND_MEMORY type!");
3653 if (IsMemOp(
Desc.operands()[
I])) {
3654#ifdef EXPENSIVE_CHECKS
3658 "Expected all five operands in the memory reference to have "
3659 "OPERAND_MEMORY type!");
3671 "Unexpected number of operands!");
3674 if (!Index.isReg() || Index.getReg() != X86::NoRegister)
3682 MI.getParent()->getParent()->getConstantPool()->getConstants();
3694 switch (
MI.getOpcode()) {
3695 case X86::TCRETURNdi:
3696 case X86::TCRETURNri:
3697 case X86::TCRETURNmi:
3698 case X86::TCRETURNdi64:
3699 case X86::TCRETURNri64:
3700 case X86::TCRETURNri64_ImpCall:
3701 case X86::TCRETURNmi64:
3720 if (Symbol ==
"__x86_indirect_thunk_r11")
3725 if (TailCall.getOpcode() != X86::TCRETURNdi &&
3726 TailCall.getOpcode() != X86::TCRETURNdi64) {
3731 if (Subtarget.isTargetWin64() && MF->
hasWinCFI()) {
3744 TailCall.getOperand(1).getImm() != 0) {
3758 while (
I !=
MBB.begin()) {
3760 if (
I->isDebugInstr())
3763 assert(0 &&
"Can't find the branch to replace!");
3767 if (CC != BranchCond[0].
getImm())
3773 unsigned Opc = TailCall.getOpcode() == X86::TCRETURNdi ? X86::TCRETURNdicc
3774 : X86::TCRETURNdi64cc;
3787 LiveRegs.stepForward(*MIB, Clobbers);
3788 for (
const auto &
C : Clobbers) {
3793 I->eraseFromParent();
3807 if (Succ->isEHPad() || (Succ ==
TBB && FallthroughBB))
3810 if (FallthroughBB && FallthroughBB !=
TBB)
3812 FallthroughBB = Succ;
3814 return FallthroughBB;
3817bool X86InstrInfo::analyzeBranchImpl(
3828 if (
I->isDebugInstr())
3833 if (!isUnpredicatedTerminator(*
I))
3842 if (
I->getOpcode() == X86::JMP_1) {
3846 TBB =
I->getOperand(0).getMBB();
3861 UnCondBrIter =
MBB.
end();
3866 TBB =
I->getOperand(0).getMBB();
3877 if (
I->findRegisterUseOperand(X86::EFLAGS,
nullptr)->isUndef())
3883 TBB =
I->getOperand(0).getMBB();
3898 if (OldBranchCode == BranchCode &&
TBB == NewTBB)
3904 if (
TBB == NewTBB &&
3937 Cond[0].setImm(BranchCode);
3948 bool AllowModify)
const {
3950 return analyzeBranchImpl(
MBB,
TBB, FBB,
Cond, CondBranches, AllowModify);
3956 assert(MemRefBegin >= 0 &&
"instr should have memory operand");
3968 if (!
Reg.isVirtual())
3973 unsigned Opcode =
MI->getOpcode();
3974 if (Opcode != X86::LEA64r && Opcode != X86::LEA32r)
3980 unsigned Opcode =
MI.getOpcode();
3983 if (Opcode == X86::JMP64m || Opcode == X86::JMP32m) {
3991 if (Opcode == X86::JMP64r || Opcode == X86::JMP32r) {
3993 if (!Reg.isVirtual())
4000 if (
Add->getOpcode() != X86::ADD64rr &&
Add->getOpcode() != X86::ADD32rr)
4013 MachineBranchPredicate &MBP,
4014 bool AllowModify)
const {
4015 using namespace std::placeholders;
4019 if (analyzeBranchImpl(
MBB, MBP.TrueDest, MBP.FalseDest,
Cond, CondBranches,
4023 if (
Cond.size() != 1)
4026 assert(MBP.TrueDest &&
"expected!");
4029 MBP.FalseDest =
MBB.getNextNode();
4034 bool SingleUseCondition =
true;
4037 if (
MI.modifiesRegister(X86::EFLAGS,
TRI)) {
4042 if (
MI.readsRegister(X86::EFLAGS,
TRI))
4043 SingleUseCondition =
false;
4049 if (SingleUseCondition) {
4050 for (
auto *Succ :
MBB.successors())
4051 if (Succ->isLiveIn(X86::EFLAGS))
4052 SingleUseCondition =
false;
4055 MBP.ConditionDef = ConditionDef;
4056 MBP.SingleUseCondition = SingleUseCondition;
4063 const unsigned TestOpcode =
4064 Subtarget.is64Bit() ? X86::TEST64rr : X86::TEST32rr;
4066 if (ConditionDef->
getOpcode() == TestOpcode &&
4073 ? MachineBranchPredicate::PRED_NE
4074 : MachineBranchPredicate::PRED_EQ;
4082 int *BytesRemoved)
const {
4083 assert(!BytesRemoved &&
"code size not handled");
4088 while (
I !=
MBB.begin()) {
4090 if (
I->isDebugInstr())
4092 if (
I->getOpcode() != X86::JMP_1 &&
4096 I->eraseFromParent();
4110 assert(
TBB &&
"insertBranch must not be told to insert a fallthrough");
4112 "X86 branch conditions have one component!");
4113 assert(!BytesAdded &&
"code size not handled");
4117 assert(!FBB &&
"Unconditional branch with multiple successors!");
4123 bool FallThru = FBB ==
nullptr;
4138 if (FBB ==
nullptr) {
4140 assert(FBB &&
"MBB cannot be the last block in function when the false "
4141 "body is a fall-through.");
4165 Register FalseReg,
int &CondCycles,
4166 int &TrueCycles,
int &FalseCycles)
const {
4168 if (!Subtarget.canUseCMOV())
4170 if (
Cond.size() != 1)
4179 RI.getCommonSubClass(
MRI.getRegClass(TrueReg),
MRI.getRegClass(FalseReg));
4184 if (X86::GR16RegClass.hasSubClassEq(RC) ||
4185 X86::GR32RegClass.hasSubClassEq(RC) ||
4186 X86::GR64RegClass.hasSubClassEq(RC)) {
4207 assert(
Cond.size() == 1 &&
"Invalid Cond array");
4210 false , Subtarget.hasNDD());
4219 return X86::GR8_ABCD_HRegClass.contains(
Reg);
4225 bool HasAVX = Subtarget.
hasAVX();
4227 bool HasEGPR = Subtarget.hasEGPR();
4234 if (X86::VK16RegClass.
contains(SrcReg)) {
4235 if (X86::GR64RegClass.
contains(DestReg)) {
4236 assert(Subtarget.hasBWI());
4237 return HasEGPR ? X86::KMOVQrk_EVEX : X86::KMOVQrk;
4239 if (X86::GR32RegClass.
contains(DestReg))
4240 return Subtarget.hasBWI() ? (HasEGPR ? X86::KMOVDrk_EVEX : X86::KMOVDrk)
4241 : (HasEGPR ? X86::KMOVWrk_EVEX : X86::KMOVWrk);
4249 if (X86::VK16RegClass.
contains(DestReg)) {
4250 if (X86::GR64RegClass.
contains(SrcReg)) {
4251 assert(Subtarget.hasBWI());
4252 return HasEGPR ? X86::KMOVQkr_EVEX : X86::KMOVQkr;
4254 if (X86::GR32RegClass.
contains(SrcReg))
4255 return Subtarget.hasBWI() ? (HasEGPR ? X86::KMOVDkr_EVEX : X86::KMOVDkr)
4256 : (HasEGPR ? X86::KMOVWkr_EVEX : X86::KMOVWkr);
4264 if (X86::GR64RegClass.
contains(DestReg)) {
4265 if (X86::VR128XRegClass.
contains(SrcReg))
4267 return HasAVX512 ? X86::VMOVPQIto64Zrr
4268 : HasAVX ? X86::VMOVPQIto64rr
4269 : X86::MOVPQIto64rr;
4270 if (X86::VR64RegClass.
contains(SrcReg))
4272 return X86::MMX_MOVD64from64rr;
4273 }
else if (X86::GR64RegClass.
contains(SrcReg)) {
4275 if (X86::VR128XRegClass.
contains(DestReg))
4276 return HasAVX512 ? X86::VMOV64toPQIZrr
4277 : HasAVX ? X86::VMOV64toPQIrr
4278 : X86::MOV64toPQIrr;
4280 if (X86::VR64RegClass.
contains(DestReg))
4281 return X86::MMX_MOVD64to64rr;
4287 if (X86::GR32RegClass.
contains(DestReg) &&
4288 X86::VR128XRegClass.
contains(SrcReg))
4290 return HasAVX512 ? X86::VMOVPDI2DIZrr
4291 : HasAVX ? X86::VMOVPDI2DIrr
4294 if (X86::VR128XRegClass.
contains(DestReg) &&
4295 X86::GR32RegClass.
contains(SrcReg))
4297 return HasAVX512 ? X86::VMOVDI2PDIZrr
4298 : HasAVX ? X86::VMOVDI2PDIrr
4307 bool RenamableDest,
bool RenamableSrc)
const {
4309 bool HasAVX = Subtarget.hasAVX();
4310 bool HasVLX = Subtarget.hasVLX();
4311 bool HasEGPR = Subtarget.hasEGPR();
4313 if (X86::GR64RegClass.
contains(DestReg, SrcReg))
4315 else if (X86::GR32RegClass.
contains(DestReg, SrcReg))
4317 else if (X86::GR16RegClass.
contains(DestReg, SrcReg))
4319 else if (X86::GR8RegClass.
contains(DestReg, SrcReg)) {
4322 if ((
isHReg(DestReg) ||
isHReg(SrcReg)) && Subtarget.is64Bit()) {
4323 Opc = X86::MOV8rr_NOREX;
4326 "8-bit H register can not be copied outside GR8_NOREX");
4329 }
else if (X86::VR64RegClass.
contains(DestReg, SrcReg))
4330 Opc = X86::MMX_MOVQ64rr;
4331 else if (X86::VR128XRegClass.
contains(DestReg, SrcReg)) {
4333 Opc = X86::VMOVAPSZ128rr;
4334 else if (X86::VR128RegClass.
contains(DestReg, SrcReg))
4335 Opc = HasAVX ? X86::VMOVAPSrr : X86::MOVAPSrr;
4339 Opc = X86::VMOVAPSZrr;
4342 TRI->getMatchingSuperReg(DestReg, X86::sub_xmm, &X86::VR512RegClass);
4344 TRI->getMatchingSuperReg(SrcReg, X86::sub_xmm, &X86::VR512RegClass);
4346 }
else if (X86::VR256XRegClass.
contains(DestReg, SrcReg)) {
4348 Opc = X86::VMOVAPSZ256rr;
4349 else if (X86::VR256RegClass.
contains(DestReg, SrcReg))
4350 Opc = X86::VMOVAPSYrr;
4354 Opc = X86::VMOVAPSZrr;
4357 TRI->getMatchingSuperReg(DestReg, X86::sub_ymm, &X86::VR512RegClass);
4359 TRI->getMatchingSuperReg(SrcReg, X86::sub_ymm, &X86::VR512RegClass);
4361 }
else if (X86::VR512RegClass.
contains(DestReg, SrcReg))
4362 Opc = X86::VMOVAPSZrr;
4365 else if (X86::VK16RegClass.
contains(DestReg, SrcReg))
4366 Opc = Subtarget.hasBWI() ? (HasEGPR ? X86::KMOVQkk_EVEX : X86::KMOVQkk)
4367 : (HasEGPR ? X86::KMOVQkk_EVEX : X86::KMOVWkk);
4377 if (SrcReg == X86::EFLAGS || DestReg == X86::EFLAGS) {
4385 LLVM_DEBUG(
dbgs() <<
"Cannot copy " << RI.getName(SrcReg) <<
" to "
4386 << RI.getName(DestReg) <<
'\n');
4390std::optional<DestSourcePair>
4392 if (
MI.isMoveReg()) {
4396 if (
MI.getOperand(0).isUndef() &&
MI.getOperand(0).getSubReg())
4397 return std::nullopt;
4401 return std::nullopt;
4406 return Load ? X86::VMOVSHZrm_alt : X86::VMOVSHZmr;
4408 return X86::MOVSHPrm;
4409 return X86::MOVSHPmr;
4414 bool IsStackAligned,
4416 bool HasAVX = STI.
hasAVX();
4418 bool HasVLX = STI.hasVLX();
4419 bool HasEGPR = STI.hasEGPR();
4421 assert(RC !=
nullptr &&
"Invalid target register class");
4426 assert(X86::GR8RegClass.hasSubClassEq(RC) &&
"Unknown 1-byte regclass");
4430 if (
isHReg(
Reg) || X86::GR8_ABCD_HRegClass.hasSubClassEq(RC))
4431 return Load ? X86::MOV8rm_NOREX : X86::MOV8mr_NOREX;
4432 return Load ? X86::MOV8rm : X86::MOV8mr;
4434 if (X86::VK16RegClass.hasSubClassEq(RC))
4435 return Load ? (HasEGPR ? X86::KMOVWkm_EVEX : X86::KMOVWkm)
4436 : (HasEGPR ? X86::KMOVWmk_EVEX : X86::KMOVWmk);
4437 assert(X86::GR16RegClass.hasSubClassEq(RC) &&
"Unknown 2-byte regclass");
4438 return Load ? X86::MOV16rm : X86::MOV16mr;
4440 if (X86::GR32RegClass.hasSubClassEq(RC))
4441 return Load ? X86::MOV32rm : X86::MOV32mr;
4442 if (X86::FR32XRegClass.hasSubClassEq(RC))
4443 return Load ? (HasAVX512 ? X86::VMOVSSZrm_alt
4444 : HasAVX ? X86::VMOVSSrm_alt
4446 : (HasAVX512 ? X86::VMOVSSZmr
4447 : HasAVX ? X86::VMOVSSmr
4449 if (X86::RFP32RegClass.hasSubClassEq(RC))
4450 return Load ? X86::LD_Fp32m : X86::ST_Fp32m;
4451 if (X86::VK32RegClass.hasSubClassEq(RC)) {
4452 assert(STI.hasBWI() &&
"KMOVD requires BWI");
4453 return Load ? (HasEGPR ? X86::KMOVDkm_EVEX : X86::KMOVDkm)
4454 : (HasEGPR ? X86::KMOVDmk_EVEX : X86::KMOVDmk);
4458 if (X86::VK1PAIRRegClass.hasSubClassEq(RC) ||
4459 X86::VK2PAIRRegClass.hasSubClassEq(RC) ||
4460 X86::VK4PAIRRegClass.hasSubClassEq(RC) ||
4461 X86::VK8PAIRRegClass.hasSubClassEq(RC) ||
4462 X86::VK16PAIRRegClass.hasSubClassEq(RC))
4463 return Load ? X86::MASKPAIR16LOAD : X86::MASKPAIR16STORE;
4464 if (X86::FR16RegClass.hasSubClassEq(RC) ||
4465 X86::FR16XRegClass.hasSubClassEq(RC))
4469 if (X86::GR64RegClass.hasSubClassEq(RC))
4470 return Load ? X86::MOV64rm : X86::MOV64mr;
4471 if (X86::FR64XRegClass.hasSubClassEq(RC))
4472 return Load ? (HasAVX512 ? X86::VMOVSDZrm_alt
4473 : HasAVX ? X86::VMOVSDrm_alt
4475 : (HasAVX512 ? X86::VMOVSDZmr
4476 : HasAVX ? X86::VMOVSDmr
4478 if (X86::VR64RegClass.hasSubClassEq(RC))
4479 return Load ? X86::MMX_MOVQ64rm : X86::MMX_MOVQ64mr;
4480 if (X86::RFP64RegClass.hasSubClassEq(RC))
4481 return Load ? X86::LD_Fp64m : X86::ST_Fp64m;
4482 if (X86::VK64RegClass.hasSubClassEq(RC)) {
4483 assert(STI.hasBWI() &&
"KMOVQ requires BWI");
4484 return Load ? (HasEGPR ? X86::KMOVQkm_EVEX : X86::KMOVQkm)
4485 : (HasEGPR ? X86::KMOVQmk_EVEX : X86::KMOVQmk);
4489 assert(X86::RFP80RegClass.hasSubClassEq(RC) &&
"Unknown 10-byte regclass");
4490 return Load ? X86::LD_Fp80m : X86::ST_FpP80m;
4492 if (X86::VR128XRegClass.hasSubClassEq(RC)) {
4495 return Load ? (HasVLX ? X86::VMOVAPSZ128rm
4496 : HasAVX512 ? X86::VMOVAPSZ128rm_NOVLX
4497 : HasAVX ? X86::VMOVAPSrm
4499 : (HasVLX ? X86::VMOVAPSZ128mr
4500 : HasAVX512 ? X86::VMOVAPSZ128mr_NOVLX
4501 : HasAVX ? X86::VMOVAPSmr
4504 return Load ? (HasVLX ? X86::VMOVUPSZ128rm
4505 : HasAVX512 ? X86::VMOVUPSZ128rm_NOVLX
4506 : HasAVX ? X86::VMOVUPSrm
4508 : (HasVLX ? X86::VMOVUPSZ128mr
4509 : HasAVX512 ? X86::VMOVUPSZ128mr_NOVLX
4510 : HasAVX ? X86::VMOVUPSmr
4516 assert(X86::VR256XRegClass.hasSubClassEq(RC) &&
"Unknown 32-byte regclass");
4519 return Load ? (HasVLX ? X86::VMOVAPSZ256rm
4520 : HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX
4522 : (HasVLX ? X86::VMOVAPSZ256mr
4523 : HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX
4526 return Load ? (HasVLX ? X86::VMOVUPSZ256rm
4527 : HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX
4529 : (HasVLX ? X86::VMOVUPSZ256mr
4530 : HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX
4533 assert(X86::VR512RegClass.hasSubClassEq(RC) &&
"Unknown 64-byte regclass");
4536 return Load ? X86::VMOVAPSZrm : X86::VMOVAPSZmr;
4538 return Load ? X86::VMOVUPSZrm : X86::VMOVUPSZmr;
4540 assert(X86::TILERegClass.hasSubClassEq(RC) &&
"Unknown 1024-byte regclass");
4541 assert(STI.hasAMXTILE() &&
"Using 8*1024-bit register requires AMX-TILE");
4542#define GET_EGPR_IF_ENABLED(OPC) (STI.hasEGPR() ? OPC##_EVEX : OPC)
4545#undef GET_EGPR_IF_ENABLED
4547 assert(X86::TILEPAIRRegClass.hasSubClassEq(RC) &&
4548 "Unknown 2048-byte regclass");
4549 assert(STI.hasAMXTILE() &&
"Using 2048-bit register requires AMX-TILE");
4550 return Load ? X86::PTILEPAIRLOAD : X86::PTILEPAIRSTORE;
4554std::optional<ExtAddrMode>
4559 if (MemRefBegin < 0)
4560 return std::nullopt;
4565 if (!BaseOp.isReg())
4566 return std::nullopt;
4570 if (!DispMO.
isImm())
4571 return std::nullopt;
4597 ErrInfo =
"Scale factor in address must be 1, 2, 4 or 8";
4602 ErrInfo =
"Displacement in address must fit into 32-bit signed "
4612 int64_t &ImmVal)
const {
4618 if (
MI.isSubregToReg()) {
4622 if (!
MI.getOperand(1).isImm())
4624 unsigned FillBits =
MI.getOperand(1).getImm();
4625 unsigned SubIdx =
MI.getOperand(3).getImm();
4626 MovReg =
MI.getOperand(2).getReg();
4627 if (SubIdx != X86::sub_32bit || FillBits != 0)
4630 MovMI =
MRI.getUniqueVRegDef(MovReg);
4635 if (MovMI->
getOpcode() == X86::MOV32r0 &&
4641 if (MovMI->
getOpcode() != X86::MOV32ri &&
4655 if (!
MI->modifiesRegister(NullValueReg,
TRI))
4657 switch (
MI->getOpcode()) {
4664 assert(
MI->getOperand(0).isDef() &&
MI->getOperand(1).isUse() &&
4665 "expected for shift opcode!");
4666 return MI->getOperand(0).getReg() == NullValueReg &&
4667 MI->getOperand(1).getReg() == NullValueReg;
4672 return TRI->isSubRegisterEq(NullValueReg, MO.getReg());
4686 if (MemRefBegin < 0)
4693 if (!BaseOp->
isReg())
4706 if (!DispMO.
isImm())
4711 if (!BaseOp->
isReg())
4714 OffsetIsScalable =
false;
4718 Width = !
MemOp.memoperands_empty() ?
MemOp.memoperands().front()->getSize()
4726 bool IsStackAligned,
4741 case X86::TILELOADD:
4742 case X86::TILESTORED:
4743 case X86::TILELOADD_EVEX:
4744 case X86::TILESTORED_EVEX:
4745 case X86::PTILEPAIRLOAD:
4746 case X86::PTILEPAIRSTORE:
4754 bool isKill)
const {
4758 case X86::TILESTORED:
4759 case X86::TILESTORED_EVEX:
4760 case X86::PTILEPAIRSTORE: {
4763 Register VirtReg = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass);
4773 case X86::TILELOADD:
4774 case X86::TILELOADD_EVEX:
4775 case X86::PTILEPAIRLOAD: {
4778 Register VirtReg = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass);
4798 "Stack slot too small for store");
4800 unsigned Alignment = std::max<uint32_t>(
TRI->getSpillSize(*RC), 16);
4802 (Subtarget.getFrameLowering()->getStackAlign() >= Alignment) ||
4821 "Load size exceeds stack slot");
4822 unsigned Alignment = std::max<uint32_t>(
TRI->getSpillSize(*RC), 16);
4824 (Subtarget.getFrameLowering()->getStackAlign() >= Alignment) ||
4836 Register &SrcReg2, int64_t &CmpMask,
4837 int64_t &CmpValue)
const {
4838 switch (
MI.getOpcode()) {
4841 case X86::CMP64ri32:
4845 SrcReg =
MI.getOperand(0).getReg();
4847 if (
MI.getOperand(1).isImm()) {
4849 CmpValue =
MI.getOperand(1).getImm();
4851 CmpMask = CmpValue = 0;
4859 SrcReg =
MI.getOperand(1).getReg();
4868 SrcReg =
MI.getOperand(1).getReg();
4869 SrcReg2 =
MI.getOperand(2).getReg();
4877 SrcReg =
MI.getOperand(1).getReg();
4879 if (
MI.getOperand(2).isImm()) {
4881 CmpValue =
MI.getOperand(2).getImm();
4883 CmpMask = CmpValue = 0;
4890 SrcReg =
MI.getOperand(0).getReg();
4891 SrcReg2 =
MI.getOperand(1).getReg();
4899 SrcReg =
MI.getOperand(0).getReg();
4900 if (
MI.getOperand(1).getReg() != SrcReg)
4907 case X86::TEST64ri32:
4911 SrcReg =
MI.getOperand(0).getReg();
4921bool X86InstrInfo::isRedundantFlagInstr(
const MachineInstr &FlagI,
4923 int64_t ImmMask, int64_t ImmValue,
4925 int64_t *ImmDelta)
const {
4940 OIMask != ImmMask || OIValue != ImmValue)
4942 if (SrcReg == OISrcReg && SrcReg2 == OISrcReg2) {
4946 if (SrcReg == OISrcReg2 && SrcReg2 == OISrcReg) {
4952 case X86::CMP64ri32:
4956 case X86::TEST64ri32:
4967 case X86::TEST8rr: {
4974 SrcReg == OISrcReg && ImmMask == OIMask) {
4975 if (OIValue == ImmValue) {
4978 }
else if (
static_cast<uint64_t
>(ImmValue) ==
4979 static_cast<uint64_t
>(OIValue) - 1) {
4982 }
else if (
static_cast<uint64_t
>(ImmValue) ==
4983 static_cast<uint64_t
>(OIValue) + 1) {
5001 bool &ClearsOverflowFlag) {
5003 ClearsOverflowFlag =
false;
5009 if (
MI.getOpcode() == X86::ADD64rm ||
MI.getOpcode() == X86::ADD32rm) {
5010 unsigned Flags =
MI.getOperand(5).getTargetFlags();
5016 switch (
MI.getOpcode()) {
5112 case X86::LZCNT16rr:
5113 case X86::LZCNT16rm:
5114 case X86::LZCNT32rr:
5115 case X86::LZCNT32rm:
5116 case X86::LZCNT64rr:
5117 case X86::LZCNT64rm:
5118 case X86::POPCNT16rr:
5119 case X86::POPCNT16rm:
5120 case X86::POPCNT32rr:
5121 case X86::POPCNT32rm:
5122 case X86::POPCNT64rr:
5123 case X86::POPCNT64rm:
5124 case X86::TZCNT16rr:
5125 case X86::TZCNT16rm:
5126 case X86::TZCNT32rr:
5127 case X86::TZCNT32rm:
5128 case X86::TZCNT64rr:
5129 case X86::TZCNT64rm:
5175 case X86::BLSMSK32rr:
5176 case X86::BLSMSK32rm:
5177 case X86::BLSMSK64rr:
5178 case X86::BLSMSK64rm:
5183 case X86::BLCFILL32rr:
5184 case X86::BLCFILL32rm:
5185 case X86::BLCFILL64rr:
5186 case X86::BLCFILL64rm:
5191 case X86::BLCIC32rr:
5192 case X86::BLCIC32rm:
5193 case X86::BLCIC64rr:
5194 case X86::BLCIC64rm:
5195 case X86::BLCMSK32rr:
5196 case X86::BLCMSK32rm:
5197 case X86::BLCMSK64rr:
5198 case X86::BLCMSK64rm:
5203 case X86::BLSFILL32rr:
5204 case X86::BLSFILL32rm:
5205 case X86::BLSFILL64rr:
5206 case X86::BLSFILL64rm:
5207 case X86::BLSIC32rr:
5208 case X86::BLSIC32rm:
5209 case X86::BLSIC64rr:
5210 case X86::BLSIC64rm:
5215 case X86::T1MSKC32rr:
5216 case X86::T1MSKC32rm:
5217 case X86::T1MSKC64rr:
5218 case X86::T1MSKC64rm:
5219 case X86::TZMSK32rr:
5220 case X86::TZMSK32rm:
5221 case X86::TZMSK64rr:
5222 case X86::TZMSK64rm:
5226 ClearsOverflowFlag =
true;
5228 case X86::BEXTR32rr:
5229 case X86::BEXTR64rr:
5230 case X86::BEXTR32rm:
5231 case X86::BEXTR64rm:
5232 case X86::BEXTRI32ri:
5233 case X86::BEXTRI32mi:
5234 case X86::BEXTRI64ri:
5235 case X86::BEXTRI64mi:
5246 switch (
MI.getOpcode()) {
5254 case X86::LZCNT16rr:
5255 case X86::LZCNT32rr:
5256 case X86::LZCNT64rr:
5258 case X86::POPCNT16rr:
5259 case X86::POPCNT32rr:
5260 case X86::POPCNT64rr:
5262 case X86::TZCNT16rr:
5263 case X86::TZCNT32rr:
5264 case X86::TZCNT64rr:
5278 case X86::BLSMSK32rr:
5279 case X86::BLSMSK64rr:
5311 unsigned NewOpcode = 0;
5312#define FROM_TO(A, B) \
5313 CASE_ND(A) NewOpcode = X86::B; \
5337 if (NewOpcode == X86::CMP64rm || NewOpcode == X86::CMP32rm ||
5338 NewOpcode == X86::CMP16rm || NewOpcode == X86::CMP8rm)
5346 bool IsCmpZero = (CmpMask != 0 && CmpValue == 0);
5352 assert(SrcRegDef &&
"Must have a definition (SSA)");
5358 bool NoSignFlag =
false;
5359 bool ClearsOverflowFlag =
false;
5360 bool ShouldUpdateCC =
false;
5361 bool IsSwapped =
false;
5362 bool HasNF = Subtarget.hasNF();
5365 int64_t ImmDelta = 0;
5378 if (&Inst == SrcRegDef) {
5401 Subtarget, NoSignFlag, ClearsOverflowFlag)) {
5410 if (Inst.modifiesRegister(X86::EFLAGS,
TRI)) {
5421 Inst.getOperand(OpNo).getReg() == SrcReg) {
5422 ShouldUpdateCC =
true;
5433 if (isRedundantFlagInstr(CmpInstr, SrcReg, SrcReg2, CmpMask, CmpValue,
5434 Inst, &IsSwapped, &ImmDelta)) {
5442 if (!Movr0Inst && Inst.
getOpcode() == X86::MOV32r0 &&
5443 Inst.registerDefIsDead(X86::EFLAGS,
TRI)) {
5457 if (HasNF && Inst.registerDefIsDead(X86::EFLAGS,
TRI) && !IsWithReloc) {
5462 InstsToUpdate.
push_back(std::make_pair(&Inst, NewOp));
5476 if (
MBB->pred_size() != 1)
5478 MBB = *
MBB->pred_begin();
5479 From =
MBB->rbegin();
5486 bool FlagsMayLiveOut =
true;
5491 bool ModifyEFLAGS = Instr.modifiesRegister(X86::EFLAGS,
TRI);
5492 bool UseEFLAGS = Instr.readsRegister(X86::EFLAGS,
TRI);
5494 if (!UseEFLAGS && ModifyEFLAGS) {
5496 FlagsMayLiveOut =
false;
5499 if (!UseEFLAGS && !ModifyEFLAGS)
5530 if (!ClearsOverflowFlag)
5549 ReplacementCC = NewCC;
5555 }
else if (IsSwapped) {
5562 ShouldUpdateCC =
true;
5563 }
else if (ImmDelta != 0) {
5564 unsigned BitWidth =
TRI->getRegSizeInBits(*
MRI->getRegClass(SrcReg));
5574 if (ImmDelta != 1 || CmpValue == 0)
5584 if (ImmDelta != 1 || CmpValue == 0)
5611 ShouldUpdateCC =
true;
5614 if (ShouldUpdateCC && ReplacementCC != OldCC) {
5618 OpsToUpdate.
push_back(std::make_pair(&Instr, ReplacementCC));
5620 if (ModifyEFLAGS || Instr.killsRegister(X86::EFLAGS,
TRI)) {
5622 FlagsMayLiveOut =
false;
5629 if ((
MI !=
nullptr || ShouldUpdateCC) && FlagsMayLiveOut) {
5636 assert((
MI ==
nullptr ||
Sub ==
nullptr) &&
"Should not have Sub and MI set");
5643 if (&CmpMBB != SubBB)
5647 InsertE =
Sub->getParent()->rend();
5648 for (; InsertI != InsertE; ++InsertI) {
5650 if (!Instr->readsRegister(X86::EFLAGS,
TRI) &&
5651 Instr->modifiesRegister(X86::EFLAGS,
TRI)) {
5658 if (InsertI == InsertE)
5663 for (
auto &Inst : InstsToUpdate) {
5664 Inst.first->setDesc(
get(Inst.second));
5665 Inst.first->removeOperand(
5666 Inst.first->findRegisterDefOperandIdx(X86::EFLAGS,
nullptr));
5671 Sub->findRegisterDefOperand(X86::EFLAGS,
nullptr);
5672 assert(FlagDef &&
"Unable to locate a def EFLAGS operand");
5678 for (
auto &
Op : OpsToUpdate) {
5679 Op.first->getOperand(
Op.first->getDesc().getNumOperands() - 1)
5684 MBB = *
MBB->pred_begin()) {
5685 assert(
MBB->pred_size() == 1 &&
"Expected exactly one predecessor");
5686 if (!
MBB->isLiveIn(X86::EFLAGS))
5687 MBB->addLiveIn(X86::EFLAGS);
5715#define FROM_TO(FROM, TO) \
5718 case X86::FROM##_ND: \
5719 return X86::TO##_ND;
5749#define FROM_TO(FROM, TO) \
5753 FROM_TO(CTEST64rr, CTEST64ri32)
5772 bool MakeChange)
const {
5778 const TargetRegisterClass *RC =
nullptr;
5780 RC =
MRI->getRegClass(
Reg);
5782 (
Reg.
isVirtual() && X86::GR64RegClass.hasSubClassEq(RC))) {
5787 if (
UseMI.findRegisterUseOperand(
Reg,
nullptr)->getSubReg())
5792 !
MRI->hasOneNonDBGUse(
Reg))
5797 if (
Opc == TargetOpcode::COPY) {
5799 const TargetRegisterClass *RC =
nullptr;
5801 RC =
MRI->getRegClass(ToReg);
5802 bool GR32Reg = (ToReg.
isVirtual() && X86::GR32RegClass.hasSubClassEq(RC)) ||
5804 bool GR64Reg = (ToReg.
isVirtual() && X86::GR64RegClass.hasSubClassEq(RC)) ||
5806 bool GR8Reg = (ToReg.
isVirtual() && X86::GR8RegClass.hasSubClassEq(RC)) ||
5817 NewOpc = X86::MOV32ri64;
5819 NewOpc = X86::MOV64ri;
5820 }
else if (GR32Reg) {
5821 NewOpc = X86::MOV32ri;
5825 if (
UseMI.getParent()->computeRegisterLiveness(
5834 UseMI.removeOperand(
5835 UseMI.findRegisterUseOperandIdx(
Reg,
nullptr));
5843 NewOpc = X86::MOV8ri;
5853 if ((NewOpc == X86::SUB64ri32 || NewOpc == X86::SUB32ri ||
5854 NewOpc == X86::SBB64ri32 || NewOpc == X86::SBB32ri ||
5855 NewOpc == X86::SUB64ri32_ND || NewOpc == X86::SUB32ri_ND ||
5856 NewOpc == X86::SBB64ri32_ND || NewOpc == X86::SBB32ri_ND) &&
5857 UseMI.findRegisterUseOperandIdx(
Reg,
nullptr) != 2)
5860 if (((NewOpc == X86::CMP64ri32 || NewOpc == X86::CMP32ri) ||
5861 (NewOpc == X86::CCMP64ri32 || NewOpc == X86::CCMP32ri)) &&
5862 UseMI.findRegisterUseOperandIdx(
Reg,
nullptr) != 1)
5865 using namespace X86;
5866 if (isSHL(
Opc) || isSHR(
Opc) || isSAR(
Opc) || isROL(
Opc) || isROR(
Opc) ||
5867 isRCL(
Opc) || isRCR(
Opc)) {
5868 unsigned RegIdx =
UseMI.findRegisterUseOperandIdx(
Reg,
nullptr);
5878 UseMI.removeOperand(RegIdx);
5892 UseMI.registerDefIsDead(X86::EFLAGS,
nullptr)) {
5896 UseMI.setDesc(
get(TargetOpcode::COPY));
5897 UseMI.removeOperand(
5898 UseMI.findRegisterUseOperandIdx(
Reg,
nullptr));
5899 UseMI.removeOperand(
5900 UseMI.findRegisterDefOperandIdx(X86::EFLAGS,
nullptr));
5901 UseMI.untieRegOperand(0);
5905 unsigned Op1 = 1, Op2 = CommuteAnyOperandIndex;
5906 unsigned ImmOpNum = 2;
5907 if (!
UseMI.getOperand(0).isDef()) {
5911 if (
Opc == TargetOpcode::COPY)
5915 commuteInstruction(
UseMI);
5919 UseMI.getOperand(ImmOpNum).ChangeToImmediate(ImmVal);
5937 return foldImmediateImpl(
UseMI, &
DefMI, Reg, ImmVal,
MRI,
true);
5949 assert(
Desc.getNumOperands() == 3 &&
"Expected two-addr instruction.");
5969 assert(
Desc.getNumOperands() == 3 &&
"Expected two-addr instruction.");
5987 MIB->
setDesc(
TII.get(MinusOne ? X86::DEC32r : X86::INC32r));
5999 assert(Imm != 0 &&
"Using push/pop for 0 is not efficient.");
6002 int StackAdjustment;
6004 if (Subtarget.is64Bit()) {
6006 MIB->
getOpcode() == X86::MOV32ImmSExti8);
6020 StackAdjustment = 8;
6026 StackAdjustment = 4;
6038 bool EmitCFI = !TFL->
hasFP(MF) && NeedsDwarfCFI;
6085 MIB->
getOpcode() == X86::XOR64_FP ? X86::XOR64rr : X86::XOR32rr;
6097 const MCInstrDesc &BroadcastDesc,
unsigned SubIdx) {
6100 if (
TRI->getEncodingValue(DestReg) < 16) {
6107 DestReg =
TRI->getMatchingSuperReg(DestReg, SubIdx, &X86::VR512RegClass);
6119 const MCInstrDesc &ExtractDesc,
unsigned SubIdx) {
6122 if (
TRI->getEncodingValue(SrcReg) < 16) {
6129 SrcReg =
TRI->getMatchingSuperReg(SrcReg, SubIdx, &X86::VR512RegClass);
6152 if (
MI.getOpcode() == X86::MOVSHPrm) {
6153 NewOpc = HasAVX ? X86::VMOVSSrm : X86::MOVSSrm;
6155 if (
Reg > X86::XMM15)
6156 NewOpc = X86::VMOVSSZrm;
6158 NewOpc = HasAVX ? X86::VMOVSSmr : X86::MOVSSmr;
6160 if (
Reg > X86::XMM15)
6161 NewOpc = X86::VMOVSSZmr;
6169 bool HasAVX = Subtarget.hasAVX();
6171 switch (
MI.getOpcode()) {
6178 case X86::MOV32ImmSExti8:
6179 case X86::MOV64ImmSExti8:
6181 case X86::SETB_C32r:
6183 case X86::SETB_C64r:
6191 case X86::FsFLD0F128:
6193 case X86::AVX_SET0: {
6194 assert(HasAVX &&
"AVX not supported");
6197 Register XReg =
TRI->getSubReg(SrcReg, X86::sub_xmm);
6203 case X86::AVX512_128_SET0:
6204 case X86::AVX512_FsFLD0SH:
6205 case X86::AVX512_FsFLD0SS:
6206 case X86::AVX512_FsFLD0SD:
6207 case X86::AVX512_FsFLD0F128: {
6208 bool HasVLX = Subtarget.hasVLX();
6211 if (HasVLX ||
TRI->getEncodingValue(SrcReg) < 16)
6213 get(HasVLX ? X86::VPXORDZ128rr : X86::VXORPSrr));
6216 TRI->getMatchingSuperReg(SrcReg, X86::sub_xmm, &X86::VR512RegClass);
6220 case X86::AVX512_256_SET0:
6221 case X86::AVX512_512_SET0: {
6222 bool HasVLX = Subtarget.hasVLX();
6225 if (HasVLX ||
TRI->getEncodingValue(SrcReg) < 16) {
6226 Register XReg =
TRI->getSubReg(SrcReg, X86::sub_xmm);
6232 if (
MI.getOpcode() == X86::AVX512_256_SET0) {
6235 TRI->getMatchingSuperReg(SrcReg, X86::sub_ymm, &X86::VR512RegClass);
6243 case X86::V_SETALLONES:
6245 get(HasAVX ? X86::VPCMPEQDrr : X86::PCMPEQDrr));
6246 case X86::AVX2_SETALLONES:
6248 case X86::AVX1_SETALLONES: {
6255 case X86::AVX512_512_SETALLONES: {
6266 case X86::AVX512_512_SEXT_MASK_32:
6267 case X86::AVX512_512_SEXT_MASK_64: {
6271 unsigned Opc = (
MI.getOpcode() == X86::AVX512_512_SEXT_MASK_64)
6272 ? X86::VPTERNLOGQZrrikz
6273 : X86::VPTERNLOGDZrrikz;
6274 MI.removeOperand(1);
6279 .
addReg(MaskReg, MaskState)
6285 case X86::VMOVAPSZ128rm_NOVLX:
6287 get(X86::VBROADCASTF32X4Zrm), X86::sub_xmm);
6288 case X86::VMOVUPSZ128rm_NOVLX:
6290 get(X86::VBROADCASTF32X4Zrm), X86::sub_xmm);
6291 case X86::VMOVAPSZ256rm_NOVLX:
6293 get(X86::VBROADCASTF64X4Zrm), X86::sub_ymm);
6294 case X86::VMOVUPSZ256rm_NOVLX:
6296 get(X86::VBROADCASTF64X4Zrm), X86::sub_ymm);
6297 case X86::VMOVAPSZ128mr_NOVLX:
6299 get(X86::VEXTRACTF32X4Zmri), X86::sub_xmm);
6300 case X86::VMOVUPSZ128mr_NOVLX:
6302 get(X86::VEXTRACTF32X4Zmri), X86::sub_xmm);
6303 case X86::VMOVAPSZ256mr_NOVLX:
6305 get(X86::VEXTRACTF64X4Zmri), X86::sub_ymm);
6306 case X86::VMOVUPSZ256mr_NOVLX:
6308 get(X86::VEXTRACTF64X4Zmri), X86::sub_ymm);
6309 case X86::MOV32ri64: {
6311 Register Reg32 = RI.getSubReg(Reg, X86::sub_32bit);
6312 MI.setDesc(
get(X86::MOV32ri));
6318 case X86::RDFLAGS32:
6319 case X86::RDFLAGS64: {
6320 unsigned Is64Bit =
MI.getOpcode() == X86::RDFLAGS64;
6324 get(Is64Bit ? X86::PUSHF64 : X86::PUSHF32))
6332 "Unexpected register in operand! Should be EFLAGS.");
6335 "Unexpected register in operand! Should be DF.");
6338 MIB->
setDesc(
get(Is64Bit ? X86::POP64r : X86::POP32r));
6342 case X86::WRFLAGS32:
6343 case X86::WRFLAGS64: {
6344 unsigned Is64Bit =
MI.getOpcode() == X86::WRFLAGS64;
6348 get(Is64Bit ? X86::PUSH64r : X86::PUSH32r))
6349 .
addReg(
MI.getOperand(0).getReg());
6351 get(Is64Bit ? X86::POPF64 : X86::POPF32));
6352 MI.eraseFromParent();
6375 case TargetOpcode::LOAD_STACK_GUARD:
6381 case X86::SHLDROT32ri:
6383 case X86::SHLDROT64ri:
6385 case X86::SHRDROT32ri:
6387 case X86::SHRDROT64ri:
6389 case X86::ADD8rr_DB:
6392 case X86::ADD16rr_DB:
6395 case X86::ADD32rr_DB:
6398 case X86::ADD64rr_DB:
6401 case X86::ADD8ri_DB:
6404 case X86::ADD16ri_DB:
6407 case X86::ADD32ri_DB:
6410 case X86::ADD64ri32_DB:
6434 bool ForLoadFold =
false) {
6436 case X86::CVTSI2SSrr:
6437 case X86::CVTSI2SSrm:
6438 case X86::CVTSI642SSrr:
6439 case X86::CVTSI642SSrm:
6440 case X86::CVTSI2SDrr:
6441 case X86::CVTSI2SDrm:
6442 case X86::CVTSI642SDrr:
6443 case X86::CVTSI642SDrm:
6446 return !ForLoadFold;
6447 case X86::CVTSD2SSrr:
6448 case X86::CVTSD2SSrm:
6449 case X86::CVTSS2SDrr:
6450 case X86::CVTSS2SDrm:
6457 case X86::RCPSSr_Int:
6458 case X86::RCPSSm_Int:
6459 case X86::ROUNDSDri:
6460 case X86::ROUNDSDmi:
6461 case X86::ROUNDSSri:
6462 case X86::ROUNDSSmi:
6465 case X86::RSQRTSSr_Int:
6466 case X86::RSQRTSSm_Int:
6469 case X86::SQRTSSr_Int:
6470 case X86::SQRTSSm_Int:
6473 case X86::SQRTSDr_Int:
6474 case X86::SQRTSDm_Int:
6476 case X86::VFCMULCPHZ128rm:
6477 case X86::VFCMULCPHZ128rmb:
6478 case X86::VFCMULCPHZ128rmbkz:
6479 case X86::VFCMULCPHZ128rmkz:
6480 case X86::VFCMULCPHZ128rr:
6481 case X86::VFCMULCPHZ128rrkz:
6482 case X86::VFCMULCPHZ256rm:
6483 case X86::VFCMULCPHZ256rmb:
6484 case X86::VFCMULCPHZ256rmbkz:
6485 case X86::VFCMULCPHZ256rmkz:
6486 case X86::VFCMULCPHZ256rr:
6487 case X86::VFCMULCPHZ256rrkz:
6488 case X86::VFCMULCPHZrm:
6489 case X86::VFCMULCPHZrmb:
6490 case X86::VFCMULCPHZrmbkz:
6491 case X86::VFCMULCPHZrmkz:
6492 case X86::VFCMULCPHZrr:
6493 case X86::VFCMULCPHZrrb:
6494 case X86::VFCMULCPHZrrbkz:
6495 case X86::VFCMULCPHZrrkz:
6496 case X86::VFMULCPHZ128rm:
6497 case X86::VFMULCPHZ128rmb:
6498 case X86::VFMULCPHZ128rmbkz:
6499 case X86::VFMULCPHZ128rmkz:
6500 case X86::VFMULCPHZ128rr:
6501 case X86::VFMULCPHZ128rrkz:
6502 case X86::VFMULCPHZ256rm:
6503 case X86::VFMULCPHZ256rmb:
6504 case X86::VFMULCPHZ256rmbkz:
6505 case X86::VFMULCPHZ256rmkz:
6506 case X86::VFMULCPHZ256rr:
6507 case X86::VFMULCPHZ256rrkz:
6508 case X86::VFMULCPHZrm:
6509 case X86::VFMULCPHZrmb:
6510 case X86::VFMULCPHZrmbkz:
6511 case X86::VFMULCPHZrmkz:
6512 case X86::VFMULCPHZrr:
6513 case X86::VFMULCPHZrrb:
6514 case X86::VFMULCPHZrrbkz:
6515 case X86::VFMULCPHZrrkz:
6516 case X86::VFCMULCSHZrm:
6517 case X86::VFCMULCSHZrmkz:
6518 case X86::VFCMULCSHZrr:
6519 case X86::VFCMULCSHZrrb:
6520 case X86::VFCMULCSHZrrbkz:
6521 case X86::VFCMULCSHZrrkz:
6522 case X86::VFMULCSHZrm:
6523 case X86::VFMULCSHZrmkz:
6524 case X86::VFMULCSHZrr:
6525 case X86::VFMULCSHZrrb:
6526 case X86::VFMULCSHZrrbkz:
6527 case X86::VFMULCSHZrrkz:
6528 return Subtarget.hasMULCFalseDeps();
6529 case X86::VPERMDYrm:
6530 case X86::VPERMDYrr:
6531 case X86::VPERMQYmi:
6532 case X86::VPERMQYri:
6533 case X86::VPERMPSYrm:
6534 case X86::VPERMPSYrr:
6535 case X86::VPERMPDYmi:
6536 case X86::VPERMPDYri:
6537 case X86::VPERMDZ256rm:
6538 case X86::VPERMDZ256rmb:
6539 case X86::VPERMDZ256rmbkz:
6540 case X86::VPERMDZ256rmkz:
6541 case X86::VPERMDZ256rr:
6542 case X86::VPERMDZ256rrkz:
6543 case X86::VPERMDZrm:
6544 case X86::VPERMDZrmb:
6545 case X86::VPERMDZrmbkz:
6546 case X86::VPERMDZrmkz:
6547 case X86::VPERMDZrr:
6548 case X86::VPERMDZrrkz:
6549 case X86::VPERMQZ256mbi:
6550 case X86::VPERMQZ256mbikz:
6551 case X86::VPERMQZ256mi:
6552 case X86::VPERMQZ256mikz:
6553 case X86::VPERMQZ256ri:
6554 case X86::VPERMQZ256rikz:
6555 case X86::VPERMQZ256rm:
6556 case X86::VPERMQZ256rmb:
6557 case X86::VPERMQZ256rmbkz:
6558 case X86::VPERMQZ256rmkz:
6559 case X86::VPERMQZ256rr:
6560 case X86::VPERMQZ256rrkz:
6561 case X86::VPERMQZmbi:
6562 case X86::VPERMQZmbikz:
6563 case X86::VPERMQZmi:
6564 case X86::VPERMQZmikz:
6565 case X86::VPERMQZri:
6566 case X86::VPERMQZrikz:
6567 case X86::VPERMQZrm:
6568 case X86::VPERMQZrmb:
6569 case X86::VPERMQZrmbkz:
6570 case X86::VPERMQZrmkz:
6571 case X86::VPERMQZrr:
6572 case X86::VPERMQZrrkz:
6573 case X86::VPERMPSZ256rm:
6574 case X86::VPERMPSZ256rmb:
6575 case X86::VPERMPSZ256rmbkz:
6576 case X86::VPERMPSZ256rmkz:
6577 case X86::VPERMPSZ256rr:
6578 case X86::VPERMPSZ256rrkz:
6579 case X86::VPERMPSZrm:
6580 case X86::VPERMPSZrmb:
6581 case X86::VPERMPSZrmbkz:
6582 case X86::VPERMPSZrmkz:
6583 case X86::VPERMPSZrr:
6584 case X86::VPERMPSZrrkz:
6585 case X86::VPERMPDZ256mbi:
6586 case X86::VPERMPDZ256mbikz:
6587 case X86::VPERMPDZ256mi:
6588 case X86::VPERMPDZ256mikz:
6589 case X86::VPERMPDZ256ri:
6590 case X86::VPERMPDZ256rikz:
6591 case X86::VPERMPDZ256rm:
6592 case X86::VPERMPDZ256rmb:
6593 case X86::VPERMPDZ256rmbkz:
6594 case X86::VPERMPDZ256rmkz:
6595 case X86::VPERMPDZ256rr:
6596 case X86::VPERMPDZ256rrkz:
6597 case X86::VPERMPDZmbi:
6598 case X86::VPERMPDZmbikz:
6599 case X86::VPERMPDZmi:
6600 case X86::VPERMPDZmikz:
6601 case X86::VPERMPDZri:
6602 case X86::VPERMPDZrikz:
6603 case X86::VPERMPDZrm:
6604 case X86::VPERMPDZrmb:
6605 case X86::VPERMPDZrmbkz:
6606 case X86::VPERMPDZrmkz:
6607 case X86::VPERMPDZrr:
6608 case X86::VPERMPDZrrkz:
6609 return Subtarget.hasPERMFalseDeps();
6610 case X86::VRANGEPDZ128rmbi:
6611 case X86::VRANGEPDZ128rmbikz:
6612 case X86::VRANGEPDZ128rmi:
6613 case X86::VRANGEPDZ128rmikz:
6614 case X86::VRANGEPDZ128rri:
6615 case X86::VRANGEPDZ128rrikz:
6616 case X86::VRANGEPDZ256rmbi:
6617 case X86::VRANGEPDZ256rmbikz:
6618 case X86::VRANGEPDZ256rmi:
6619 case X86::VRANGEPDZ256rmikz:
6620 case X86::VRANGEPDZ256rri:
6621 case X86::VRANGEPDZ256rrikz:
6622 case X86::VRANGEPDZrmbi:
6623 case X86::VRANGEPDZrmbikz:
6624 case X86::VRANGEPDZrmi:
6625 case X86::VRANGEPDZrmikz:
6626 case X86::VRANGEPDZrri:
6627 case X86::VRANGEPDZrrib:
6628 case X86::VRANGEPDZrribkz:
6629 case X86::VRANGEPDZrrikz:
6630 case X86::VRANGEPSZ128rmbi:
6631 case X86::VRANGEPSZ128rmbikz:
6632 case X86::VRANGEPSZ128rmi:
6633 case X86::VRANGEPSZ128rmikz:
6634 case X86::VRANGEPSZ128rri:
6635 case X86::VRANGEPSZ128rrikz:
6636 case X86::VRANGEPSZ256rmbi:
6637 case X86::VRANGEPSZ256rmbikz:
6638 case X86::VRANGEPSZ256rmi:
6639 case X86::VRANGEPSZ256rmikz:
6640 case X86::VRANGEPSZ256rri:
6641 case X86::VRANGEPSZ256rrikz:
6642 case X86::VRANGEPSZrmbi:
6643 case X86::VRANGEPSZrmbikz:
6644 case X86::VRANGEPSZrmi:
6645 case X86::VRANGEPSZrmikz:
6646 case X86::VRANGEPSZrri:
6647 case X86::VRANGEPSZrrib:
6648 case X86::VRANGEPSZrribkz:
6649 case X86::VRANGEPSZrrikz:
6650 case X86::VRANGESDZrmi:
6651 case X86::VRANGESDZrmikz:
6652 case X86::VRANGESDZrri:
6653 case X86::VRANGESDZrrib:
6654 case X86::VRANGESDZrribkz:
6655 case X86::VRANGESDZrrikz:
6656 case X86::VRANGESSZrmi:
6657 case X86::VRANGESSZrmikz:
6658 case X86::VRANGESSZrri:
6659 case X86::VRANGESSZrrib:
6660 case X86::VRANGESSZrribkz:
6661 case X86::VRANGESSZrrikz:
6662 return Subtarget.hasRANGEFalseDeps();
6663 case X86::VGETMANTSSZrmi:
6664 case X86::VGETMANTSSZrmikz:
6665 case X86::VGETMANTSSZrri:
6666 case X86::VGETMANTSSZrrib:
6667 case X86::VGETMANTSSZrribkz:
6668 case X86::VGETMANTSSZrrikz:
6669 case X86::VGETMANTSDZrmi:
6670 case X86::VGETMANTSDZrmikz:
6671 case X86::VGETMANTSDZrri:
6672 case X86::VGETMANTSDZrrib:
6673 case X86::VGETMANTSDZrribkz:
6674 case X86::VGETMANTSDZrrikz:
6675 case X86::VGETMANTSHZrmi:
6676 case X86::VGETMANTSHZrmikz:
6677 case X86::VGETMANTSHZrri:
6678 case X86::VGETMANTSHZrrib:
6679 case X86::VGETMANTSHZrribkz:
6680 case X86::VGETMANTSHZrrikz:
6681 case X86::VGETMANTPSZ128rmbi:
6682 case X86::VGETMANTPSZ128rmbikz:
6683 case X86::VGETMANTPSZ128rmi:
6684 case X86::VGETMANTPSZ128rmikz:
6685 case X86::VGETMANTPSZ256rmbi:
6686 case X86::VGETMANTPSZ256rmbikz:
6687 case X86::VGETMANTPSZ256rmi:
6688 case X86::VGETMANTPSZ256rmikz:
6689 case X86::VGETMANTPSZrmbi:
6690 case X86::VGETMANTPSZrmbikz:
6691 case X86::VGETMANTPSZrmi:
6692 case X86::VGETMANTPSZrmikz:
6693 case X86::VGETMANTPDZ128rmbi:
6694 case X86::VGETMANTPDZ128rmbikz:
6695 case X86::VGETMANTPDZ128rmi:
6696 case X86::VGETMANTPDZ128rmikz:
6697 case X86::VGETMANTPDZ256rmbi:
6698 case X86::VGETMANTPDZ256rmbikz:
6699 case X86::VGETMANTPDZ256rmi:
6700 case X86::VGETMANTPDZ256rmikz:
6701 case X86::VGETMANTPDZrmbi:
6702 case X86::VGETMANTPDZrmbikz:
6703 case X86::VGETMANTPDZrmi:
6704 case X86::VGETMANTPDZrmikz:
6705 return Subtarget.hasGETMANTFalseDeps();
6706 case X86::VPMULLQZ128rm:
6707 case X86::VPMULLQZ128rmb:
6708 case X86::VPMULLQZ128rmbkz:
6709 case X86::VPMULLQZ128rmkz:
6710 case X86::VPMULLQZ128rr:
6711 case X86::VPMULLQZ128rrkz:
6712 case X86::VPMULLQZ256rm:
6713 case X86::VPMULLQZ256rmb:
6714 case X86::VPMULLQZ256rmbkz:
6715 case X86::VPMULLQZ256rmkz:
6716 case X86::VPMULLQZ256rr:
6717 case X86::VPMULLQZ256rrkz:
6718 case X86::VPMULLQZrm:
6719 case X86::VPMULLQZrmb:
6720 case X86::VPMULLQZrmbkz:
6721 case X86::VPMULLQZrmkz:
6722 case X86::VPMULLQZrr:
6723 case X86::VPMULLQZrrkz:
6724 return Subtarget.hasMULLQFalseDeps();
6726 case X86::POPCNT32rm:
6727 case X86::POPCNT32rr:
6728 case X86::POPCNT64rm:
6729 case X86::POPCNT64rr:
6730 return Subtarget.hasPOPCNTFalseDeps();
6731 case X86::LZCNT32rm:
6732 case X86::LZCNT32rr:
6733 case X86::LZCNT64rm:
6734 case X86::LZCNT64rr:
6735 case X86::TZCNT32rm:
6736 case X86::TZCNT32rr:
6737 case X86::TZCNT64rm:
6738 case X86::TZCNT64rr:
6739 return Subtarget.hasLZCNTFalseDeps();
6756 bool HasNDDPartialWrite =
false;
6759 if (!Reg.isVirtual())
6760 HasNDDPartialWrite =
6761 X86::GR8RegClass.contains(Reg) || X86::GR16RegClass.contains(Reg);
6774 bool ReadsReg =
false;
6775 if (Reg.isVirtual())
6776 ReadsReg = (MO.
readsReg() ||
MI.readsVirtualRegister(Reg));
6778 ReadsReg =
MI.readsRegister(Reg,
TRI);
6779 if (ReadsReg != HasNDDPartialWrite)
6793 bool ForLoadFold =
false) {
6796 case X86::MMX_PUNPCKHBWrr:
6797 case X86::MMX_PUNPCKHWDrr:
6798 case X86::MMX_PUNPCKHDQrr:
6799 case X86::MMX_PUNPCKLBWrr:
6800 case X86::MMX_PUNPCKLWDrr:
6801 case X86::MMX_PUNPCKLDQrr:
6802 case X86::MOVHLPSrr:
6803 case X86::PACKSSWBrr:
6804 case X86::PACKUSWBrr:
6805 case X86::PACKSSDWrr:
6806 case X86::PACKUSDWrr:
6807 case X86::PUNPCKHBWrr:
6808 case X86::PUNPCKLBWrr:
6809 case X86::PUNPCKHWDrr:
6810 case X86::PUNPCKLWDrr:
6811 case X86::PUNPCKHDQrr:
6812 case X86::PUNPCKLDQrr:
6813 case X86::PUNPCKHQDQrr:
6814 case X86::PUNPCKLQDQrr:
6815 case X86::SHUFPDrri:
6816 case X86::SHUFPSrri:
6822 return OpNum == 2 && !ForLoadFold;
6824 case X86::VMOVLHPSrr:
6825 case X86::VMOVLHPSZrr:
6826 case X86::VPACKSSWBrr:
6827 case X86::VPACKUSWBrr:
6828 case X86::VPACKSSDWrr:
6829 case X86::VPACKUSDWrr:
6830 case X86::VPACKSSWBZ128rr:
6831 case X86::VPACKUSWBZ128rr:
6832 case X86::VPACKSSDWZ128rr:
6833 case X86::VPACKUSDWZ128rr:
6834 case X86::VPERM2F128rri:
6835 case X86::VPERM2I128rri:
6836 case X86::VSHUFF32X4Z256rri:
6837 case X86::VSHUFF32X4Zrri:
6838 case X86::VSHUFF64X2Z256rri:
6839 case X86::VSHUFF64X2Zrri:
6840 case X86::VSHUFI32X4Z256rri:
6841 case X86::VSHUFI32X4Zrri:
6842 case X86::VSHUFI64X2Z256rri:
6843 case X86::VSHUFI64X2Zrri:
6844 case X86::VPUNPCKHBWrr:
6845 case X86::VPUNPCKLBWrr:
6846 case X86::VPUNPCKHBWYrr:
6847 case X86::VPUNPCKLBWYrr:
6848 case X86::VPUNPCKHBWZ128rr:
6849 case X86::VPUNPCKLBWZ128rr:
6850 case X86::VPUNPCKHBWZ256rr:
6851 case X86::VPUNPCKLBWZ256rr:
6852 case X86::VPUNPCKHBWZrr:
6853 case X86::VPUNPCKLBWZrr:
6854 case X86::VPUNPCKHWDrr:
6855 case X86::VPUNPCKLWDrr:
6856 case X86::VPUNPCKHWDYrr:
6857 case X86::VPUNPCKLWDYrr:
6858 case X86::VPUNPCKHWDZ128rr:
6859 case X86::VPUNPCKLWDZ128rr:
6860 case X86::VPUNPCKHWDZ256rr:
6861 case X86::VPUNPCKLWDZ256rr:
6862 case X86::VPUNPCKHWDZrr:
6863 case X86::VPUNPCKLWDZrr:
6864 case X86::VPUNPCKHDQrr:
6865 case X86::VPUNPCKLDQrr:
6866 case X86::VPUNPCKHDQYrr:
6867 case X86::VPUNPCKLDQYrr:
6868 case X86::VPUNPCKHDQZ128rr:
6869 case X86::VPUNPCKLDQZ128rr:
6870 case X86::VPUNPCKHDQZ256rr:
6871 case X86::VPUNPCKLDQZ256rr:
6872 case X86::VPUNPCKHDQZrr:
6873 case X86::VPUNPCKLDQZrr:
6874 case X86::VPUNPCKHQDQrr:
6875 case X86::VPUNPCKLQDQrr:
6876 case X86::VPUNPCKHQDQYrr:
6877 case X86::VPUNPCKLQDQYrr:
6878 case X86::VPUNPCKHQDQZ128rr:
6879 case X86::VPUNPCKLQDQZ128rr:
6880 case X86::VPUNPCKHQDQZ256rr:
6881 case X86::VPUNPCKLQDQZ256rr:
6882 case X86::VPUNPCKHQDQZrr:
6883 case X86::VPUNPCKLQDQZrr:
6887 return (OpNum == 1 || OpNum == 2) && !ForLoadFold;
6889 case X86::VCVTSI2SSrr:
6890 case X86::VCVTSI2SSrm:
6891 case X86::VCVTSI2SSrr_Int:
6892 case X86::VCVTSI2SSrm_Int:
6893 case X86::VCVTSI642SSrr:
6894 case X86::VCVTSI642SSrm:
6895 case X86::VCVTSI642SSrr_Int:
6896 case X86::VCVTSI642SSrm_Int:
6897 case X86::VCVTSI2SDrr:
6898 case X86::VCVTSI2SDrm:
6899 case X86::VCVTSI2SDrr_Int:
6900 case X86::VCVTSI2SDrm_Int:
6901 case X86::VCVTSI642SDrr:
6902 case X86::VCVTSI642SDrm:
6903 case X86::VCVTSI642SDrr_Int:
6904 case X86::VCVTSI642SDrm_Int:
6906 case X86::VCVTSI2SSZrr:
6907 case X86::VCVTSI2SSZrm:
6908 case X86::VCVTSI2SSZrr_Int:
6909 case X86::VCVTSI2SSZrrb_Int:
6910 case X86::VCVTSI2SSZrm_Int:
6911 case X86::VCVTSI642SSZrr:
6912 case X86::VCVTSI642SSZrm:
6913 case X86::VCVTSI642SSZrr_Int:
6914 case X86::VCVTSI642SSZrrb_Int:
6915 case X86::VCVTSI642SSZrm_Int:
6916 case X86::VCVTSI2SDZrr:
6917 case X86::VCVTSI2SDZrm:
6918 case X86::VCVTSI2SDZrr_Int:
6919 case X86::VCVTSI2SDZrm_Int:
6920 case X86::VCVTSI642SDZrr:
6921 case X86::VCVTSI642SDZrm:
6922 case X86::VCVTSI642SDZrr_Int:
6923 case X86::VCVTSI642SDZrrb_Int:
6924 case X86::VCVTSI642SDZrm_Int:
6925 case X86::VCVTUSI2SSZrr:
6926 case X86::VCVTUSI2SSZrm:
6927 case X86::VCVTUSI2SSZrr_Int:
6928 case X86::VCVTUSI2SSZrrb_Int:
6929 case X86::VCVTUSI2SSZrm_Int:
6930 case X86::VCVTUSI642SSZrr:
6931 case X86::VCVTUSI642SSZrm:
6932 case X86::VCVTUSI642SSZrr_Int:
6933 case X86::VCVTUSI642SSZrrb_Int:
6934 case X86::VCVTUSI642SSZrm_Int:
6935 case X86::VCVTUSI2SDZrr:
6936 case X86::VCVTUSI2SDZrm:
6937 case X86::VCVTUSI2SDZrr_Int:
6938 case X86::VCVTUSI2SDZrm_Int:
6939 case X86::VCVTUSI642SDZrr:
6940 case X86::VCVTUSI642SDZrm:
6941 case X86::VCVTUSI642SDZrr_Int:
6942 case X86::VCVTUSI642SDZrrb_Int:
6943 case X86::VCVTUSI642SDZrm_Int:
6944 case X86::VCVTSI2SHZrr:
6945 case X86::VCVTSI2SHZrm:
6946 case X86::VCVTSI2SHZrr_Int:
6947 case X86::VCVTSI2SHZrrb_Int:
6948 case X86::VCVTSI2SHZrm_Int:
6949 case X86::VCVTSI642SHZrr:
6950 case X86::VCVTSI642SHZrm:
6951 case X86::VCVTSI642SHZrr_Int:
6952 case X86::VCVTSI642SHZrrb_Int:
6953 case X86::VCVTSI642SHZrm_Int:
6954 case X86::VCVTUSI2SHZrr:
6955 case X86::VCVTUSI2SHZrm:
6956 case X86::VCVTUSI2SHZrr_Int:
6957 case X86::VCVTUSI2SHZrrb_Int:
6958 case X86::VCVTUSI2SHZrm_Int:
6959 case X86::VCVTUSI642SHZrr:
6960 case X86::VCVTUSI642SHZrm:
6961 case X86::VCVTUSI642SHZrr_Int:
6962 case X86::VCVTUSI642SHZrrb_Int:
6963 case X86::VCVTUSI642SHZrm_Int:
6966 return OpNum == 1 && !ForLoadFold;
6967 case X86::VCVTSD2SSrr:
6968 case X86::VCVTSD2SSrm:
6969 case X86::VCVTSD2SSrr_Int:
6970 case X86::VCVTSD2SSrm_Int:
6971 case X86::VCVTSS2SDrr:
6972 case X86::VCVTSS2SDrm:
6973 case X86::VCVTSS2SDrr_Int:
6974 case X86::VCVTSS2SDrm_Int:
6976 case X86::VRCPSSr_Int:
6978 case X86::VRCPSSm_Int:
6979 case X86::VROUNDSDri:
6980 case X86::VROUNDSDmi:
6981 case X86::VROUNDSDri_Int:
6982 case X86::VROUNDSDmi_Int:
6983 case X86::VROUNDSSri:
6984 case X86::VROUNDSSmi:
6985 case X86::VROUNDSSri_Int:
6986 case X86::VROUNDSSmi_Int:
6987 case X86::VRSQRTSSr:
6988 case X86::VRSQRTSSr_Int:
6989 case X86::VRSQRTSSm:
6990 case X86::VRSQRTSSm_Int:
6992 case X86::VSQRTSSr_Int:
6994 case X86::VSQRTSSm_Int:
6996 case X86::VSQRTSDr_Int:
6998 case X86::VSQRTSDm_Int:
7000 case X86::VCVTSD2SSZrr:
7001 case X86::VCVTSD2SSZrr_Int:
7002 case X86::VCVTSD2SSZrrb_Int:
7003 case X86::VCVTSD2SSZrm:
7004 case X86::VCVTSD2SSZrm_Int:
7005 case X86::VCVTSS2SDZrr:
7006 case X86::VCVTSS2SDZrr_Int:
7007 case X86::VCVTSS2SDZrrb_Int:
7008 case X86::VCVTSS2SDZrm:
7009 case X86::VCVTSS2SDZrm_Int:
7010 case X86::VGETEXPSDZr:
7011 case X86::VGETEXPSDZrb:
7012 case X86::VGETEXPSDZm:
7013 case X86::VGETEXPSSZr:
7014 case X86::VGETEXPSSZrb:
7015 case X86::VGETEXPSSZm:
7016 case X86::VGETMANTSDZrri:
7017 case X86::VGETMANTSDZrrib:
7018 case X86::VGETMANTSDZrmi:
7019 case X86::VGETMANTSSZrri:
7020 case X86::VGETMANTSSZrrib:
7021 case X86::VGETMANTSSZrmi:
7022 case X86::VRNDSCALESDZrri:
7023 case X86::VRNDSCALESDZrri_Int:
7024 case X86::VRNDSCALESDZrrib_Int:
7025 case X86::VRNDSCALESDZrmi:
7026 case X86::VRNDSCALESDZrmi_Int:
7027 case X86::VRNDSCALESSZrri:
7028 case X86::VRNDSCALESSZrri_Int:
7029 case X86::VRNDSCALESSZrrib_Int:
7030 case X86::VRNDSCALESSZrmi:
7031 case X86::VRNDSCALESSZrmi_Int:
7032 case X86::VRCP14SDZrr:
7033 case X86::VRCP14SDZrm:
7034 case X86::VRCP14SSZrr:
7035 case X86::VRCP14SSZrm:
7036 case X86::VRCPSHZrr:
7037 case X86::VRCPSHZrm:
7038 case X86::VRSQRTSHZrr:
7039 case X86::VRSQRTSHZrm:
7040 case X86::VREDUCESHZrmi:
7041 case X86::VREDUCESHZrri:
7042 case X86::VREDUCESHZrrib:
7043 case X86::VGETEXPSHZr:
7044 case X86::VGETEXPSHZrb:
7045 case X86::VGETEXPSHZm:
7046 case X86::VGETMANTSHZrri:
7047 case X86::VGETMANTSHZrrib:
7048 case X86::VGETMANTSHZrmi:
7049 case X86::VRNDSCALESHZrri:
7050 case X86::VRNDSCALESHZrri_Int:
7051 case X86::VRNDSCALESHZrrib_Int:
7052 case X86::VRNDSCALESHZrmi:
7053 case X86::VRNDSCALESHZrmi_Int:
7054 case X86::VSQRTSHZr:
7055 case X86::VSQRTSHZr_Int:
7056 case X86::VSQRTSHZrb_Int:
7057 case X86::VSQRTSHZm:
7058 case X86::VSQRTSHZm_Int:
7059 case X86::VRCP28SDZr:
7060 case X86::VRCP28SDZrb:
7061 case X86::VRCP28SDZm:
7062 case X86::VRCP28SSZr:
7063 case X86::VRCP28SSZrb:
7064 case X86::VRCP28SSZm:
7065 case X86::VREDUCESSZrmi:
7066 case X86::VREDUCESSZrri:
7067 case X86::VREDUCESSZrrib:
7068 case X86::VRSQRT14SDZrr:
7069 case X86::VRSQRT14SDZrm:
7070 case X86::VRSQRT14SSZrr:
7071 case X86::VRSQRT14SSZrm:
7072 case X86::VRSQRT28SDZr:
7073 case X86::VRSQRT28SDZrb:
7074 case X86::VRSQRT28SDZm:
7075 case X86::VRSQRT28SSZr:
7076 case X86::VRSQRT28SSZrb:
7077 case X86::VRSQRT28SSZm:
7078 case X86::VSQRTSSZr:
7079 case X86::VSQRTSSZr_Int:
7080 case X86::VSQRTSSZrb_Int:
7081 case X86::VSQRTSSZm:
7082 case X86::VSQRTSSZm_Int:
7083 case X86::VSQRTSDZr:
7084 case X86::VSQRTSDZr_Int:
7085 case X86::VSQRTSDZrb_Int:
7086 case X86::VSQRTSDZm:
7087 case X86::VSQRTSDZm_Int:
7088 case X86::VCVTSD2SHZrr:
7089 case X86::VCVTSD2SHZrr_Int:
7090 case X86::VCVTSD2SHZrrb_Int:
7091 case X86::VCVTSD2SHZrm:
7092 case X86::VCVTSD2SHZrm_Int:
7093 case X86::VCVTSS2SHZrr:
7094 case X86::VCVTSS2SHZrr_Int:
7095 case X86::VCVTSS2SHZrrb_Int:
7096 case X86::VCVTSS2SHZrm:
7097 case X86::VCVTSS2SHZrm_Int:
7098 case X86::VCVTSH2SDZrr:
7099 case X86::VCVTSH2SDZrr_Int:
7100 case X86::VCVTSH2SDZrrb_Int:
7101 case X86::VCVTSH2SDZrm:
7102 case X86::VCVTSH2SDZrm_Int:
7103 case X86::VCVTSH2SSZrr:
7104 case X86::VCVTSH2SSZrr_Int:
7105 case X86::VCVTSH2SSZrrb_Int:
7106 case X86::VCVTSH2SSZrm:
7107 case X86::VCVTSH2SSZrm_Int:
7109 case X86::VMOVSSZrrk:
7110 case X86::VMOVSDZrrk:
7111 return OpNum == 3 && !ForLoadFold;
7112 case X86::VMOVSSZrrkz:
7113 case X86::VMOVSDZrrkz:
7114 return OpNum == 2 && !ForLoadFold;
7146 Register Reg =
MI.getOperand(OpNum).getReg();
7148 if (
MI.killsRegister(Reg,
TRI))
7151 if (X86::VR128RegClass.
contains(Reg)) {
7154 unsigned Opc = Subtarget.hasAVX() ? X86::VXORPSrr : X86::XORPSrr;
7158 MI.addRegisterKilled(Reg,
TRI,
true);
7159 }
else if (X86::VR256RegClass.
contains(Reg)) {
7162 Register XReg =
TRI->getSubReg(Reg, X86::sub_xmm);
7167 MI.addRegisterKilled(Reg,
TRI,
true);
7168 }
else if (X86::VR128XRegClass.
contains(Reg)) {
7170 if (!Subtarget.hasVLX())
7173 BuildMI(*
MI.getParent(),
MI,
MI.getDebugLoc(),
get(X86::VPXORDZ128rr), Reg)
7176 MI.addRegisterKilled(Reg,
TRI,
true);
7177 }
else if (X86::VR256XRegClass.
contains(Reg) ||
7178 X86::VR512RegClass.
contains(Reg)) {
7180 if (!Subtarget.hasVLX())
7184 Register XReg =
TRI->getSubReg(Reg, X86::sub_xmm);
7185 BuildMI(*
MI.getParent(),
MI,
MI.getDebugLoc(),
get(X86::VPXORDZ128rr), XReg)
7189 MI.addRegisterKilled(Reg,
TRI,
true);
7190 }
else if (X86::GR64RegClass.
contains(Reg)) {
7193 Register XReg =
TRI->getSubReg(Reg, X86::sub_32bit);
7198 MI.addRegisterKilled(Reg,
TRI,
true);
7199 }
else if (X86::GR32RegClass.
contains(Reg)) {
7203 MI.addRegisterKilled(Reg,
TRI,
true);
7204 }
else if ((X86::GR16RegClass.
contains(Reg) ||
7213 if (!
MI.definesRegister(SuperReg,
nullptr))
7219 int PtrOffset = 0) {
7220 unsigned NumAddrOps = MOs.
size();
7222 if (NumAddrOps < 4) {
7224 for (
unsigned i = 0; i != NumAddrOps; ++i)
7230 assert(MOs.
size() == 5 &&
"Unexpected memory operand list length");
7231 for (
unsigned i = 0; i != NumAddrOps; ++i) {
7233 if (i == 3 && PtrOffset != 0) {
7254 if (!
Reg.isVirtual())
7261 dbgs() <<
"WARNING: Unable to update register constraint for operand "
7262 << Idx <<
" of instruction:\n";
7276 MF.CreateMachineInstr(
TII.get(Opcode),
MI.getDebugLoc(),
true);
7281 unsigned NumOps =
MI.getDesc().getNumOperands() - 2;
7282 for (
unsigned i = 0; i !=
NumOps; ++i) {
7292 MBB->insert(InsertPt, NewMI);
7301 int PtrOffset = 0) {
7304 MF.CreateMachineInstr(
TII.get(Opcode),
MI.getDebugLoc(),
true);
7307 for (
unsigned i = 0, e =
MI.getNumOperands(); i != e; ++i) {
7310 assert(MO.
isReg() &&
"Expected to fold into reg operand!");
7324 MBB->insert(InsertPt, NewMI);
7334 MI.getDebugLoc(),
TII.get(Opcode));
7343 switch (
MI.getOpcode()) {
7344 case X86::INSERTPSrri:
7345 case X86::VINSERTPSrri:
7346 case X86::VINSERTPSZrri:
7350 unsigned Imm =
MI.getOperand(
MI.getNumOperands() - 1).getImm();
7351 unsigned ZMask =
Imm & 15;
7352 unsigned DstIdx = (
Imm >> 4) & 3;
7353 unsigned SrcIdx = (
Imm >> 6) & 3;
7356 const TargetRegisterClass *RC =
getRegClass(
MI.getDesc(), OpNum, &RI);
7357 unsigned RCSize =
TRI.getRegSizeInBits(*RC) / 8;
7358 if ((
Size == 0 ||
Size >= 16) && RCSize >= 16 &&
7359 (
MI.getOpcode() != X86::INSERTPSrri || Alignment >=
Align(4))) {
7360 int PtrOffset = SrcIdx * 4;
7361 unsigned NewImm = (DstIdx << 4) | ZMask;
7362 unsigned NewOpCode =
7363 (
MI.getOpcode() == X86::VINSERTPSZrri) ? X86::VINSERTPSZrmi
7364 : (
MI.getOpcode() == X86::VINSERTPSrri) ? X86::VINSERTPSrmi
7366 MachineInstr *NewMI =
7367 fuseInst(MF, NewOpCode, OpNum, MOs, InsertPt,
MI, *
this, PtrOffset);
7373 case X86::MOVHLPSrr:
7374 case X86::VMOVHLPSrr:
7375 case X86::VMOVHLPSZrr:
7381 const TargetRegisterClass *RC =
getRegClass(
MI.getDesc(), OpNum, &RI);
7382 unsigned RCSize =
TRI.getRegSizeInBits(*RC) / 8;
7383 if ((
Size == 0 ||
Size >= 16) && RCSize >= 16 && Alignment >=
Align(8)) {
7384 unsigned NewOpCode =
7385 (
MI.getOpcode() == X86::VMOVHLPSZrr) ? X86::VMOVLPSZ128rm
7386 : (
MI.getOpcode() == X86::VMOVHLPSrr) ? X86::VMOVLPSrm
7388 MachineInstr *NewMI =
7389 fuseInst(MF, NewOpCode, OpNum, MOs, InsertPt,
MI, *
this, 8);
7394 case X86::UNPCKLPDrr:
7400 const TargetRegisterClass *RC =
getRegClass(
MI.getDesc(), OpNum, &RI);
7401 unsigned RCSize =
TRI.getRegSizeInBits(*RC) / 8;
7402 if ((
Size == 0 ||
Size >= 16) && RCSize >= 16 && Alignment <
Align(16)) {
7403 MachineInstr *NewMI =
7404 fuseInst(MF, X86::MOVHPDrm, OpNum, MOs, InsertPt,
MI, *
this);
7411 makeM0Inst(*
this, (
Size == 4) ? X86::MOV32mi : X86::MOV64mi32, MOs,
7423 !
MI.getOperand(1).isReg())
7431 if (
MI.getOperand(1).isUndef())
7440 unsigned Idx1)
const {
7441 unsigned Idx2 = CommuteAnyOperandIndex;
7445 bool HasDef =
MI.getDesc().getNumDefs();
7447 Register Reg1 =
MI.getOperand(Idx1).getReg();
7448 Register Reg2 =
MI.getOperand(Idx2).getReg();
7449 bool Tied1 = 0 ==
MI.getDesc().getOperandConstraint(Idx1,
MCOI::TIED_TO);
7450 bool Tied2 = 0 ==
MI.getDesc().getOperandConstraint(Idx2,
MCOI::TIED_TO);
7454 if ((HasDef && Reg0 == Reg1 && Tied1) || (HasDef && Reg0 == Reg2 && Tied2))
7457 return commuteInstruction(
MI,
false, Idx1, Idx2) ? Idx2 : Idx1;
7462 dbgs() <<
"We failed to fuse operand " << Idx <<
" in " <<
MI;
7468 unsigned Size,
Align Alignment,
bool AllowCommute)
const {
7469 bool isSlowTwoMemOps = Subtarget.slowTwoMemOps();
7470 unsigned Opc =
MI.getOpcode();
7476 (
Opc == X86::CALL32r ||
Opc == X86::CALL64r ||
7477 Opc == X86::CALL64r_ImpCall ||
Opc == X86::PUSH16r ||
7478 Opc == X86::PUSH32r ||
Opc == X86::PUSH64r))
7487 unsigned NumOps =
MI.getDesc().getNumOperands();
7488 bool IsTwoAddr =
NumOps > 1 && OpNum < 2 &&
MI.getOperand(0).isReg() &&
7489 MI.getOperand(1).isReg() &&
7490 MI.getOperand(0).getReg() ==
MI.getOperand(1).getReg();
7494 if (
Opc == X86::ADD32ri &&
7503 Opc != X86::ADD64rr)
7508 if (
MI.isCall() &&
MI.getCFIType())
7512 if (
auto *CustomMI = foldMemoryOperandCustom(MF,
MI, OpNum, MOs, InsertPt,
7528 unsigned Opcode =
I->DstOp;
7532 bool NarrowToMOV32rm =
false;
7536 unsigned RCSize =
TRI.getRegSizeInBits(*RC) / 8;
7544 if (Opcode != X86::MOV64rm || RCSize != 8 ||
Size != 4)
7546 if (
MI.getOperand(0).getSubReg() ||
MI.getOperand(1).getSubReg())
7548 Opcode = X86::MOV32rm;
7549 NarrowToMOV32rm =
true;
7559 :
fuseInst(MF, Opcode, OpNum, MOs, InsertPt,
MI, *
this);
7561 if (NarrowToMOV32rm) {
7577 unsigned CommuteOpIdx2 = commuteOperandsForFold(
MI, OpNum);
7578 if (CommuteOpIdx2 == OpNum) {
7588 commuteInstruction(
MI,
false, OpNum, CommuteOpIdx2);
7610 for (
auto Op :
Ops) {
7615 if (
MI.getOpcode() == X86::MOV32r0 &&
SubReg == X86::sub_32bit)
7626 if (!RI.hasStackRealignment(MF))
7628 std::min(Alignment, Subtarget.getFrameLowering()->getStackAlign());
7633 Size, Alignment,
true);
7635 if (
Ops.size() == 2 &&
Ops[0] == 0 &&
Ops[1] == 1) {
7636 unsigned NewOpc = 0;
7637 unsigned RCSize = 0;
7638 unsigned Opc =
MI.getOpcode();
7645 NewOpc = X86::CMP8ri;
7649 NewOpc = X86::CMP16ri;
7653 NewOpc = X86::CMP32ri;
7657 NewOpc = X86::CMP64ri32;
7666 MI.setDesc(
get(NewOpc));
7667 MI.getOperand(1).ChangeToImmediate(0);
7668 }
else if (
Ops.size() != 1)
7696 unsigned RegSize =
TRI.getRegSizeInBits(*RC);
7698 if ((
Opc == X86::MOVSSrm ||
Opc == X86::VMOVSSrm ||
Opc == X86::VMOVSSZrm ||
7699 Opc == X86::MOVSSrm_alt ||
Opc == X86::VMOVSSrm_alt ||
7700 Opc == X86::VMOVSSZrm_alt) &&
7706 case X86::CVTSS2SDrr_Int:
7707 case X86::VCVTSS2SDrr_Int:
7708 case X86::VCVTSS2SDZrr_Int:
7709 case X86::VCVTSS2SDZrrk_Int:
7710 case X86::VCVTSS2SDZrrkz_Int:
7711 case X86::CVTSS2SIrr_Int:
7712 case X86::CVTSS2SI64rr_Int:
7713 case X86::VCVTSS2SIrr_Int:
7714 case X86::VCVTSS2SI64rr_Int:
7715 case X86::VCVTSS2SIZrr_Int:
7716 case X86::VCVTSS2SI64Zrr_Int:
7717 case X86::CVTTSS2SIrr_Int:
7718 case X86::CVTTSS2SI64rr_Int:
7719 case X86::VCVTTSS2SIrr_Int:
7720 case X86::VCVTTSS2SI64rr_Int:
7721 case X86::VCVTTSS2SIZrr_Int:
7722 case X86::VCVTTSS2SI64Zrr_Int:
7723 case X86::VCVTSS2USIZrr_Int:
7724 case X86::VCVTSS2USI64Zrr_Int:
7725 case X86::VCVTTSS2USIZrr_Int:
7726 case X86::VCVTTSS2USI64Zrr_Int:
7727 case X86::RCPSSr_Int:
7728 case X86::VRCPSSr_Int:
7729 case X86::RSQRTSSr_Int:
7730 case X86::VRSQRTSSr_Int:
7731 case X86::ROUNDSSri_Int:
7732 case X86::VROUNDSSri_Int:
7733 case X86::COMISSrr_Int:
7734 case X86::VCOMISSrr_Int:
7735 case X86::VCOMISSZrr_Int:
7736 case X86::UCOMISSrr_Int:
7737 case X86::VUCOMISSrr_Int:
7738 case X86::VUCOMISSZrr_Int:
7739 case X86::ADDSSrr_Int:
7740 case X86::VADDSSrr_Int:
7741 case X86::VADDSSZrr_Int:
7742 case X86::CMPSSrri_Int:
7743 case X86::VCMPSSrri_Int:
7744 case X86::VCMPSSZrri_Int:
7745 case X86::DIVSSrr_Int:
7746 case X86::VDIVSSrr_Int:
7747 case X86::VDIVSSZrr_Int:
7748 case X86::MAXSSrr_Int:
7749 case X86::VMAXSSrr_Int:
7750 case X86::VMAXSSZrr_Int:
7751 case X86::MINSSrr_Int:
7752 case X86::VMINSSrr_Int:
7753 case X86::VMINSSZrr_Int:
7754 case X86::MULSSrr_Int:
7755 case X86::VMULSSrr_Int:
7756 case X86::VMULSSZrr_Int:
7757 case X86::SQRTSSr_Int:
7758 case X86::VSQRTSSr_Int:
7759 case X86::VSQRTSSZr_Int:
7760 case X86::SUBSSrr_Int:
7761 case X86::VSUBSSrr_Int:
7762 case X86::VSUBSSZrr_Int:
7763 case X86::VADDSSZrrk_Int:
7764 case X86::VADDSSZrrkz_Int:
7765 case X86::VCMPSSZrrik_Int:
7766 case X86::VDIVSSZrrk_Int:
7767 case X86::VDIVSSZrrkz_Int:
7768 case X86::VMAXSSZrrk_Int:
7769 case X86::VMAXSSZrrkz_Int:
7770 case X86::VMINSSZrrk_Int:
7771 case X86::VMINSSZrrkz_Int:
7772 case X86::VMULSSZrrk_Int:
7773 case X86::VMULSSZrrkz_Int:
7774 case X86::VSQRTSSZrk_Int:
7775 case X86::VSQRTSSZrkz_Int:
7776 case X86::VSUBSSZrrk_Int:
7777 case X86::VSUBSSZrrkz_Int:
7778 case X86::VFMADDSS4rr_Int:
7779 case X86::VFNMADDSS4rr_Int:
7780 case X86::VFMSUBSS4rr_Int:
7781 case X86::VFNMSUBSS4rr_Int:
7782 case X86::VFMADD132SSr_Int:
7783 case X86::VFNMADD132SSr_Int:
7784 case X86::VFMADD213SSr_Int:
7785 case X86::VFNMADD213SSr_Int:
7786 case X86::VFMADD231SSr_Int:
7787 case X86::VFNMADD231SSr_Int:
7788 case X86::VFMSUB132SSr_Int:
7789 case X86::VFNMSUB132SSr_Int:
7790 case X86::VFMSUB213SSr_Int:
7791 case X86::VFNMSUB213SSr_Int:
7792 case X86::VFMSUB231SSr_Int:
7793 case X86::VFNMSUB231SSr_Int:
7794 case X86::VFMADD132SSZr_Int:
7795 case X86::VFNMADD132SSZr_Int:
7796 case X86::VFMADD213SSZr_Int:
7797 case X86::VFNMADD213SSZr_Int:
7798 case X86::VFMADD231SSZr_Int:
7799 case X86::VFNMADD231SSZr_Int:
7800 case X86::VFMSUB132SSZr_Int:
7801 case X86::VFNMSUB132SSZr_Int:
7802 case X86::VFMSUB213SSZr_Int:
7803 case X86::VFNMSUB213SSZr_Int:
7804 case X86::VFMSUB231SSZr_Int:
7805 case X86::VFNMSUB231SSZr_Int:
7806 case X86::VFMADD132SSZrk_Int:
7807 case X86::VFNMADD132SSZrk_Int:
7808 case X86::VFMADD213SSZrk_Int:
7809 case X86::VFNMADD213SSZrk_Int:
7810 case X86::VFMADD231SSZrk_Int:
7811 case X86::VFNMADD231SSZrk_Int:
7812 case X86::VFMSUB132SSZrk_Int:
7813 case X86::VFNMSUB132SSZrk_Int:
7814 case X86::VFMSUB213SSZrk_Int:
7815 case X86::VFNMSUB213SSZrk_Int:
7816 case X86::VFMSUB231SSZrk_Int:
7817 case X86::VFNMSUB231SSZrk_Int:
7818 case X86::VFMADD132SSZrkz_Int:
7819 case X86::VFNMADD132SSZrkz_Int:
7820 case X86::VFMADD213SSZrkz_Int:
7821 case X86::VFNMADD213SSZrkz_Int:
7822 case X86::VFMADD231SSZrkz_Int:
7823 case X86::VFNMADD231SSZrkz_Int:
7824 case X86::VFMSUB132SSZrkz_Int:
7825 case X86::VFNMSUB132SSZrkz_Int:
7826 case X86::VFMSUB213SSZrkz_Int:
7827 case X86::VFNMSUB213SSZrkz_Int:
7828 case X86::VFMSUB231SSZrkz_Int:
7829 case X86::VFNMSUB231SSZrkz_Int:
7830 case X86::VFIXUPIMMSSZrri:
7831 case X86::VFIXUPIMMSSZrrik:
7832 case X86::VFIXUPIMMSSZrrikz:
7833 case X86::VFPCLASSSSZri:
7834 case X86::VFPCLASSSSZrik:
7835 case X86::VGETEXPSSZr:
7836 case X86::VGETEXPSSZrk:
7837 case X86::VGETEXPSSZrkz:
7838 case X86::VGETMANTSSZrri:
7839 case X86::VGETMANTSSZrrik:
7840 case X86::VGETMANTSSZrrikz:
7841 case X86::VRANGESSZrri:
7842 case X86::VRANGESSZrrik:
7843 case X86::VRANGESSZrrikz:
7844 case X86::VRCP14SSZrr:
7845 case X86::VRCP14SSZrrk:
7846 case X86::VRCP14SSZrrkz:
7847 case X86::VRCP28SSZr:
7848 case X86::VRCP28SSZrk:
7849 case X86::VRCP28SSZrkz:
7850 case X86::VREDUCESSZrri:
7851 case X86::VREDUCESSZrrik:
7852 case X86::VREDUCESSZrrikz:
7853 case X86::VRNDSCALESSZrri_Int:
7854 case X86::VRNDSCALESSZrrik_Int:
7855 case X86::VRNDSCALESSZrrikz_Int:
7856 case X86::VRSQRT14SSZrr:
7857 case X86::VRSQRT14SSZrrk:
7858 case X86::VRSQRT14SSZrrkz:
7859 case X86::VRSQRT28SSZr:
7860 case X86::VRSQRT28SSZrk:
7861 case X86::VRSQRT28SSZrkz:
7862 case X86::VSCALEFSSZrr:
7863 case X86::VSCALEFSSZrrk:
7864 case X86::VSCALEFSSZrrkz:
7871 if ((
Opc == X86::MOVSDrm ||
Opc == X86::VMOVSDrm ||
Opc == X86::VMOVSDZrm ||
7872 Opc == X86::MOVSDrm_alt ||
Opc == X86::VMOVSDrm_alt ||
7873 Opc == X86::VMOVSDZrm_alt) &&
7879 case X86::CVTSD2SSrr_Int:
7880 case X86::VCVTSD2SSrr_Int:
7881 case X86::VCVTSD2SSZrr_Int:
7882 case X86::VCVTSD2SSZrrk_Int:
7883 case X86::VCVTSD2SSZrrkz_Int:
7884 case X86::CVTSD2SIrr_Int:
7885 case X86::CVTSD2SI64rr_Int:
7886 case X86::VCVTSD2SIrr_Int:
7887 case X86::VCVTSD2SI64rr_Int:
7888 case X86::VCVTSD2SIZrr_Int:
7889 case X86::VCVTSD2SI64Zrr_Int:
7890 case X86::CVTTSD2SIrr_Int:
7891 case X86::CVTTSD2SI64rr_Int:
7892 case X86::VCVTTSD2SIrr_Int:
7893 case X86::VCVTTSD2SI64rr_Int:
7894 case X86::VCVTTSD2SIZrr_Int:
7895 case X86::VCVTTSD2SI64Zrr_Int:
7896 case X86::VCVTSD2USIZrr_Int:
7897 case X86::VCVTSD2USI64Zrr_Int:
7898 case X86::VCVTTSD2USIZrr_Int:
7899 case X86::VCVTTSD2USI64Zrr_Int:
7900 case X86::ROUNDSDri_Int:
7901 case X86::VROUNDSDri_Int:
7902 case X86::COMISDrr_Int:
7903 case X86::VCOMISDrr_Int:
7904 case X86::VCOMISDZrr_Int:
7905 case X86::UCOMISDrr_Int:
7906 case X86::VUCOMISDrr_Int:
7907 case X86::VUCOMISDZrr_Int:
7908 case X86::ADDSDrr_Int:
7909 case X86::VADDSDrr_Int:
7910 case X86::VADDSDZrr_Int:
7911 case X86::CMPSDrri_Int:
7912 case X86::VCMPSDrri_Int:
7913 case X86::VCMPSDZrri_Int:
7914 case X86::DIVSDrr_Int:
7915 case X86::VDIVSDrr_Int:
7916 case X86::VDIVSDZrr_Int:
7917 case X86::MAXSDrr_Int:
7918 case X86::VMAXSDrr_Int:
7919 case X86::VMAXSDZrr_Int:
7920 case X86::MINSDrr_Int:
7921 case X86::VMINSDrr_Int:
7922 case X86::VMINSDZrr_Int:
7923 case X86::MULSDrr_Int:
7924 case X86::VMULSDrr_Int:
7925 case X86::VMULSDZrr_Int:
7926 case X86::SQRTSDr_Int:
7927 case X86::VSQRTSDr_Int:
7928 case X86::VSQRTSDZr_Int:
7929 case X86::SUBSDrr_Int:
7930 case X86::VSUBSDrr_Int:
7931 case X86::VSUBSDZrr_Int:
7932 case X86::VADDSDZrrk_Int:
7933 case X86::VADDSDZrrkz_Int:
7934 case X86::VCMPSDZrrik_Int:
7935 case X86::VDIVSDZrrk_Int:
7936 case X86::VDIVSDZrrkz_Int:
7937 case X86::VMAXSDZrrk_Int:
7938 case X86::VMAXSDZrrkz_Int:
7939 case X86::VMINSDZrrk_Int:
7940 case X86::VMINSDZrrkz_Int:
7941 case X86::VMULSDZrrk_Int:
7942 case X86::VMULSDZrrkz_Int:
7943 case X86::VSQRTSDZrk_Int:
7944 case X86::VSQRTSDZrkz_Int:
7945 case X86::VSUBSDZrrk_Int:
7946 case X86::VSUBSDZrrkz_Int:
7947 case X86::VFMADDSD4rr_Int:
7948 case X86::VFNMADDSD4rr_Int:
7949 case X86::VFMSUBSD4rr_Int:
7950 case X86::VFNMSUBSD4rr_Int:
7951 case X86::VFMADD132SDr_Int:
7952 case X86::VFNMADD132SDr_Int:
7953 case X86::VFMADD213SDr_Int:
7954 case X86::VFNMADD213SDr_Int:
7955 case X86::VFMADD231SDr_Int:
7956 case X86::VFNMADD231SDr_Int:
7957 case X86::VFMSUB132SDr_Int:
7958 case X86::VFNMSUB132SDr_Int:
7959 case X86::VFMSUB213SDr_Int:
7960 case X86::VFNMSUB213SDr_Int:
7961 case X86::VFMSUB231SDr_Int:
7962 case X86::VFNMSUB231SDr_Int:
7963 case X86::VFMADD132SDZr_Int:
7964 case X86::VFNMADD132SDZr_Int:
7965 case X86::VFMADD213SDZr_Int:
7966 case X86::VFNMADD213SDZr_Int:
7967 case X86::VFMADD231SDZr_Int:
7968 case X86::VFNMADD231SDZr_Int:
7969 case X86::VFMSUB132SDZr_Int:
7970 case X86::VFNMSUB132SDZr_Int:
7971 case X86::VFMSUB213SDZr_Int:
7972 case X86::VFNMSUB213SDZr_Int:
7973 case X86::VFMSUB231SDZr_Int:
7974 case X86::VFNMSUB231SDZr_Int:
7975 case X86::VFMADD132SDZrk_Int:
7976 case X86::VFNMADD132SDZrk_Int:
7977 case X86::VFMADD213SDZrk_Int:
7978 case X86::VFNMADD213SDZrk_Int:
7979 case X86::VFMADD231SDZrk_Int:
7980 case X86::VFNMADD231SDZrk_Int:
7981 case X86::VFMSUB132SDZrk_Int:
7982 case X86::VFNMSUB132SDZrk_Int:
7983 case X86::VFMSUB213SDZrk_Int:
7984 case X86::VFNMSUB213SDZrk_Int:
7985 case X86::VFMSUB231SDZrk_Int:
7986 case X86::VFNMSUB231SDZrk_Int:
7987 case X86::VFMADD132SDZrkz_Int:
7988 case X86::VFNMADD132SDZrkz_Int:
7989 case X86::VFMADD213SDZrkz_Int:
7990 case X86::VFNMADD213SDZrkz_Int:
7991 case X86::VFMADD231SDZrkz_Int:
7992 case X86::VFNMADD231SDZrkz_Int:
7993 case X86::VFMSUB132SDZrkz_Int:
7994 case X86::VFNMSUB132SDZrkz_Int:
7995 case X86::VFMSUB213SDZrkz_Int:
7996 case X86::VFNMSUB213SDZrkz_Int:
7997 case X86::VFMSUB231SDZrkz_Int:
7998 case X86::VFNMSUB231SDZrkz_Int:
7999 case X86::VFIXUPIMMSDZrri:
8000 case X86::VFIXUPIMMSDZrrik:
8001 case X86::VFIXUPIMMSDZrrikz:
8002 case X86::VFPCLASSSDZri:
8003 case X86::VFPCLASSSDZrik:
8004 case X86::VGETEXPSDZr:
8005 case X86::VGETEXPSDZrk:
8006 case X86::VGETEXPSDZrkz:
8007 case X86::VGETMANTSDZrri:
8008 case X86::VGETMANTSDZrrik:
8009 case X86::VGETMANTSDZrrikz:
8010 case X86::VRANGESDZrri:
8011 case X86::VRANGESDZrrik:
8012 case X86::VRANGESDZrrikz:
8013 case X86::VRCP14SDZrr:
8014 case X86::VRCP14SDZrrk:
8015 case X86::VRCP14SDZrrkz:
8016 case X86::VRCP28SDZr:
8017 case X86::VRCP28SDZrk:
8018 case X86::VRCP28SDZrkz:
8019 case X86::VREDUCESDZrri:
8020 case X86::VREDUCESDZrrik:
8021 case X86::VREDUCESDZrrikz:
8022 case X86::VRNDSCALESDZrri_Int:
8023 case X86::VRNDSCALESDZrrik_Int:
8024 case X86::VRNDSCALESDZrrikz_Int:
8025 case X86::VRSQRT14SDZrr:
8026 case X86::VRSQRT14SDZrrk:
8027 case X86::VRSQRT14SDZrrkz:
8028 case X86::VRSQRT28SDZr:
8029 case X86::VRSQRT28SDZrk:
8030 case X86::VRSQRT28SDZrkz:
8031 case X86::VSCALEFSDZrr:
8032 case X86::VSCALEFSDZrrk:
8033 case X86::VSCALEFSDZrrkz:
8040 if ((
Opc == X86::VMOVSHZrm ||
Opc == X86::VMOVSHZrm_alt) &&
RegSize > 16) {
8045 case X86::VADDSHZrr_Int:
8046 case X86::VCMPSHZrri_Int:
8047 case X86::VDIVSHZrr_Int:
8048 case X86::VMAXSHZrr_Int:
8049 case X86::VMINSHZrr_Int:
8050 case X86::VMULSHZrr_Int:
8051 case X86::VSUBSHZrr_Int:
8052 case X86::VADDSHZrrk_Int:
8053 case X86::VADDSHZrrkz_Int:
8054 case X86::VCMPSHZrrik_Int:
8055 case X86::VDIVSHZrrk_Int:
8056 case X86::VDIVSHZrrkz_Int:
8057 case X86::VMAXSHZrrk_Int:
8058 case X86::VMAXSHZrrkz_Int:
8059 case X86::VMINSHZrrk_Int:
8060 case X86::VMINSHZrrkz_Int:
8061 case X86::VMULSHZrrk_Int:
8062 case X86::VMULSHZrrkz_Int:
8063 case X86::VSUBSHZrrk_Int:
8064 case X86::VSUBSHZrrkz_Int:
8065 case X86::VFMADD132SHZr_Int:
8066 case X86::VFNMADD132SHZr_Int:
8067 case X86::VFMADD213SHZr_Int:
8068 case X86::VFNMADD213SHZr_Int:
8069 case X86::VFMADD231SHZr_Int:
8070 case X86::VFNMADD231SHZr_Int:
8071 case X86::VFMSUB132SHZr_Int:
8072 case X86::VFNMSUB132SHZr_Int:
8073 case X86::VFMSUB213SHZr_Int:
8074 case X86::VFNMSUB213SHZr_Int:
8075 case X86::VFMSUB231SHZr_Int:
8076 case X86::VFNMSUB231SHZr_Int:
8077 case X86::VFMADD132SHZrk_Int:
8078 case X86::VFNMADD132SHZrk_Int:
8079 case X86::VFMADD213SHZrk_Int:
8080 case X86::VFNMADD213SHZrk_Int:
8081 case X86::VFMADD231SHZrk_Int:
8082 case X86::VFNMADD231SHZrk_Int:
8083 case X86::VFMSUB132SHZrk_Int:
8084 case X86::VFNMSUB132SHZrk_Int:
8085 case X86::VFMSUB213SHZrk_Int:
8086 case X86::VFNMSUB213SHZrk_Int:
8087 case X86::VFMSUB231SHZrk_Int:
8088 case X86::VFNMSUB231SHZrk_Int:
8089 case X86::VFMADD132SHZrkz_Int:
8090 case X86::VFNMADD132SHZrkz_Int:
8091 case X86::VFMADD213SHZrkz_Int:
8092 case X86::VFNMADD213SHZrkz_Int:
8093 case X86::VFMADD231SHZrkz_Int:
8094 case X86::VFNMADD231SHZrkz_Int:
8095 case X86::VFMSUB132SHZrkz_Int:
8096 case X86::VFNMSUB132SHZrkz_Int:
8097 case X86::VFMSUB213SHZrkz_Int:
8098 case X86::VFNMSUB213SHZrkz_Int:
8099 case X86::VFMSUB231SHZrkz_Int:
8100 case X86::VFNMSUB231SHZrkz_Int:
8117 for (
auto Op :
Ops) {
8118 if (
MI.getOperand(
Op).getSubReg())
8156 case X86::AVX512_512_SET0:
8157 case X86::AVX512_512_SETALLONES:
8158 Alignment =
Align(64);
8160 case X86::AVX2_SETALLONES:
8161 case X86::AVX1_SETALLONES:
8163 case X86::AVX512_256_SET0:
8164 Alignment =
Align(32);
8167 case X86::V_SETALLONES:
8168 case X86::AVX512_128_SET0:
8169 case X86::FsFLD0F128:
8170 case X86::AVX512_FsFLD0F128:
8171 Alignment =
Align(16);
8175 case X86::AVX512_FsFLD0SD:
8176 Alignment =
Align(8);
8179 case X86::AVX512_FsFLD0SS:
8180 Alignment =
Align(4);
8183 case X86::AVX512_FsFLD0SH:
8184 Alignment =
Align(2);
8189 if (
Ops.size() == 2 &&
Ops[0] == 0 &&
Ops[1] == 1) {
8190 unsigned NewOpc = 0;
8191 switch (
MI.getOpcode()) {
8195 NewOpc = X86::CMP8ri;
8198 NewOpc = X86::CMP16ri;
8201 NewOpc = X86::CMP32ri;
8204 NewOpc = X86::CMP64ri32;
8208 MI.setDesc(
get(NewOpc));
8209 MI.getOperand(1).ChangeToImmediate(0);
8210 }
else if (
Ops.size() != 1)
8222 case X86::V_SETALLONES:
8223 case X86::AVX2_SETALLONES:
8224 case X86::AVX1_SETALLONES:
8226 case X86::AVX512_128_SET0:
8227 case X86::AVX512_256_SET0:
8228 case X86::AVX512_512_SET0:
8229 case X86::AVX512_512_SETALLONES:
8231 case X86::AVX512_FsFLD0SH:
8233 case X86::AVX512_FsFLD0SD:
8235 case X86::AVX512_FsFLD0SS:
8236 case X86::FsFLD0F128:
8237 case X86::AVX512_FsFLD0F128: {
8246 unsigned PICBase = 0;
8249 if (Subtarget.is64Bit()) {
8262 bool IsAllOnes =
false;
8265 case X86::AVX512_FsFLD0SS:
8269 case X86::AVX512_FsFLD0SD:
8272 case X86::FsFLD0F128:
8273 case X86::AVX512_FsFLD0F128:
8277 case X86::AVX512_FsFLD0SH:
8280 case X86::AVX512_512_SETALLONES:
8283 case X86::AVX512_512_SET0:
8287 case X86::AVX1_SETALLONES:
8288 case X86::AVX2_SETALLONES:
8291 case X86::AVX512_256_SET0:
8301 case X86::V_SETALLONES:
8305 case X86::AVX512_128_SET0:
8323 case X86::VPBROADCASTBZ128rm:
8324 case X86::VPBROADCASTBZ256rm:
8325 case X86::VPBROADCASTBZrm:
8326 case X86::VBROADCASTF32X2Z256rm:
8327 case X86::VBROADCASTF32X2Zrm:
8328 case X86::VBROADCASTI32X2Z128rm:
8329 case X86::VBROADCASTI32X2Z256rm:
8330 case X86::VBROADCASTI32X2Zrm:
8334#define FOLD_BROADCAST(SIZE) \
8335 MOs.append(LoadMI.operands_begin() + NumOps - X86::AddrNumOperands, \
8336 LoadMI.operands_begin() + NumOps); \
8337 return foldMemoryBroadcast(MF, MI, Ops[0], MOs, InsertPt, SIZE, \
8339 case X86::VPBROADCASTWZ128rm:
8340 case X86::VPBROADCASTWZ256rm:
8341 case X86::VPBROADCASTWZrm:
8343 case X86::VPBROADCASTDZ128rm:
8344 case X86::VPBROADCASTDZ256rm:
8345 case X86::VPBROADCASTDZrm:
8346 case X86::VBROADCASTSSZ128rm:
8347 case X86::VBROADCASTSSZ256rm:
8348 case X86::VBROADCASTSSZrm:
8350 case X86::VPBROADCASTQZ128rm:
8351 case X86::VPBROADCASTQZ256rm:
8352 case X86::VPBROADCASTQZrm:
8353 case X86::VBROADCASTSDZ256rm:
8354 case X86::VBROADCASTSDZrm:
8367 0, Alignment,
true);
8374 unsigned BitsSize,
bool AllowCommute)
const {
8378 ?
fuseInst(MF,
I->DstOp, OpNum, MOs, InsertPt,
MI, *
this)
8384 unsigned CommuteOpIdx2 = commuteOperandsForFold(
MI, OpNum);
8385 if (CommuteOpIdx2 == OpNum) {
8390 foldMemoryBroadcast(MF,
MI, CommuteOpIdx2, MOs, InsertPt, BitsSize,
8395 commuteInstruction(
MI,
false, OpNum, CommuteOpIdx2);
8410 if (!MMO->isStore()) {
8428 if (!MMO->isStore())
8431 if (!MMO->isLoad()) {
8449 assert((SpillSize == 64 || STI.hasVLX()) &&
8450 "Can't broadcast less than 64 bytes without AVX512VL!");
8452#define CASE_BCAST_TYPE_OPC(TYPE, OP16, OP32, OP64) \
8454 switch (SpillSize) { \
8456 llvm_unreachable("Unknown spill size"); \
8490 unsigned Opc =
I->DstOp;
8494 if (UnfoldLoad && !FoldedLoad)
8496 UnfoldLoad &= FoldedLoad;
8497 if (UnfoldStore && !FoldedStore)
8499 UnfoldStore &= FoldedStore;
8506 if (!
MI.hasOneMemOperand() && RC == &X86::VR128RegClass &&
8507 Subtarget.isUnalignedMem16Slow())
8516 for (
unsigned i = 0, e =
MI.getNumOperands(); i != e; ++i) {
8520 else if (
Op.isReg() &&
Op.isImplicit())
8536 unsigned Alignment = std::max<uint32_t>(
TRI.getSpillSize(*RC), 16);
8537 bool isAligned = !MMOs.empty() && MMOs.front()->getAlign() >= Alignment;
8581 case X86::CMP64ri32:
8592 case X86::CMP64ri32:
8593 NewOpc = X86::TEST64rr;
8596 NewOpc = X86::TEST32rr;
8599 NewOpc = X86::TEST16rr;
8602 NewOpc = X86::TEST8rr;
8616 unsigned Alignment = std::max<uint32_t>(
TRI.getSpillSize(*DstRC), 16);
8617 bool isAligned = !MMOs.empty() && MMOs.front()->getAlign() >= Alignment;
8633 if (!
N->isMachineOpcode())
8639 unsigned Opc =
I->DstOp;
8647 unsigned NumDefs =
MCID.NumDefs;
8648 std::vector<SDValue> AddrOps;
8649 std::vector<SDValue> BeforeOps;
8650 std::vector<SDValue> AfterOps;
8652 unsigned NumOps =
N->getNumOperands();
8653 for (
unsigned i = 0; i !=
NumOps - 1; ++i) {
8656 AddrOps.push_back(
Op);
8657 else if (i < Index - NumDefs)
8658 BeforeOps.push_back(
Op);
8659 else if (i > Index - NumDefs)
8660 AfterOps.push_back(
Op);
8663 AddrOps.push_back(Chain);
8668 EVT VT = *
TRI.legalclasstypes_begin(*RC);
8670 if (MMOs.empty() && RC == &X86::VR128RegClass &&
8671 Subtarget.isUnalignedMem16Slow())
8681 unsigned Alignment = std::max<uint32_t>(
TRI.getSpillSize(*RC), 16);
8682 bool isAligned = !MMOs.empty() && MMOs.front()->getAlign() >= Alignment;
8694 std::vector<EVT> VTs;
8696 if (
MCID.getNumDefs() > 0) {
8698 VTs.push_back(*
TRI.legalclasstypes_begin(*DstRC));
8700 for (
unsigned i = 0, e =
N->getNumValues(); i != e; ++i) {
8701 EVT VT =
N->getValueType(i);
8702 if (VT != MVT::Other && i >= (
unsigned)
MCID.getNumDefs())
8706 BeforeOps.push_back(
SDValue(Load, 0));
8712 case X86::CMP64ri32:
8720 case X86::CMP64ri32:
8721 Opc = X86::TEST64rr;
8724 Opc = X86::TEST32rr;
8727 Opc = X86::TEST16rr;
8733 BeforeOps[1] = BeforeOps[0];
8742 AddrOps.push_back(
SDValue(NewNode, 0));
8743 AddrOps.push_back(Chain);
8745 if (MMOs.empty() && RC == &X86::VR128RegClass &&
8746 Subtarget.isUnalignedMem16Slow())
8751 unsigned Alignment = std::max<uint32_t>(
TRI.getSpillSize(*RC), 16);
8752 bool isAligned = !MMOs.empty() && MMOs.front()->getAlign() >= Alignment;
8755 dl, MVT::Other, AddrOps);
8768 unsigned *LoadRegIndex)
const {
8774 if (UnfoldLoad && !FoldedLoad)
8776 if (UnfoldStore && !FoldedStore)
8785 int64_t &Offset2)
const {
8789 auto IsLoadOpcode = [&](
unsigned Opcode) {
8801 case X86::MOVSSrm_alt:
8803 case X86::MOVSDrm_alt:
8804 case X86::MMX_MOVD64rm:
8805 case X86::MMX_MOVQ64rm:
8814 case X86::VMOVSSrm_alt:
8816 case X86::VMOVSDrm_alt:
8817 case X86::VMOVAPSrm:
8818 case X86::VMOVUPSrm:
8819 case X86::VMOVAPDrm:
8820 case X86::VMOVUPDrm:
8821 case X86::VMOVDQArm:
8822 case X86::VMOVDQUrm:
8823 case X86::VMOVAPSYrm:
8824 case X86::VMOVUPSYrm:
8825 case X86::VMOVAPDYrm:
8826 case X86::VMOVUPDYrm:
8827 case X86::VMOVDQAYrm:
8828 case X86::VMOVDQUYrm:
8830 case X86::VMOVSSZrm:
8831 case X86::VMOVSSZrm_alt:
8832 case X86::VMOVSDZrm:
8833 case X86::VMOVSDZrm_alt:
8834 case X86::VMOVAPSZ128rm:
8835 case X86::VMOVUPSZ128rm:
8836 case X86::VMOVAPSZ128rm_NOVLX:
8837 case X86::VMOVUPSZ128rm_NOVLX:
8838 case X86::VMOVAPDZ128rm:
8839 case X86::VMOVUPDZ128rm:
8840 case X86::VMOVDQU8Z128rm:
8841 case X86::VMOVDQU16Z128rm:
8842 case X86::VMOVDQA32Z128rm:
8843 case X86::VMOVDQU32Z128rm:
8844 case X86::VMOVDQA64Z128rm:
8845 case X86::VMOVDQU64Z128rm:
8846 case X86::VMOVAPSZ256rm:
8847 case X86::VMOVUPSZ256rm:
8848 case X86::VMOVAPSZ256rm_NOVLX:
8849 case X86::VMOVUPSZ256rm_NOVLX:
8850 case X86::VMOVAPDZ256rm:
8851 case X86::VMOVUPDZ256rm:
8852 case X86::VMOVDQU8Z256rm:
8853 case X86::VMOVDQU16Z256rm:
8854 case X86::VMOVDQA32Z256rm:
8855 case X86::VMOVDQU32Z256rm:
8856 case X86::VMOVDQA64Z256rm:
8857 case X86::VMOVDQU64Z256rm:
8858 case X86::VMOVAPSZrm:
8859 case X86::VMOVUPSZrm:
8860 case X86::VMOVAPDZrm:
8861 case X86::VMOVUPDZrm:
8862 case X86::VMOVDQU8Zrm:
8863 case X86::VMOVDQU16Zrm:
8864 case X86::VMOVDQA32Zrm:
8865 case X86::VMOVDQU32Zrm:
8866 case X86::VMOVDQA64Zrm:
8867 case X86::VMOVDQU64Zrm:
8869 case X86::KMOVBkm_EVEX:
8871 case X86::KMOVWkm_EVEX:
8873 case X86::KMOVDkm_EVEX:
8875 case X86::KMOVQkm_EVEX:
8885 auto HasSameOp = [&](
int I) {
8901 if (!Disp1 || !Disp2)
8904 Offset1 = Disp1->getSExtValue();
8905 Offset2 = Disp2->getSExtValue();
8910 int64_t Offset1, int64_t Offset2,
8911 unsigned NumLoads)
const {
8912 assert(Offset2 > Offset1);
8913 if ((Offset2 - Offset1) / 8 > 64)
8927 case X86::MMX_MOVD64rm:
8928 case X86::MMX_MOVQ64rm:
8937 if (Subtarget.is64Bit()) {
8940 }
else if (NumLoads) {
8963 unsigned Opcode =
MI.getOpcode();
8964 if (Opcode == X86::ENDBR64 || Opcode == X86::ENDBR32 ||
8965 Opcode == X86::PLDTILECFGV)
8978 assert(
Cond.size() == 1 &&
"Invalid X86 branch condition!");
8988 return !(RC == &X86::CCRRegClass || RC == &X86::DFCCRRegClass ||
8989 RC == &X86::RFP32RegClass || RC == &X86::RFP64RegClass ||
8990 RC == &X86::RFP80RegClass);
9003 return GlobalBaseReg;
9008 GlobalBaseReg = RegInfo.createVirtualRegister(
9009 Subtarget.is64Bit() ? &X86::GR64_NOSPRegClass : &X86::GR32_NOSPRegClass);
9011 return GlobalBaseReg;
9019 for (
const uint16_t(&Row)[3] : Table)
9020 if (Row[domain - 1] == opcode)
9028 for (
const uint16_t(&Row)[4] : Table)
9029 if (Row[domain - 1] == opcode || (domain == 3 && Row[3] == opcode))
9036 unsigned NewWidth,
unsigned *pNewMask =
nullptr) {
9037 assert(((OldWidth % NewWidth) == 0 || (NewWidth % OldWidth) == 0) &&
9038 "Illegal blend mask scale");
9039 unsigned NewMask = 0;
9041 if ((OldWidth % NewWidth) == 0) {
9042 unsigned Scale = OldWidth / NewWidth;
9043 unsigned SubMask = (1u << Scale) - 1;
9044 for (
unsigned i = 0; i != NewWidth; ++i) {
9045 unsigned Sub = (OldMask >> (i * Scale)) & SubMask;
9047 NewMask |= (1u << i);
9048 else if (
Sub != 0x0)
9052 unsigned Scale = NewWidth / OldWidth;
9053 unsigned SubMask = (1u << Scale) - 1;
9054 for (
unsigned i = 0; i != OldWidth; ++i) {
9055 if (OldMask & (1 << i)) {
9056 NewMask |= (SubMask << (i * Scale));
9062 *pNewMask = NewMask;
9067 unsigned Opcode =
MI.getOpcode();
9068 unsigned NumOperands =
MI.getDesc().getNumOperands();
9070 auto GetBlendDomains = [&](
unsigned ImmWidth,
bool Is256) {
9072 if (
MI.getOperand(NumOperands - 1).isImm()) {
9073 unsigned Imm =
MI.getOperand(NumOperands - 1).getImm();
9075 validDomains |= 0x2;
9077 validDomains |= 0x4;
9078 if (!Is256 || Subtarget.hasAVX2())
9079 validDomains |= 0x8;
9081 return validDomains;
9085 case X86::BLENDPDrmi:
9086 case X86::BLENDPDrri:
9087 case X86::VBLENDPDrmi:
9088 case X86::VBLENDPDrri:
9089 return GetBlendDomains(2,
false);
9090 case X86::VBLENDPDYrmi:
9091 case X86::VBLENDPDYrri:
9092 return GetBlendDomains(4,
true);
9093 case X86::BLENDPSrmi:
9094 case X86::BLENDPSrri:
9095 case X86::VBLENDPSrmi:
9096 case X86::VBLENDPSrri:
9097 case X86::VPBLENDDrmi:
9098 case X86::VPBLENDDrri:
9099 return GetBlendDomains(4,
false);
9100 case X86::VBLENDPSYrmi:
9101 case X86::VBLENDPSYrri:
9102 case X86::VPBLENDDYrmi:
9103 case X86::VPBLENDDYrri:
9104 return GetBlendDomains(8,
true);
9105 case X86::PBLENDWrmi:
9106 case X86::PBLENDWrri:
9107 case X86::VPBLENDWrmi:
9108 case X86::VPBLENDWrri:
9110 case X86::VPBLENDWYrmi:
9111 case X86::VPBLENDWYrri:
9112 return GetBlendDomains(8,
false);
9113 case X86::VPANDDZ128rr:
9114 case X86::VPANDDZ128rm:
9115 case X86::VPANDDZ256rr:
9116 case X86::VPANDDZ256rm:
9117 case X86::VPANDQZ128rr:
9118 case X86::VPANDQZ128rm:
9119 case X86::VPANDQZ256rr:
9120 case X86::VPANDQZ256rm:
9121 case X86::VPANDNDZ128rr:
9122 case X86::VPANDNDZ128rm:
9123 case X86::VPANDNDZ256rr:
9124 case X86::VPANDNDZ256rm:
9125 case X86::VPANDNQZ128rr:
9126 case X86::VPANDNQZ128rm:
9127 case X86::VPANDNQZ256rr:
9128 case X86::VPANDNQZ256rm:
9129 case X86::VPORDZ128rr:
9130 case X86::VPORDZ128rm:
9131 case X86::VPORDZ256rr:
9132 case X86::VPORDZ256rm:
9133 case X86::VPORQZ128rr:
9134 case X86::VPORQZ128rm:
9135 case X86::VPORQZ256rr:
9136 case X86::VPORQZ256rm:
9137 case X86::VPXORDZ128rr:
9138 case X86::VPXORDZ128rm:
9139 case X86::VPXORDZ256rr:
9140 case X86::VPXORDZ256rm:
9141 case X86::VPXORQZ128rr:
9142 case X86::VPXORQZ128rm:
9143 case X86::VPXORQZ256rr:
9144 case X86::VPXORQZ256rm:
9147 if (Subtarget.hasDQI())
9150 if (RI.getEncodingValue(
MI.getOperand(0).getReg()) >= 16)
9152 if (RI.getEncodingValue(
MI.getOperand(1).getReg()) >= 16)
9155 if (NumOperands == 3 &&
9156 RI.getEncodingValue(
MI.getOperand(2).getReg()) >= 16)
9161 case X86::MOVHLPSrr:
9168 if (
MI.getOperand(1).getReg() ==
MI.getOperand(2).getReg() &&
9169 MI.getOperand(0).getSubReg() == 0 &&
9170 MI.getOperand(1).getSubReg() == 0 &&
MI.getOperand(2).getSubReg() == 0)
9173 case X86::SHUFPDrri:
9179#include "X86ReplaceableInstrs.def"
9185 assert(dom &&
"Not an SSE instruction");
9187 unsigned Opcode =
MI.getOpcode();
9188 unsigned NumOperands =
MI.getDesc().getNumOperands();
9190 auto SetBlendDomain = [&](
unsigned ImmWidth,
bool Is256) {
9191 if (
MI.getOperand(NumOperands - 1).isImm()) {
9192 unsigned Imm =
MI.getOperand(NumOperands - 1).getImm() & 255;
9193 Imm = (ImmWidth == 16 ? ((Imm << 8) | Imm) : Imm);
9194 unsigned NewImm = Imm;
9196 const uint16_t *table =
lookup(Opcode, dom, ReplaceableBlendInstrs);
9198 table =
lookup(Opcode, dom, ReplaceableBlendAVX2Instrs);
9202 }
else if (
Domain == 2) {
9204 }
else if (
Domain == 3) {
9205 if (Subtarget.hasAVX2()) {
9207 if ((ImmWidth / (Is256 ? 2 : 1)) != 8) {
9208 table =
lookup(Opcode, dom, ReplaceableBlendAVX2Instrs);
9212 assert(!Is256 &&
"128-bit vector expected");
9217 assert(table && table[
Domain - 1] &&
"Unknown domain op");
9219 MI.getOperand(NumOperands - 1).setImm(NewImm & 255);
9225 case X86::BLENDPDrmi:
9226 case X86::BLENDPDrri:
9227 case X86::VBLENDPDrmi:
9228 case X86::VBLENDPDrri:
9229 return SetBlendDomain(2,
false);
9230 case X86::VBLENDPDYrmi:
9231 case X86::VBLENDPDYrri:
9232 return SetBlendDomain(4,
true);
9233 case X86::BLENDPSrmi:
9234 case X86::BLENDPSrri:
9235 case X86::VBLENDPSrmi:
9236 case X86::VBLENDPSrri:
9237 case X86::VPBLENDDrmi:
9238 case X86::VPBLENDDrri:
9239 return SetBlendDomain(4,
false);
9240 case X86::VBLENDPSYrmi:
9241 case X86::VBLENDPSYrri:
9242 case X86::VPBLENDDYrmi:
9243 case X86::VPBLENDDYrri:
9244 return SetBlendDomain(8,
true);
9245 case X86::PBLENDWrmi:
9246 case X86::PBLENDWrri:
9247 case X86::VPBLENDWrmi:
9248 case X86::VPBLENDWrri:
9249 return SetBlendDomain(8,
false);
9250 case X86::VPBLENDWYrmi:
9251 case X86::VPBLENDWYrri:
9252 return SetBlendDomain(16,
true);
9253 case X86::VPANDDZ128rr:
9254 case X86::VPANDDZ128rm:
9255 case X86::VPANDDZ256rr:
9256 case X86::VPANDDZ256rm:
9257 case X86::VPANDQZ128rr:
9258 case X86::VPANDQZ128rm:
9259 case X86::VPANDQZ256rr:
9260 case X86::VPANDQZ256rm:
9261 case X86::VPANDNDZ128rr:
9262 case X86::VPANDNDZ128rm:
9263 case X86::VPANDNDZ256rr:
9264 case X86::VPANDNDZ256rm:
9265 case X86::VPANDNQZ128rr:
9266 case X86::VPANDNQZ128rm:
9267 case X86::VPANDNQZ256rr:
9268 case X86::VPANDNQZ256rm:
9269 case X86::VPORDZ128rr:
9270 case X86::VPORDZ128rm:
9271 case X86::VPORDZ256rr:
9272 case X86::VPORDZ256rm:
9273 case X86::VPORQZ128rr:
9274 case X86::VPORQZ128rm:
9275 case X86::VPORQZ256rr:
9276 case X86::VPORQZ256rm:
9277 case X86::VPXORDZ128rr:
9278 case X86::VPXORDZ128rm:
9279 case X86::VPXORDZ256rr:
9280 case X86::VPXORDZ256rm:
9281 case X86::VPXORQZ128rr:
9282 case X86::VPXORQZ128rm:
9283 case X86::VPXORQZ256rr:
9284 case X86::VPXORQZ256rm: {
9286 if (Subtarget.hasDQI())
9290 lookupAVX512(
MI.getOpcode(), dom, ReplaceableCustomAVX512LogicInstrs);
9291 assert(table &&
"Instruction not found in table?");
9294 if (
Domain == 3 && (dom == 1 || table[3] ==
MI.getOpcode()))
9299 case X86::UNPCKHPDrr:
9300 case X86::MOVHLPSrr:
9303 MI.getOperand(1).getReg() ==
MI.getOperand(2).getReg() &&
9304 MI.getOperand(0).getSubReg() == 0 &&
9305 MI.getOperand(1).getSubReg() == 0 &&
9306 MI.getOperand(2).getSubReg() == 0) {
9307 commuteInstruction(
MI,
false);
9311 if (Opcode == X86::MOVHLPSrr)
9314 case X86::SHUFPDrri: {
9316 unsigned Imm =
MI.getOperand(3).getImm();
9317 unsigned NewImm = 0x44;
9322 MI.getOperand(3).setImm(NewImm);
9323 MI.setDesc(
get(X86::SHUFPSrri));
9331std::pair<uint16_t, uint16_t>
9334 unsigned opcode =
MI.getOpcode();
9340 return std::make_pair(domain, validDomains);
9342 if (
lookup(opcode, domain, ReplaceableInstrs)) {
9344 }
else if (
lookup(opcode, domain, ReplaceableInstrsAVX2)) {
9345 validDomains = Subtarget.hasAVX2() ? 0xe : 0x6;
9346 }
else if (
lookup(opcode, domain, ReplaceableInstrsFP)) {
9348 }
else if (
lookup(opcode, domain, ReplaceableInstrsAVX2InsertExtract)) {
9351 if (!Subtarget.hasAVX2())
9352 return std::make_pair(0, 0);
9354 }
else if (
lookupAVX512(opcode, domain, ReplaceableInstrsAVX512)) {
9356 }
else if (Subtarget.hasDQI() &&
9357 lookupAVX512(opcode, domain, ReplaceableInstrsAVX512DQ)) {
9359 }
else if (Subtarget.hasDQI()) {
9361 lookupAVX512(opcode, domain, ReplaceableInstrsAVX512DQMasked)) {
9362 if (domain == 1 || (domain == 3 && table[3] == opcode))
9369 return std::make_pair(domain, validDomains);
9375 assert(dom &&
"Not an SSE instruction");
9384 "256-bit vector operations only available in AVX2");
9385 table =
lookup(
MI.getOpcode(), dom, ReplaceableInstrsAVX2);
9388 table =
lookup(
MI.getOpcode(), dom, ReplaceableInstrsFP);
9390 "Can only select PackedSingle or PackedDouble");
9393 assert(Subtarget.hasAVX2() &&
9394 "256-bit insert/extract only available in AVX2");
9395 table =
lookup(
MI.getOpcode(), dom, ReplaceableInstrsAVX2InsertExtract);
9398 assert(Subtarget.hasAVX512() &&
"Requires AVX-512");
9399 table =
lookupAVX512(
MI.getOpcode(), dom, ReplaceableInstrsAVX512);
9401 if (table &&
Domain == 3 && table[3] ==
MI.getOpcode())
9405 assert((Subtarget.hasDQI() ||
Domain >= 3) &&
"Requires AVX-512DQ");
9406 table =
lookupAVX512(
MI.getOpcode(), dom, ReplaceableInstrsAVX512DQ);
9409 if (table &&
Domain == 3 && (dom == 1 || table[3] ==
MI.getOpcode()))
9413 assert((Subtarget.hasDQI() ||
Domain >= 3) &&
"Requires AVX-512DQ");
9414 table =
lookupAVX512(
MI.getOpcode(), dom, ReplaceableInstrsAVX512DQMasked);
9415 if (table &&
Domain == 3 && (dom == 1 || table[3] ==
MI.getOpcode()))
9418 assert(table &&
"Cannot change domain");
9444 case X86::DIVSDrm_Int:
9446 case X86::DIVSDrr_Int:
9448 case X86::DIVSSrm_Int:
9450 case X86::DIVSSrr_Int:
9456 case X86::SQRTSDm_Int:
9458 case X86::SQRTSDr_Int:
9460 case X86::SQRTSSm_Int:
9462 case X86::SQRTSSr_Int:
9466 case X86::VDIVPDYrm:
9467 case X86::VDIVPDYrr:
9470 case X86::VDIVPSYrm:
9471 case X86::VDIVPSYrr:
9473 case X86::VDIVSDrm_Int:
9475 case X86::VDIVSDrr_Int:
9477 case X86::VDIVSSrm_Int:
9479 case X86::VDIVSSrr_Int:
9482 case X86::VSQRTPDYm:
9483 case X86::VSQRTPDYr:
9486 case X86::VSQRTPSYm:
9487 case X86::VSQRTPSYr:
9489 case X86::VSQRTSDm_Int:
9491 case X86::VSQRTSDr_Int:
9493 case X86::VSQRTSSm_Int:
9495 case X86::VSQRTSSr_Int:
9497 case X86::VDIVPDZ128rm:
9498 case X86::VDIVPDZ128rmb:
9499 case X86::VDIVPDZ128rmbk:
9500 case X86::VDIVPDZ128rmbkz:
9501 case X86::VDIVPDZ128rmk:
9502 case X86::VDIVPDZ128rmkz:
9503 case X86::VDIVPDZ128rr:
9504 case X86::VDIVPDZ128rrk:
9505 case X86::VDIVPDZ128rrkz:
9506 case X86::VDIVPDZ256rm:
9507 case X86::VDIVPDZ256rmb:
9508 case X86::VDIVPDZ256rmbk:
9509 case X86::VDIVPDZ256rmbkz:
9510 case X86::VDIVPDZ256rmk:
9511 case X86::VDIVPDZ256rmkz:
9512 case X86::VDIVPDZ256rr:
9513 case X86::VDIVPDZ256rrk:
9514 case X86::VDIVPDZ256rrkz:
9515 case X86::VDIVPDZrrb:
9516 case X86::VDIVPDZrrbk:
9517 case X86::VDIVPDZrrbkz:
9518 case X86::VDIVPDZrm:
9519 case X86::VDIVPDZrmb:
9520 case X86::VDIVPDZrmbk:
9521 case X86::VDIVPDZrmbkz:
9522 case X86::VDIVPDZrmk:
9523 case X86::VDIVPDZrmkz:
9524 case X86::VDIVPDZrr:
9525 case X86::VDIVPDZrrk:
9526 case X86::VDIVPDZrrkz:
9527 case X86::VDIVPSZ128rm:
9528 case X86::VDIVPSZ128rmb:
9529 case X86::VDIVPSZ128rmbk:
9530 case X86::VDIVPSZ128rmbkz:
9531 case X86::VDIVPSZ128rmk:
9532 case X86::VDIVPSZ128rmkz:
9533 case X86::VDIVPSZ128rr:
9534 case X86::VDIVPSZ128rrk:
9535 case X86::VDIVPSZ128rrkz:
9536 case X86::VDIVPSZ256rm:
9537 case X86::VDIVPSZ256rmb:
9538 case X86::VDIVPSZ256rmbk:
9539 case X86::VDIVPSZ256rmbkz:
9540 case X86::VDIVPSZ256rmk:
9541 case X86::VDIVPSZ256rmkz:
9542 case X86::VDIVPSZ256rr:
9543 case X86::VDIVPSZ256rrk:
9544 case X86::VDIVPSZ256rrkz:
9545 case X86::VDIVPSZrrb:
9546 case X86::VDIVPSZrrbk:
9547 case X86::VDIVPSZrrbkz:
9548 case X86::VDIVPSZrm:
9549 case X86::VDIVPSZrmb:
9550 case X86::VDIVPSZrmbk:
9551 case X86::VDIVPSZrmbkz:
9552 case X86::VDIVPSZrmk:
9553 case X86::VDIVPSZrmkz:
9554 case X86::VDIVPSZrr:
9555 case X86::VDIVPSZrrk:
9556 case X86::VDIVPSZrrkz:
9557 case X86::VDIVSDZrm:
9558 case X86::VDIVSDZrr:
9559 case X86::VDIVSDZrm_Int:
9560 case X86::VDIVSDZrmk_Int:
9561 case X86::VDIVSDZrmkz_Int:
9562 case X86::VDIVSDZrr_Int:
9563 case X86::VDIVSDZrrk_Int:
9564 case X86::VDIVSDZrrkz_Int:
9565 case X86::VDIVSDZrrb_Int:
9566 case X86::VDIVSDZrrbk_Int:
9567 case X86::VDIVSDZrrbkz_Int:
9568 case X86::VDIVSSZrm:
9569 case X86::VDIVSSZrr:
9570 case X86::VDIVSSZrm_Int:
9571 case X86::VDIVSSZrmk_Int:
9572 case X86::VDIVSSZrmkz_Int:
9573 case X86::VDIVSSZrr_Int:
9574 case X86::VDIVSSZrrk_Int:
9575 case X86::VDIVSSZrrkz_Int:
9576 case X86::VDIVSSZrrb_Int:
9577 case X86::VDIVSSZrrbk_Int:
9578 case X86::VDIVSSZrrbkz_Int:
9579 case X86::VSQRTPDZ128m:
9580 case X86::VSQRTPDZ128mb:
9581 case X86::VSQRTPDZ128mbk:
9582 case X86::VSQRTPDZ128mbkz:
9583 case X86::VSQRTPDZ128mk:
9584 case X86::VSQRTPDZ128mkz:
9585 case X86::VSQRTPDZ128r:
9586 case X86::VSQRTPDZ128rk:
9587 case X86::VSQRTPDZ128rkz:
9588 case X86::VSQRTPDZ256m:
9589 case X86::VSQRTPDZ256mb:
9590 case X86::VSQRTPDZ256mbk:
9591 case X86::VSQRTPDZ256mbkz:
9592 case X86::VSQRTPDZ256mk:
9593 case X86::VSQRTPDZ256mkz:
9594 case X86::VSQRTPDZ256r:
9595 case X86::VSQRTPDZ256rk:
9596 case X86::VSQRTPDZ256rkz:
9597 case X86::VSQRTPDZm:
9598 case X86::VSQRTPDZmb:
9599 case X86::VSQRTPDZmbk:
9600 case X86::VSQRTPDZmbkz:
9601 case X86::VSQRTPDZmk:
9602 case X86::VSQRTPDZmkz:
9603 case X86::VSQRTPDZr:
9604 case X86::VSQRTPDZrb:
9605 case X86::VSQRTPDZrbk:
9606 case X86::VSQRTPDZrbkz:
9607 case X86::VSQRTPDZrk:
9608 case X86::VSQRTPDZrkz:
9609 case X86::VSQRTPSZ128m:
9610 case X86::VSQRTPSZ128mb:
9611 case X86::VSQRTPSZ128mbk:
9612 case X86::VSQRTPSZ128mbkz:
9613 case X86::VSQRTPSZ128mk:
9614 case X86::VSQRTPSZ128mkz:
9615 case X86::VSQRTPSZ128r:
9616 case X86::VSQRTPSZ128rk:
9617 case X86::VSQRTPSZ128rkz:
9618 case X86::VSQRTPSZ256m:
9619 case X86::VSQRTPSZ256mb:
9620 case X86::VSQRTPSZ256mbk:
9621 case X86::VSQRTPSZ256mbkz:
9622 case X86::VSQRTPSZ256mk:
9623 case X86::VSQRTPSZ256mkz:
9624 case X86::VSQRTPSZ256r:
9625 case X86::VSQRTPSZ256rk:
9626 case X86::VSQRTPSZ256rkz:
9627 case X86::VSQRTPSZm:
9628 case X86::VSQRTPSZmb:
9629 case X86::VSQRTPSZmbk:
9630 case X86::VSQRTPSZmbkz:
9631 case X86::VSQRTPSZmk:
9632 case X86::VSQRTPSZmkz:
9633 case X86::VSQRTPSZr:
9634 case X86::VSQRTPSZrb:
9635 case X86::VSQRTPSZrbk:
9636 case X86::VSQRTPSZrbkz:
9637 case X86::VSQRTPSZrk:
9638 case X86::VSQRTPSZrkz:
9639 case X86::VSQRTSDZm:
9640 case X86::VSQRTSDZm_Int:
9641 case X86::VSQRTSDZmk_Int:
9642 case X86::VSQRTSDZmkz_Int:
9643 case X86::VSQRTSDZr:
9644 case X86::VSQRTSDZr_Int:
9645 case X86::VSQRTSDZrk_Int:
9646 case X86::VSQRTSDZrkz_Int:
9647 case X86::VSQRTSDZrb_Int:
9648 case X86::VSQRTSDZrbk_Int:
9649 case X86::VSQRTSDZrbkz_Int:
9650 case X86::VSQRTSSZm:
9651 case X86::VSQRTSSZm_Int:
9652 case X86::VSQRTSSZmk_Int:
9653 case X86::VSQRTSSZmkz_Int:
9654 case X86::VSQRTSSZr:
9655 case X86::VSQRTSSZr_Int:
9656 case X86::VSQRTSSZrk_Int:
9657 case X86::VSQRTSSZrkz_Int:
9658 case X86::VSQRTSSZrb_Int:
9659 case X86::VSQRTSSZrbk_Int:
9660 case X86::VSQRTSSZrbkz_Int:
9662 case X86::VGATHERDPDYrm:
9663 case X86::VGATHERDPDZ128rm:
9664 case X86::VGATHERDPDZ256rm:
9665 case X86::VGATHERDPDZrm:
9666 case X86::VGATHERDPDrm:
9667 case X86::VGATHERDPSYrm:
9668 case X86::VGATHERDPSZ128rm:
9669 case X86::VGATHERDPSZ256rm:
9670 case X86::VGATHERDPSZrm:
9671 case X86::VGATHERDPSrm:
9672 case X86::VGATHERPF0DPDm:
9673 case X86::VGATHERPF0DPSm:
9674 case X86::VGATHERPF0QPDm:
9675 case X86::VGATHERPF0QPSm:
9676 case X86::VGATHERPF1DPDm:
9677 case X86::VGATHERPF1DPSm:
9678 case X86::VGATHERPF1QPDm:
9679 case X86::VGATHERPF1QPSm:
9680 case X86::VGATHERQPDYrm:
9681 case X86::VGATHERQPDZ128rm:
9682 case X86::VGATHERQPDZ256rm:
9683 case X86::VGATHERQPDZrm:
9684 case X86::VGATHERQPDrm:
9685 case X86::VGATHERQPSYrm:
9686 case X86::VGATHERQPSZ128rm:
9687 case X86::VGATHERQPSZ256rm:
9688 case X86::VGATHERQPSZrm:
9689 case X86::VGATHERQPSrm:
9690 case X86::VPGATHERDDYrm:
9691 case X86::VPGATHERDDZ128rm:
9692 case X86::VPGATHERDDZ256rm:
9693 case X86::VPGATHERDDZrm:
9694 case X86::VPGATHERDDrm:
9695 case X86::VPGATHERDQYrm:
9696 case X86::VPGATHERDQZ128rm:
9697 case X86::VPGATHERDQZ256rm:
9698 case X86::VPGATHERDQZrm:
9699 case X86::VPGATHERDQrm:
9700 case X86::VPGATHERQDYrm:
9701 case X86::VPGATHERQDZ128rm:
9702 case X86::VPGATHERQDZ256rm:
9703 case X86::VPGATHERQDZrm:
9704 case X86::VPGATHERQDrm:
9705 case X86::VPGATHERQQYrm:
9706 case X86::VPGATHERQQZ128rm:
9707 case X86::VPGATHERQQZ256rm:
9708 case X86::VPGATHERQQZrm:
9709 case X86::VPGATHERQQrm:
9710 case X86::VSCATTERDPDZ128mr:
9711 case X86::VSCATTERDPDZ256mr:
9712 case X86::VSCATTERDPDZmr:
9713 case X86::VSCATTERDPSZ128mr:
9714 case X86::VSCATTERDPSZ256mr:
9715 case X86::VSCATTERDPSZmr:
9716 case X86::VSCATTERPF0DPDm:
9717 case X86::VSCATTERPF0DPSm:
9718 case X86::VSCATTERPF0QPDm:
9719 case X86::VSCATTERPF0QPSm:
9720 case X86::VSCATTERPF1DPDm:
9721 case X86::VSCATTERPF1DPSm:
9722 case X86::VSCATTERPF1QPDm:
9723 case X86::VSCATTERPF1QPSm:
9724 case X86::VSCATTERQPDZ128mr:
9725 case X86::VSCATTERQPDZ256mr:
9726 case X86::VSCATTERQPDZmr:
9727 case X86::VSCATTERQPSZ128mr:
9728 case X86::VSCATTERQPSZ256mr:
9729 case X86::VSCATTERQPSZmr:
9730 case X86::VPSCATTERDDZ128mr:
9731 case X86::VPSCATTERDDZ256mr:
9732 case X86::VPSCATTERDDZmr:
9733 case X86::VPSCATTERDQZ128mr:
9734 case X86::VPSCATTERDQZ256mr:
9735 case X86::VPSCATTERDQZmr:
9736 case X86::VPSCATTERQDZ128mr:
9737 case X86::VPSCATTERQDZ256mr:
9738 case X86::VPSCATTERQDZmr:
9739 case X86::VPSCATTERQQZ128mr:
9740 case X86::VPSCATTERQQZ256mr:
9741 case X86::VPSCATTERQQZmr:
9751 unsigned UseIdx)
const {
9758 Inst.
getNumDefs() <= 2 &&
"Reassociation needs binary operators");
9768 assert((Inst.
getNumDefs() == 1 || FlagDef) &&
"Implicit def isn't flags?");
9769 if (FlagDef && !FlagDef->
isDead())
9780 bool Invert)
const {
9832 case X86::VPANDDZ128rr:
9833 case X86::VPANDDZ256rr:
9834 case X86::VPANDDZrr:
9835 case X86::VPANDQZ128rr:
9836 case X86::VPANDQZ256rr:
9837 case X86::VPANDQZrr:
9840 case X86::VPORDZ128rr:
9841 case X86::VPORDZ256rr:
9843 case X86::VPORQZ128rr:
9844 case X86::VPORQZ256rr:
9848 case X86::VPXORDZ128rr:
9849 case X86::VPXORDZ256rr:
9850 case X86::VPXORDZrr:
9851 case X86::VPXORQZ128rr:
9852 case X86::VPXORQZ256rr:
9853 case X86::VPXORQZrr:
9856 case X86::VANDPDYrr:
9857 case X86::VANDPSYrr:
9858 case X86::VANDPDZ128rr:
9859 case X86::VANDPSZ128rr:
9860 case X86::VANDPDZ256rr:
9861 case X86::VANDPSZ256rr:
9862 case X86::VANDPDZrr:
9863 case X86::VANDPSZrr:
9868 case X86::VORPDZ128rr:
9869 case X86::VORPSZ128rr:
9870 case X86::VORPDZ256rr:
9871 case X86::VORPSZ256rr:
9876 case X86::VXORPDYrr:
9877 case X86::VXORPSYrr:
9878 case X86::VXORPDZ128rr:
9879 case X86::VXORPSZ128rr:
9880 case X86::VXORPDZ256rr:
9881 case X86::VXORPSZ256rr:
9882 case X86::VXORPDZrr:
9883 case X86::VXORPSZrr:
9904 case X86::VPADDBYrr:
9905 case X86::VPADDWYrr:
9906 case X86::VPADDDYrr:
9907 case X86::VPADDQYrr:
9908 case X86::VPADDBZ128rr:
9909 case X86::VPADDWZ128rr:
9910 case X86::VPADDDZ128rr:
9911 case X86::VPADDQZ128rr:
9912 case X86::VPADDBZ256rr:
9913 case X86::VPADDWZ256rr:
9914 case X86::VPADDDZ256rr:
9915 case X86::VPADDQZ256rr:
9916 case X86::VPADDBZrr:
9917 case X86::VPADDWZrr:
9918 case X86::VPADDDZrr:
9919 case X86::VPADDQZrr:
9920 case X86::VPMULLWrr:
9921 case X86::VPMULLWYrr:
9922 case X86::VPMULLWZ128rr:
9923 case X86::VPMULLWZ256rr:
9924 case X86::VPMULLWZrr:
9925 case X86::VPMULLDrr:
9926 case X86::VPMULLDYrr:
9927 case X86::VPMULLDZ128rr:
9928 case X86::VPMULLDZ256rr:
9929 case X86::VPMULLDZrr:
9930 case X86::VPMULLQZ128rr:
9931 case X86::VPMULLQZ256rr:
9932 case X86::VPMULLQZrr:
9933 case X86::VPMAXSBrr:
9934 case X86::VPMAXSBYrr:
9935 case X86::VPMAXSBZ128rr:
9936 case X86::VPMAXSBZ256rr:
9937 case X86::VPMAXSBZrr:
9938 case X86::VPMAXSDrr:
9939 case X86::VPMAXSDYrr:
9940 case X86::VPMAXSDZ128rr:
9941 case X86::VPMAXSDZ256rr:
9942 case X86::VPMAXSDZrr:
9943 case X86::VPMAXSQZ128rr:
9944 case X86::VPMAXSQZ256rr:
9945 case X86::VPMAXSQZrr:
9946 case X86::VPMAXSWrr:
9947 case X86::VPMAXSWYrr:
9948 case X86::VPMAXSWZ128rr:
9949 case X86::VPMAXSWZ256rr:
9950 case X86::VPMAXSWZrr:
9951 case X86::VPMAXUBrr:
9952 case X86::VPMAXUBYrr:
9953 case X86::VPMAXUBZ128rr:
9954 case X86::VPMAXUBZ256rr:
9955 case X86::VPMAXUBZrr:
9956 case X86::VPMAXUDrr:
9957 case X86::VPMAXUDYrr:
9958 case X86::VPMAXUDZ128rr:
9959 case X86::VPMAXUDZ256rr:
9960 case X86::VPMAXUDZrr:
9961 case X86::VPMAXUQZ128rr:
9962 case X86::VPMAXUQZ256rr:
9963 case X86::VPMAXUQZrr:
9964 case X86::VPMAXUWrr:
9965 case X86::VPMAXUWYrr:
9966 case X86::VPMAXUWZ128rr:
9967 case X86::VPMAXUWZ256rr:
9968 case X86::VPMAXUWZrr:
9969 case X86::VPMINSBrr:
9970 case X86::VPMINSBYrr:
9971 case X86::VPMINSBZ128rr:
9972 case X86::VPMINSBZ256rr:
9973 case X86::VPMINSBZrr:
9974 case X86::VPMINSDrr:
9975 case X86::VPMINSDYrr:
9976 case X86::VPMINSDZ128rr:
9977 case X86::VPMINSDZ256rr:
9978 case X86::VPMINSDZrr:
9979 case X86::VPMINSQZ128rr:
9980 case X86::VPMINSQZ256rr:
9981 case X86::VPMINSQZrr:
9982 case X86::VPMINSWrr:
9983 case X86::VPMINSWYrr:
9984 case X86::VPMINSWZ128rr:
9985 case X86::VPMINSWZ256rr:
9986 case X86::VPMINSWZrr:
9987 case X86::VPMINUBrr:
9988 case X86::VPMINUBYrr:
9989 case X86::VPMINUBZ128rr:
9990 case X86::VPMINUBZ256rr:
9991 case X86::VPMINUBZrr:
9992 case X86::VPMINUDrr:
9993 case X86::VPMINUDYrr:
9994 case X86::VPMINUDZ128rr:
9995 case X86::VPMINUDZ256rr:
9996 case X86::VPMINUDZrr:
9997 case X86::VPMINUQZ128rr:
9998 case X86::VPMINUQZ256rr:
9999 case X86::VPMINUQZrr:
10000 case X86::VPMINUWrr:
10001 case X86::VPMINUWYrr:
10002 case X86::VPMINUWZ128rr:
10003 case X86::VPMINUWZ256rr:
10004 case X86::VPMINUWZrr:
10008 case X86::MAXCPDrr:
10009 case X86::MAXCPSrr:
10010 case X86::MAXCSDrr:
10011 case X86::MAXCSSrr:
10012 case X86::MINCPDrr:
10013 case X86::MINCPSrr:
10014 case X86::MINCSDrr:
10015 case X86::MINCSSrr:
10016 case X86::VMAXCPDrr:
10017 case X86::VMAXCPSrr:
10018 case X86::VMAXCPDYrr:
10019 case X86::VMAXCPSYrr:
10020 case X86::VMAXCPDZ128rr:
10021 case X86::VMAXCPSZ128rr:
10022 case X86::VMAXCPDZ256rr:
10023 case X86::VMAXCPSZ256rr:
10024 case X86::VMAXCPDZrr:
10025 case X86::VMAXCPSZrr:
10026 case X86::VMAXCSDrr:
10027 case X86::VMAXCSSrr:
10028 case X86::VMAXCSDZrr:
10029 case X86::VMAXCSSZrr:
10030 case X86::VMINCPDrr:
10031 case X86::VMINCPSrr:
10032 case X86::VMINCPDYrr:
10033 case X86::VMINCPSYrr:
10034 case X86::VMINCPDZ128rr:
10035 case X86::VMINCPSZ128rr:
10036 case X86::VMINCPDZ256rr:
10037 case X86::VMINCPSZ256rr:
10038 case X86::VMINCPDZrr:
10039 case X86::VMINCPSZrr:
10040 case X86::VMINCSDrr:
10041 case X86::VMINCSSrr:
10042 case X86::VMINCSDZrr:
10043 case X86::VMINCSSZrr:
10044 case X86::VMAXCPHZ128rr:
10045 case X86::VMAXCPHZ256rr:
10046 case X86::VMAXCPHZrr:
10047 case X86::VMAXCSHZrr:
10048 case X86::VMINCPHZ128rr:
10049 case X86::VMINCPHZ256rr:
10050 case X86::VMINCPHZrr:
10051 case X86::VMINCSHZrr:
10061 case X86::VADDPDrr:
10062 case X86::VADDPSrr:
10063 case X86::VADDPDYrr:
10064 case X86::VADDPSYrr:
10065 case X86::VADDPDZ128rr:
10066 case X86::VADDPSZ128rr:
10067 case X86::VADDPDZ256rr:
10068 case X86::VADDPSZ256rr:
10069 case X86::VADDPDZrr:
10070 case X86::VADDPSZrr:
10071 case X86::VADDSDrr:
10072 case X86::VADDSSrr:
10073 case X86::VADDSDZrr:
10074 case X86::VADDSSZrr:
10075 case X86::VMULPDrr:
10076 case X86::VMULPSrr:
10077 case X86::VMULPDYrr:
10078 case X86::VMULPSYrr:
10079 case X86::VMULPDZ128rr:
10080 case X86::VMULPSZ128rr:
10081 case X86::VMULPDZ256rr:
10082 case X86::VMULPSZ256rr:
10083 case X86::VMULPDZrr:
10084 case X86::VMULPSZrr:
10085 case X86::VMULSDrr:
10086 case X86::VMULSSrr:
10087 case X86::VMULSDZrr:
10088 case X86::VMULSSZrr:
10089 case X86::VADDPHZ128rr:
10090 case X86::VADDPHZ256rr:
10091 case X86::VADDPHZrr:
10092 case X86::VADDSHZrr:
10093 case X86::VMULPHZ128rr:
10094 case X86::VMULPHZ256rr:
10095 case X86::VMULPHZrr:
10096 case X86::VMULSHZrr:
10107static std::optional<ParamLoadedValue>
10110 Register DestReg =
MI.getOperand(0).getReg();
10111 Register SrcReg =
MI.getOperand(1).getReg();
10116 if (DestReg == DescribedReg)
10121 if (
unsigned SubRegIdx =
TRI->getSubRegIndex(DestReg, DescribedReg)) {
10122 Register SrcSubReg =
TRI->getSubReg(SrcReg, SubRegIdx);
10132 if (
MI.getOpcode() == X86::MOV8rr ||
MI.getOpcode() == X86::MOV16rr ||
10133 !
TRI->isSuperRegister(DestReg, DescribedReg))
10134 return std::nullopt;
10136 assert(
MI.getOpcode() == X86::MOV32rr &&
"Unexpected super-register case");
10140std::optional<ParamLoadedValue>
10147 switch (
MI.getOpcode()) {
10150 case X86::LEA64_32r: {
10152 if (!
TRI->isSuperRegisterEq(
MI.getOperand(0).getReg(), Reg))
10153 return std::nullopt;
10157 if (!
MI.getOperand(4).isImm() || !
MI.getOperand(2).isImm())
10158 return std::nullopt;
10167 if ((Op1.
isReg() && Op1.
getReg() ==
MI.getOperand(0).getReg()) ||
10168 Op2.
getReg() ==
MI.getOperand(0).getReg())
10169 return std::nullopt;
10170 else if ((Op1.
isReg() && Op1.
getReg() != X86::NoRegister &&
10171 TRI->regsOverlap(Op1.
getReg(),
MI.getOperand(0).getReg())) ||
10172 (Op2.
getReg() != X86::NoRegister &&
10173 TRI->regsOverlap(Op2.
getReg(),
MI.getOperand(0).getReg())))
10174 return std::nullopt;
10176 int64_t Coef =
MI.getOperand(2).getImm();
10177 int64_t
Offset =
MI.getOperand(4).getImm();
10180 if ((Op1.
isReg() && Op1.
getReg() != X86::NoRegister)) {
10182 }
else if (Op1.
isFI())
10185 if (
Op &&
Op->isReg() &&
Op->getReg() == Op2.
getReg() && Coef > 0) {
10186 Ops.push_back(dwarf::DW_OP_constu);
10187 Ops.push_back(Coef + 1);
10188 Ops.push_back(dwarf::DW_OP_mul);
10190 if (
Op && Op2.
getReg() != X86::NoRegister) {
10191 int dwarfReg =
TRI->getDwarfRegNum(Op2.
getReg(),
false);
10193 return std::nullopt;
10194 else if (dwarfReg < 32) {
10195 Ops.push_back(dwarf::DW_OP_breg0 + dwarfReg);
10198 Ops.push_back(dwarf::DW_OP_bregx);
10199 Ops.push_back(dwarfReg);
10209 Ops.push_back(dwarf::DW_OP_constu);
10210 Ops.push_back(Coef);
10211 Ops.push_back(dwarf::DW_OP_mul);
10214 if (((Op1.
isReg() && Op1.
getReg() != X86::NoRegister) || Op1.
isFI()) &&
10215 Op2.
getReg() != X86::NoRegister) {
10216 Ops.push_back(dwarf::DW_OP_plus);
10228 return std::nullopt;
10231 case X86::MOV64ri32:
10234 if (!
TRI->isSuperRegisterEq(
MI.getOperand(0).getReg(), Reg))
10235 return std::nullopt;
10242 case X86::XOR32rr: {
10245 if (!
TRI->isSuperRegisterEq(
MI.getOperand(0).getReg(), Reg))
10246 return std::nullopt;
10247 if (
MI.getOperand(1).getReg() ==
MI.getOperand(2).getReg())
10249 return std::nullopt;
10251 case X86::MOVSX64rr32: {
10258 if (!
TRI->isSubRegisterEq(
MI.getOperand(0).getReg(), Reg))
10259 return std::nullopt;
10268 if (Reg ==
MI.getOperand(0).getReg())
10271 assert(X86MCRegisterClasses[X86::GR32RegClassID].
contains(Reg) &&
10272 "Unhandled sub-register case for MOVSX64rr32");
10277 assert(!
MI.isMoveImmediate() &&
"Unexpected MoveImm instruction");
10294 assert(!OldFlagDef1 == !OldFlagDef2 &&
10295 "Unexpected instruction type for reassociation");
10297 if (!OldFlagDef1 || !OldFlagDef2)
10301 "Must have dead EFLAGS operand in reassociable instruction");
10308 assert(NewFlagDef1 && NewFlagDef2 &&
10309 "Unexpected operand in reassociable instruction");
10319std::pair<unsigned, unsigned>
10321 return std::make_pair(TF, 0u);
10326 using namespace X86II;
10327 static const std::pair<unsigned, const char *> TargetFlags[] = {
10328 {MO_GOT_ABSOLUTE_ADDRESS,
"x86-got-absolute-address"},
10329 {MO_PIC_BASE_OFFSET,
"x86-pic-base-offset"},
10330 {MO_GOT,
"x86-got"},
10331 {MO_GOTOFF,
"x86-gotoff"},
10332 {MO_GOTPCREL,
"x86-gotpcrel"},
10333 {MO_GOTPCREL_NORELAX,
"x86-gotpcrel-norelax"},
10334 {MO_PLT,
"x86-plt"},
10335 {MO_TLSGD,
"x86-tlsgd"},
10336 {MO_TLSLD,
"x86-tlsld"},
10337 {MO_TLSLDM,
"x86-tlsldm"},
10338 {MO_GOTTPOFF,
"x86-gottpoff"},
10339 {MO_INDNTPOFF,
"x86-indntpoff"},
10340 {MO_TPOFF,
"x86-tpoff"},
10341 {MO_DTPOFF,
"x86-dtpoff"},
10342 {MO_NTPOFF,
"x86-ntpoff"},
10343 {MO_GOTNTPOFF,
"x86-gotntpoff"},
10344 {MO_DLLIMPORT,
"x86-dllimport"},
10345 {MO_DARWIN_NONLAZY,
"x86-darwin-nonlazy"},
10346 {MO_DARWIN_NONLAZY_PIC_BASE,
"x86-darwin-nonlazy-pic-base"},
10347 {MO_TLVP,
"x86-tlvp"},
10348 {MO_TLVP_PIC_BASE,
"x86-tlvp-pic-base"},
10349 {MO_SECREL,
"x86-secrel"},
10350 {MO_COFFSTUB,
"x86-coffstub"}};
10367 if (!TM->isPositionIndependent())
10374 if (GlobalBaseReg == 0)
10386 PC =
RegInfo.createVirtualRegister(&X86::GR32RegClass);
10388 PC = GlobalBaseReg;
10390 if (STI.is64Bit()) {
10443 StringRef getPassName()
const override {
10444 return "X86 PIC Global Base Reg Initialization";
10447 void getAnalysisUsage(AnalysisUsage &AU)
const override {
10462 bool runOnMachineFunction(MachineFunction &MF)
override {
10466 X86MachineFunctionInfo *MFI = MF.
getInfo<X86MachineFunctionInfo>();
10472 MachineDominatorTree *DT =
10473 &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
10483 MachineBasicBlock *BB =
Node->getBlock();
10489 switch (
I->getOpcode()) {
10490 case X86::TLS_base_addr32:
10491 case X86::TLS_base_addr64:
10492 if (TLSBaseAddrReg)
10493 I = ReplaceTLSBaseAddrCall(*
I, TLSBaseAddrReg);
10495 I = SetRegister(*
I, &TLSBaseAddrReg);
10504 for (
auto &
I : *Node) {
10505 Changed |= VisitNode(
I, TLSBaseAddrReg);
10513 MachineInstr *ReplaceTLSBaseAddrCall(MachineInstr &
I,
10515 MachineFunction *MF =
I.getParent()->getParent();
10516 const X86Subtarget &STI = MF->
getSubtarget<X86Subtarget>();
10517 const bool is64Bit = STI.is64Bit();
10521 MachineInstr *
Copy =
10523 TII->get(TargetOpcode::COPY),
is64Bit ? X86::RAX : X86::EAX)
10524 .
addReg(TLSBaseAddrReg);
10527 I.eraseFromParent();
10534 MachineInstr *SetRegister(MachineInstr &
I,
Register *TLSBaseAddrReg) {
10535 MachineFunction *MF =
I.getParent()->getParent();
10536 const X86Subtarget &STI = MF->
getSubtarget<X86Subtarget>();
10537 const bool is64Bit = STI.is64Bit();
10541 MachineRegisterInfo &RegInfo = MF->
getRegInfo();
10543 is64Bit ? &X86::GR64RegClass : &X86::GR32RegClass);
10546 MachineInstr *
Next =
I.getNextNode();
10548 TII->get(TargetOpcode::COPY), *TLSBaseAddrReg)
10554 StringRef getPassName()
const override {
10555 return "Local Dynamic TLS Access Clean-up";
10558 void getAnalysisUsage(AnalysisUsage &AU)
const override {
10560 AU.
addRequired<MachineDominatorTreeWrapperPass>();
10566char LDTLSCleanup::ID = 0;
10568 return new LDTLSCleanup();
10601std::optional<std::unique_ptr<outliner::OutlinedFunction>>
10604 std::vector<outliner::Candidate> &RepeatedSequenceLocs,
10605 unsigned MinRepeats)
const {
10606 unsigned SequenceSize = 0;
10607 for (
auto &
MI : RepeatedSequenceLocs[0]) {
10611 if (
MI.isDebugInstr() ||
MI.isKill())
10618 unsigned CFICount = 0;
10619 for (
auto &
I : RepeatedSequenceLocs[0]) {
10620 if (
I.isCFIInstruction())
10630 std::vector<MCCFIInstruction> CFIInstructions =
10631 C.getMF()->getFrameInstructions();
10633 if (CFICount > 0 && CFICount != CFIInstructions.size())
10634 return std::nullopt;
10638 if (RepeatedSequenceLocs[0].back().isTerminator()) {
10642 return std::make_unique<outliner::OutlinedFunction>(
10643 RepeatedSequenceLocs, SequenceSize,
10650 return std::nullopt;
10655 return std::make_unique<outliner::OutlinedFunction>(
10665 if (Subtarget.getFrameLowering()->has128ByteRedZone(MF)) {
10674 if (!OutlineFromLinkOnceODRs &&
F.hasLinkOnceODRLinkage())
10684 unsigned Flags)
const {
10688 if (
MI.isTerminator())
10702 if (
MI.modifiesRegister(X86::RSP, &RI) ||
MI.readsRegister(X86::RSP, &RI) ||
10703 MI.getDesc().hasImplicitUseOfPhysReg(X86::RSP) ||
10704 MI.getDesc().hasImplicitDefOfPhysReg(X86::RSP))
10708 if (
MI.readsRegister(X86::RIP, &RI) ||
10709 MI.getDesc().hasImplicitUseOfPhysReg(X86::RIP) ||
10710 MI.getDesc().hasImplicitDefOfPhysReg(X86::RIP))
10714 if (
MI.isCFIInstruction())
10730 MBB.insert(
MBB.end(), retq);
10740 .addGlobalAddress(M.getNamedValue(MF.
getName())));
10744 .addGlobalAddress(M.getNamedValue(MF.
getName())));
10753 bool AllowSideEffects)
const {
10758 if (ST.hasMMX() && X86::VR64RegClass.contains(Reg))
10762 if (
TRI.isGeneralPurposeRegister(MF, Reg)) {
10767 if (!AllowSideEffects)
10774 }
else if (X86::VR128RegClass.
contains(Reg)) {
10783 }
else if (X86::VR256RegClass.
contains(Reg)) {
10792 }
else if (X86::VR512RegClass.
contains(Reg)) {
10794 if (!ST.hasAVX512())
10801 }
else if (X86::VK1RegClass.
contains(Reg) || X86::VK2RegClass.
contains(Reg) ||
10803 X86::VK16RegClass.
contains(Reg)) {
10808 unsigned Op = ST.hasBWI() ? X86::KXORQkk : X86::KXORWkk;
10817 bool DoRegPressureReduce)
const {
10820 case X86::VPDPWSSDrr:
10821 case X86::VPDPWSSDrm:
10822 case X86::VPDPWSSDYrr:
10823 case X86::VPDPWSSDYrm: {
10824 if (!Subtarget.hasFastDPWSSD()) {
10830 case X86::VPDPWSSDZ128rr:
10831 case X86::VPDPWSSDZ128rm:
10832 case X86::VPDPWSSDZ256rr:
10833 case X86::VPDPWSSDZ256rm:
10834 case X86::VPDPWSSDZrr:
10835 case X86::VPDPWSSDZrm: {
10836 if (Subtarget.hasBWI() && !Subtarget.hasFastDPWSSD()) {
10844 Patterns, DoRegPressureReduce);
10856 unsigned AddOpc = 0;
10857 unsigned MaddOpc = 0;
10860 assert(
false &&
"It should not reach here");
10866 case X86::VPDPWSSDrr:
10867 MaddOpc = X86::VPMADDWDrr;
10868 AddOpc = X86::VPADDDrr;
10870 case X86::VPDPWSSDrm:
10871 MaddOpc = X86::VPMADDWDrm;
10872 AddOpc = X86::VPADDDrr;
10874 case X86::VPDPWSSDZ128rr:
10875 MaddOpc = X86::VPMADDWDZ128rr;
10876 AddOpc = X86::VPADDDZ128rr;
10878 case X86::VPDPWSSDZ128rm:
10879 MaddOpc = X86::VPMADDWDZ128rm;
10880 AddOpc = X86::VPADDDZ128rr;
10886 case X86::VPDPWSSDYrr:
10887 MaddOpc = X86::VPMADDWDYrr;
10888 AddOpc = X86::VPADDDYrr;
10890 case X86::VPDPWSSDYrm:
10891 MaddOpc = X86::VPMADDWDYrm;
10892 AddOpc = X86::VPADDDYrr;
10894 case X86::VPDPWSSDZ256rr:
10895 MaddOpc = X86::VPMADDWDZ256rr;
10896 AddOpc = X86::VPADDDZ256rr;
10898 case X86::VPDPWSSDZ256rm:
10899 MaddOpc = X86::VPMADDWDZ256rm;
10900 AddOpc = X86::VPADDDZ256rr;
10906 case X86::VPDPWSSDZrr:
10907 MaddOpc = X86::VPMADDWDZrr;
10908 AddOpc = X86::VPADDDZrr;
10910 case X86::VPDPWSSDZrm:
10911 MaddOpc = X86::VPMADDWDZrm;
10912 AddOpc = X86::VPADDDZrr;
10924 InstrIdxForVirtReg.
insert(std::make_pair(NewReg, 0));
10946 DelInstrs, InstrIdxForVirtReg);
10950 InstrIdxForVirtReg);
10960 M.Base.FrameIndex = FI;
10961 M.getFullAddress(
Ops);
10964#define GET_INSTRINFO_HELPERS
10965#include "X86GenInstrInfo.inc"
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
MachineOutlinerClass
Constants defining how certain sequences should be outlined.
@ MachineOutlinerTailCall
Emit a save, restore, call, and return.
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
DXIL Forward Handle Accesses
const HexagonInstrInfo * TII
Module.h This file contains the declarations for the Module class.
static bool lookup(const GsymReader &GR, DataExtractor &Data, uint64_t &Offset, uint64_t BaseAddr, uint64_t Addr, SourceLocations &SrcLocs, llvm::Error &Err)
A Lookup helper functions.
const size_t AbstractManglingParser< Derived, Alloc >::NumOps
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
static bool Expand2AddrUndef(MachineInstrBuilder &MIB, const MCInstrDesc &Desc)
Expand a single-def pseudo instruction to a two-addr instruction with two undef reads of the register...
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
Register const TargetRegisterInfo * TRI
Promote Memory to Register
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
const SmallVectorImpl< MachineOperand > & Cond
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Provides some synthesis utilities to produce sequences of values.
static SPCC::CondCodes GetOppositeBranchCondition(SPCC::CondCodes CC)
#define FROM_TO(FROM, TO)
cl::opt< bool > X86EnableAPXForRelocation
static bool is64Bit(const char *name)
#define GET_EGPR_IF_ENABLED(OPC)
static bool isLEA(unsigned Opcode)
static void addOperands(MachineInstrBuilder &MIB, ArrayRef< MachineOperand > MOs, int PtrOffset=0)
static std::optional< ParamLoadedValue > describeMOVrrLoadedValue(const MachineInstr &MI, Register DescribedReg, const TargetRegisterInfo *TRI)
If DescribedReg overlaps with the MOVrr instruction's destination register then, if possible,...
static cl::opt< unsigned > PartialRegUpdateClearance("partial-reg-update-clearance", cl::desc("Clearance between two register writes " "for inserting XOR to avoid partial " "register update"), cl::init(64), cl::Hidden)
static bool shouldPreventUndefRegUpdateMemFold(MachineFunction &MF, MachineInstr &MI)
static unsigned CopyToFromAsymmetricReg(Register DestReg, Register SrcReg, const X86Subtarget &Subtarget)
static bool isConvertibleLEA(MachineInstr *MI)
static bool ExpandMOVImmSExti8(MachineInstrBuilder &MIB, const TargetInstrInfo &TII, const X86Subtarget &Subtarget)
static bool isAMXOpcode(unsigned Opc)
static int getJumpTableIndexFromReg(const MachineRegisterInfo &MRI, Register Reg)
static void updateOperandRegConstraints(MachineFunction &MF, MachineInstr &NewMI, const TargetInstrInfo &TII)
static int getJumpTableIndexFromAddr(const MachineInstr &MI)
static bool AdjustBlendMask(unsigned OldMask, unsigned OldWidth, unsigned NewWidth, unsigned *pNewMask=nullptr)
static bool expandMOV32r1(MachineInstrBuilder &MIB, const TargetInstrInfo &TII, bool MinusOne)
static unsigned getNewOpcFromTable(ArrayRef< X86TableEntry > Table, unsigned Opc)
static unsigned getStoreRegOpcode(Register SrcReg, const TargetRegisterClass *RC, bool IsStackAligned, const X86Subtarget &STI)
#define FOLD_BROADCAST(SIZE)
static cl::opt< unsigned > UndefRegClearance("undef-reg-clearance", cl::desc("How many idle instructions we would like before " "certain undef register reads"), cl::init(128), cl::Hidden)
#define CASE_BCAST_TYPE_OPC(TYPE, OP16, OP32, OP64)
static bool isTruncatedShiftCountForLEA(unsigned ShAmt)
Check whether the given shift count is appropriate can be represented by a LEA instruction.
static cl::opt< bool > ReMatPICStubLoad("remat-pic-stub-load", cl::desc("Re-materialize load from stub in PIC mode"), cl::init(false), cl::Hidden)
static SmallVector< MachineMemOperand *, 2 > extractLoadMMOs(ArrayRef< MachineMemOperand * > MMOs, MachineFunction &MF)
static MachineInstr * fuseTwoAddrInst(MachineFunction &MF, unsigned Opcode, ArrayRef< MachineOperand > MOs, MachineBasicBlock::iterator InsertPt, MachineInstr &MI, const TargetInstrInfo &TII)
static void printFailMsgforFold(const MachineInstr &MI, unsigned Idx)
static bool canConvert2Copy(unsigned Opc)
static cl::opt< bool > NoFusing("disable-spill-fusing", cl::desc("Disable fusing of spill code into instructions"), cl::Hidden)
static bool expandNOVLXStore(MachineInstrBuilder &MIB, const TargetRegisterInfo *TRI, const MCInstrDesc &StoreDesc, const MCInstrDesc &ExtractDesc, unsigned SubIdx)
static bool isX87Reg(Register Reg)
Return true if the Reg is X87 register.
static bool Expand2AddrKreg(MachineInstrBuilder &MIB, const MCInstrDesc &Desc, Register Reg)
Expand a single-def pseudo instruction to a two-addr instruction with two k0 reads.
static bool isFrameLoadOpcode(int Opcode, TypeSize &MemBytes)
#define VPERM_CASES_BROADCAST(Suffix)
static std::pair< X86::CondCode, unsigned > isUseDefConvertible(const MachineInstr &MI)
Check whether the use can be converted to remove a comparison against zero.
static bool findRedundantFlagInstr(MachineInstr &CmpInstr, MachineInstr &CmpValDefInstr, const MachineRegisterInfo *MRI, MachineInstr **AndInstr, const TargetRegisterInfo *TRI, const X86Subtarget &ST, bool &NoSignFlag, bool &ClearsOverflowFlag)
static bool expandSHXDROT(MachineInstrBuilder &MIB, const MCInstrDesc &Desc)
static unsigned getLoadRegOpcode(Register DestReg, const TargetRegisterClass *RC, bool IsStackAligned, const X86Subtarget &STI)
static void expandLoadStackGuard(MachineInstrBuilder &MIB, const TargetInstrInfo &TII)
static bool hasUndefRegUpdate(unsigned Opcode, unsigned OpNum, bool ForLoadFold=false)
static MachineInstr * makeM0Inst(const TargetInstrInfo &TII, unsigned Opcode, ArrayRef< MachineOperand > MOs, MachineBasicBlock::iterator InsertPt, MachineInstr &MI)
#define GET_ND_IF_ENABLED(OPC)
static bool expandMOVSHP(MachineInstrBuilder &MIB, MachineInstr &MI, const TargetInstrInfo &TII, bool HasAVX)
static bool hasPartialRegUpdate(unsigned Opcode, const X86Subtarget &Subtarget, bool ForLoadFold=false)
Return true for all instructions that only update the first 32 or 64-bits of the destination register...
static const uint16_t * lookupAVX512(unsigned opcode, unsigned domain, ArrayRef< uint16_t[4]> Table)
static unsigned getLoadStoreRegOpcode(Register Reg, const TargetRegisterClass *RC, bool IsStackAligned, const X86Subtarget &STI, bool Load)
#define VPERM_CASES(Suffix)
#define FROM_TO_SIZE(A, B, S)
static void commuteVPTERNLOG(MachineInstr &MI, unsigned SrcOpIdx1, unsigned SrcOpIdx2)
static bool isDefConvertible(const MachineInstr &MI, bool &NoSignFlag, bool &ClearsOverflowFlag)
Check whether the definition can be converted to remove a comparison against zero.
static MachineInstr * fuseInst(MachineFunction &MF, unsigned Opcode, unsigned OpNo, ArrayRef< MachineOperand > MOs, MachineBasicBlock::iterator InsertPt, MachineInstr &MI, const TargetInstrInfo &TII, int PtrOffset=0)
static X86::CondCode getSwappedCondition(X86::CondCode CC)
Assuming the flags are set by MI(a,b), return the condition code if we modify the instructions such t...
static unsigned getCommutedVPERMV3Opcode(unsigned Opcode)
static bool expandXorFP(MachineInstrBuilder &MIB, const TargetInstrInfo &TII)
static MachineBasicBlock * getFallThroughMBB(MachineBasicBlock *MBB, MachineBasicBlock *TBB)
static bool isNonFoldablePartialRegisterLoad(const MachineInstr &LoadMI, const MachineInstr &UserMI, const MachineFunction &MF)
Check if LoadMI is a partial register load that we can't fold into MI because the latter uses content...
static unsigned getLoadStoreOpcodeForFP16(bool Load, const X86Subtarget &STI)
static bool isHReg(Register Reg)
Test if the given register is a physical h register.
static cl::opt< bool > PrintFailedFusing("print-failed-fuse-candidates", cl::desc("Print instructions that the allocator wants to" " fuse, but the X86 backend currently can't"), cl::Hidden)
static bool expandNOVLXLoad(MachineInstrBuilder &MIB, const TargetRegisterInfo *TRI, const MCInstrDesc &LoadDesc, const MCInstrDesc &BroadcastDesc, unsigned SubIdx)
static void genAlternativeDpCodeSequence(MachineInstr &Root, const TargetInstrInfo &TII, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< Register, unsigned > &InstrIdxForVirtReg)
static unsigned getThreeSrcCommuteCase(uint64_t TSFlags, unsigned SrcOpIdx1, unsigned SrcOpIdx2)
This determines which of three possible cases of a three source commute the source indexes correspond...
static bool isFrameStoreOpcode(int Opcode, TypeSize &MemBytes)
static unsigned getTruncatedShiftCount(const MachineInstr &MI, unsigned ShiftAmtOperandIdx)
Check whether the shift count for a machine operand is non-zero.
static SmallVector< MachineMemOperand *, 2 > extractStoreMMOs(ArrayRef< MachineMemOperand * > MMOs, MachineFunction &MF)
static unsigned getBroadcastOpcode(const X86FoldTableEntry *I, const TargetRegisterClass *RC, const X86Subtarget &STI)
static unsigned convertALUrr2ALUri(unsigned Opc)
Convert an ALUrr opcode to corresponding ALUri opcode.
static bool regIsPICBase(Register BaseReg, const MachineRegisterInfo &MRI)
Return true if register is PIC base; i.e.g defined by X86::MOVPC32r.
static bool isCommutableVPERMV3Instruction(unsigned Opcode)
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
AnalysisUsage & addRequired()
LLVM_ABI void setPreservesCFG()
This function should be called by the pass, iff they do not:
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
@ ICMP_SLT
signed less than
@ ICMP_SLE
signed less or equal
@ FCMP_OLT
0 1 0 0 True if ordered and less than
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
@ ICMP_UGE
unsigned greater or equal
@ ICMP_UGT
unsigned greater than
@ ICMP_SGT
signed greater than
@ FCMP_ULT
1 1 0 0 True if unordered or less than
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
@ ICMP_ULT
unsigned less than
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
@ ICMP_SGE
signed greater or equal
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
@ ICMP_ULE
unsigned less or equal
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
This is an important base class in LLVM.
static LLVM_ABI Constant * getAllOnesValue(Type *Ty)
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
static LLVM_ABI void appendOffset(SmallVectorImpl< uint64_t > &Ops, int64_t Offset)
Append Ops with operations to apply the Offset.
static LLVM_ABI DIExpression * appendExt(const DIExpression *Expr, unsigned FromSize, unsigned ToSize, bool Signed)
Append a zero- or sign-extension to Expr.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
DomTreeNodeBase< NodeT > * getRootNode()
getRootNode - This returns the entry node for the CFG of the function.
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
FunctionPass class - This class is used to implement most global optimizations.
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
LiveInterval - This class represents the liveness of a register, or stack slot.
SlotIndex InsertMachineInstrInMaps(MachineInstr &MI)
SlotIndex getInstructionIndex(const MachineInstr &Instr) const
Returns the base index of the given instruction.
LiveInterval & getInterval(Register Reg)
SlotIndex ReplaceMachineInstrInMaps(MachineInstr &MI, MachineInstr &NewMI)
A set of physical registers with utility functions to track liveness when walking backward/forward th...
const Segment * getSegmentContaining(SlotIndex Idx) const
Return the segment that contains the specified index, or null if there is none.
LLVM_ABI void replaceKillInstruction(Register Reg, MachineInstr &OldMI, MachineInstr &NewMI)
replaceKillInstruction - Update register kill info by replacing a kill instruction with a new one.
LLVM_ABI VarInfo & getVarInfo(Register Reg)
getVarInfo - Return the VarInfo structure for the specified VIRTUAL register.
static LocationSize precise(uint64_t Value)
bool usesWindowsCFI() const
static MCCFIInstruction createAdjustCfaOffset(MCSymbol *L, int64_t Adjustment, SMLoc Loc={})
.cfi_adjust_cfa_offset Same as .cfi_def_cfa_offset, but Offset is a relative value that is added/subt...
Instances of this class represent a single low-level machine instruction.
void setOpcode(unsigned Op)
Describe properties that are true of each instruction in the target description file.
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
This holds information about one operand of a machine instruction, indicating the register class for ...
Wrapper class representing physical registers. Should be passed by value.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
MachineInstrBundleIterator< const MachineInstr > const_iterator
void push_back(MachineInstr *MI)
MachineInstr * remove(MachineInstr *I)
Remove the unbundled instruction from the instruction list without deleting it.
LLVM_ABI DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any debug instructions.
MachineInstrBundleIterator< MachineInstr, true > reverse_iterator
LLVM_ABI bool isLayoutSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB will be emitted immediately after this block, such that if this bloc...
LLVM_ABI void eraseFromParent()
This method unlinks 'this' from the containing function and deletes it.
LLVM_ABI instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
iterator_range< succ_iterator > successors()
MachineInstrBundleIterator< MachineInstr > iterator
@ LQR_Dead
Register is known to be fully dead.
This class is a data container for one entry in a MachineConstantPool.
union llvm::MachineConstantPoolEntry::@004270020304201266316354007027341142157160323045 Val
The constant itself.
bool isMachineConstantPoolEntry() const
isMachineConstantPoolEntry - Return true if the MachineConstantPoolEntry is indeed a target specific ...
const Constant * ConstVal
The MachineConstantPool class keeps track of constants referenced by a function which must be spilled...
unsigned getConstantPoolIndex(const Constant *C, Align Alignment)
getConstantPoolIndex - Create a new entry in the constant pool or return an existing one.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
MCSymbol * getPICBaseSymbol() const
getPICBaseSymbol - Return a function-local symbol to represent the PIC base.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
bool needsFrameMoves() const
True if this function needs frame moves for debug or exceptions.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
MachineConstantPool * getConstantPool()
getConstantPool - Return the constant pool object for the current function.
const MachineBasicBlock & front() const
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addDisp(const MachineOperand &Disp, int64_t off, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & copyImplicitOps(const MachineInstr &OtherMI) const
Copy all the implicit operands from OtherMI onto this one.
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
mop_iterator operands_begin()
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool isImplicitDef() const
const MachineBasicBlock * getParent() const
void dropDebugNumber()
Drop any variable location debugging information associated with this instruction.
LLVM_ABI void setPreInstrSymbol(MachineFunction &MF, MCSymbol *Symbol)
Set a symbol that will be emitted just prior to the instruction itself.
LLVM_ABI void addImplicitDefUseOperands(MachineFunction &MF)
Add all implicit def and use operands to this instruction.
bool getFlag(MIFlag Flag) const
Return whether an MI flag is set.
unsigned getNumOperands() const
Retuns the total number of operands.
LLVM_ABI void addOperand(MachineFunction &MF, const MachineOperand &Op)
Add the specified operand to the instruction.
LLVM_ABI unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
bool modifiesRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr modifies (fully define or partially define) the specified register.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
void untieRegOperand(unsigned OpIdx)
Break any tie involving OpIdx.
LLVM_ABI void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
LLVM_ABI unsigned getNumExplicitDefs() const
Returns the number of non-implicit definitions.
LLVM_ABI void eraseFromBundle()
Unlink 'this' from its basic block and delete it.
bool hasOneMemOperand() const
Return true if this instruction has exactly one MachineMemOperand.
LLVM_ABI void substituteRegister(Register FromReg, Register ToReg, unsigned SubIdx, const TargetRegisterInfo &RegInfo)
Replace all occurrences of FromReg with ToReg:SubIdx, properly composing subreg indices where necessa...
mmo_iterator memoperands_begin() const
Access to memory operands of the instruction.
LLVM_ABI bool isIdenticalTo(const MachineInstr &Other, MICheckType Check=CheckDefs) const
Return true if this instruction is identical to Other.
LLVM_ABI const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
void setFlag(MIFlag Flag)
Set a MI flag.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
LLVM_ABI void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
LLVM_ABI void removeOperand(unsigned OpNo)
Erase an operand from an instruction, leaving it with one fewer operand than it started with.
LLVM_ABI void dump() const
const MachineOperand & getOperand(unsigned i) const
unsigned getNumDefs() const
Returns the total number of definitions.
void setDebugLoc(DebugLoc DL)
Replace current source information with new such.
MachineOperand * findRegisterDefOperand(Register Reg, const TargetRegisterInfo *TRI, bool isDead=false, bool Overlap=false)
Wrapper for findRegisterDefOperandIdx, it returns a pointer to the MachineOperand rather than an inde...
A description of a memory reference used in the backend.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
This class contains meta information specific to a module.
MachineOperand class - Representation of each machine instruction operand.
void setSubReg(unsigned subReg)
unsigned getSubReg() const
void setImplicit(bool Val=true)
void setImm(int64_t immVal)
bool readsReg() const
readsReg - Returns true if this operand reads the previous value of its register.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
bool isCPI() const
isCPI - Tests if this is a MO_ConstantPoolIndex operand.
void setIsDead(bool Val=true)
LLVM_ABI void setReg(Register Reg)
Change the register this operand corresponds to.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
void setIsKill(bool Val=true)
bool isJTI() const
isJTI - Tests if this is a MO_JumpTableIndex operand.
LLVM_ABI void ChangeToRegister(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isDebug=false)
ChangeToRegister - Replace this operand with a new register operand of the specified value.
static MachineOperand CreateImm(int64_t Val)
void setIsUndef(bool Val=true)
Register getReg() const
getReg - Returns the register number.
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
LLVM_ABI bool isIdenticalTo(const MachineOperand &Other) const
Returns true if this operand is identical to the specified operand except for liveness related flags ...
static MachineOperand CreateCPI(unsigned Idx, int Offset, unsigned TargetFlags=0)
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
int64_t getOffset() const
Return the offset from the symbol in this operand.
static MachineOperand CreateFI(int Idx)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
LLVM_ABI Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
LLVM_ABI const TargetRegisterClass * constrainRegClass(Register Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
constrainRegClass - Constrain the register class of the specified virtual register to be a common sub...
A Module instance is used to store all the information related to an LLVM module.
Wrapper class representing virtual and physical registers.
constexpr bool isValid() const
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
LLVM_ABI void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
MachineFunction & getMachineFunction() const
SlotIndex - An opaque wrapper around machine indexes.
SlotIndex getBaseIndex() const
Returns the base index for associated with this index.
SlotIndex getRegSlot(bool EC=false) const
Returns the register use/def slot in the current instruction for a normal or early-clobber def.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
Information about stack frame layout on the target.
bool hasFP(const MachineFunction &MF) const
hasFP - Return true if the specified function should have a dedicated frame pointer register.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
TargetInstrInfo - Interface to description of machine instruction set.
virtual bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx1, unsigned &SrcOpIdx2) const
Returns true iff the routine could find two commutable operands in the given machine instruction.
virtual const TargetRegisterClass * getRegClass(const MCInstrDesc &MCID, unsigned OpNum, const TargetRegisterInfo *TRI) const
Given a machine instruction descriptor, returns the register class constraint for OpNum,...
virtual bool hasReassociableOperands(const MachineInstr &Inst, const MachineBasicBlock *MBB) const
Return true when \P Inst has reassociable operands in the same \P MBB.
virtual void genAlternativeCodeSequence(MachineInstr &Root, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< Register, unsigned > &InstIdxForVirtReg) const
When getMachineCombinerPatterns() finds patterns, this function generates the instructions that could...
virtual std::optional< ParamLoadedValue > describeLoadedValue(const MachineInstr &MI, Register Reg) const
Produce the expression describing the MI loading a value into the physical register Reg.
virtual bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce) const
Return true when there is potentially a faster code sequence for an instruction chain ending in Root.
virtual bool isReMaterializableImpl(const MachineInstr &MI) const
For instructions with opcodes for which the M_REMATERIALIZABLE flag is set, this hook lets the target...
virtual bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const
Test if the given instruction should be considered a scheduling boundary.
virtual MachineInstr * commuteInstructionImpl(MachineInstr &MI, bool NewMI, unsigned OpIdx1, unsigned OpIdx2) const
This method commutes the operands of the given machine instruction MI.
bool isPositionIndependent() const
CodeModel::Model getCodeModel() const
Returns the code model.
const MCAsmInfo * getMCAsmInfo() const
Return target specific asm information.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Provide an instruction scheduling machine model to CodeGen passes.
virtual const TargetFrameLowering * getFrameLowering() const
virtual const TargetRegisterInfo * getRegisterInfo() const =0
Return the target's register information.
Target - Wrapper for Target specific information.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
static constexpr TypeSize getZero()
The instances of the Type class are immutable: once they are created, they are never changed.
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
static LLVM_ABI Type * getFP128Ty(LLVMContext &C)
static LLVM_ABI Type * getDoubleTy(LLVMContext &C)
static LLVM_ABI Type * getFloatTy(LLVMContext &C)
static LLVM_ABI Type * getHalfTy(LLVMContext &C)
SlotIndex def
The index of the defining instruction.
LLVM Value Representation.
void BuildCFI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, const MCCFIInstruction &CFIInst, MachineInstr::MIFlag Flag=MachineInstr::NoFlags) const
Wraps up getting a CFI index and building a MachineInstr for it.
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg, MachineInstr::MIFlag Flags=MachineInstr::NoFlags) const override
void getFrameIndexOperands(SmallVectorImpl< MachineOperand > &Ops, int FI) const override
bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int64_t CmpMask, int64_t CmpValue, const MachineRegisterInfo *MRI) const override
Check if there exists an earlier instruction that operates on the same source operands and sets eflag...
bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce) const override
void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, Register DestReg, Register SrcReg, bool KillSrc, bool RenamableDest=false, bool RenamableSrc=false) const override
bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const override
Overrides the isSchedulingBoundary from Codegen/TargetInstrInfo.cpp to make it capable of identifying...
MachineBasicBlock::iterator insertOutlinedCall(Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It, MachineFunction &MF, outliner::Candidate &C) const override
void replaceBranchWithTailCall(MachineBasicBlock &MBB, SmallVectorImpl< MachineOperand > &Cond, const MachineInstr &TailCall) const override
void reMaterialize(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg, unsigned SubIdx, const MachineInstr &Orig, const TargetRegisterInfo &TRI) const override
bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify) const override
bool canInsertSelect(const MachineBasicBlock &, ArrayRef< MachineOperand > Cond, Register, Register, Register, int &, int &, int &) const override
void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, Register DstReg, ArrayRef< MachineOperand > Cond, Register TrueReg, Register FalseReg) const override
unsigned getOpcodeAfterMemoryUnfold(unsigned Opc, bool UnfoldLoad, bool UnfoldStore, unsigned *LoadRegIndex=nullptr) const override
bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx1, unsigned &SrcOpIdx2) const override
Returns true iff the routine could find two commutable operands in the given machine instruction.
bool areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2, int64_t &Offset1, int64_t &Offset2) const override
X86InstrInfo(const X86Subtarget &STI)
static bool isDataInvariantLoad(MachineInstr &MI)
Returns true if the instruction has no behavior (specified or otherwise) that is based on the value l...
MachineInstr * commuteInstructionImpl(MachineInstr &MI, bool NewMI, unsigned CommuteOpIdx1, unsigned CommuteOpIdx2) const override
bool isFunctionSafeToOutlineFrom(MachineFunction &MF, bool OutlineFromLinkOnceODRs) const override
const X86RegisterInfo & getRegisterInfo() const
getRegisterInfo - TargetInstrInfo is a superset of MRegister info.
bool hasCommutePreference(MachineInstr &MI, bool &Commute) const override
Returns true if we have preference on the operands order in MI, the commute decision is returned in C...
bool hasLiveCondCodeDef(MachineInstr &MI) const
True if MI has a condition code def, e.g.
std::optional< ParamLoadedValue > describeLoadedValue(const MachineInstr &MI, Register Reg) const override
bool canMakeTailCallConditional(SmallVectorImpl< MachineOperand > &Cond, const MachineInstr &TailCall) const override
bool getMemOperandsWithOffsetWidth(const MachineInstr &LdSt, SmallVectorImpl< const MachineOperand * > &BaseOps, int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width, const TargetRegisterInfo *TRI) const override
bool unfoldMemoryOperand(MachineFunction &MF, MachineInstr &MI, Register Reg, bool UnfoldLoad, bool UnfoldStore, SmallVectorImpl< MachineInstr * > &NewMIs) const override
std::optional< DestSourcePair > isCopyInstrImpl(const MachineInstr &MI) const override
MachineInstr * convertToThreeAddress(MachineInstr &MI, LiveVariables *LV, LiveIntervals *LIS) const override
convertToThreeAddress - This method must be implemented by targets that set the M_CONVERTIBLE_TO_3_AD...
std::pair< unsigned, unsigned > decomposeMachineOperandsTargetFlags(unsigned TF) const override
bool expandPostRAPseudo(MachineInstr &MI) const override
bool isAssociativeAndCommutative(const MachineInstr &Inst, bool Invert) const override
MCInst getNop() const override
Return the noop instruction to use for a noop.
outliner::InstrType getOutliningTypeImpl(const MachineModuleInfo &MMI, MachineBasicBlock::iterator &MIT, unsigned Flags) const override
bool shouldScheduleLoadsNear(SDNode *Load1, SDNode *Load2, int64_t Offset1, int64_t Offset2, unsigned NumLoads) const override
This is a used by the pre-regalloc scheduler to determine (in conjunction with areLoadsFromSameBasePt...
MachineInstr * foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, ArrayRef< unsigned > Ops, MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS=nullptr, VirtRegMap *VRM=nullptr) const override
Fold a load or store of the specified stack slot into the specified machine instruction for the speci...
bool analyzeCompare(const MachineInstr &MI, Register &SrcReg, Register &SrcReg2, int64_t &CmpMask, int64_t &CmpValue) const override
bool getConstValDefinedInReg(const MachineInstr &MI, const Register Reg, int64_t &ImmVal) const override
std::optional< ExtAddrMode > getAddrModeFromMemoryOp(const MachineInstr &MemI, const TargetRegisterInfo *TRI) const override
Register isStoreToStackSlotPostFE(const MachineInstr &MI, int &FrameIndex) const override
isStoreToStackSlotPostFE - Check for post-frame ptr elimination stack locations as well.
bool isUnconditionalTailCall(const MachineInstr &MI) const override
bool reverseBranchCondition(SmallVectorImpl< MachineOperand > &Cond) const override
std::optional< std::unique_ptr< outliner::OutlinedFunction > > getOutliningCandidateInfo(const MachineModuleInfo &MMI, std::vector< outliner::Candidate > &RepeatedSequenceLocs, unsigned MinRepeats) const override
bool classifyLEAReg(MachineInstr &MI, const MachineOperand &Src, unsigned LEAOpcode, bool AllowSP, Register &NewSrc, unsigned &NewSrcSubReg, bool &isKill, MachineOperand &ImplicitOp, LiveVariables *LV, LiveIntervals *LIS) const
Given an operand within a MachineInstr, insert preceding code to put it into the right format for a p...
Register isLoadFromStackSlotPostFE(const MachineInstr &MI, int &FrameIndex) const override
isLoadFromStackSlotPostFE - Check for post-frame ptr elimination stack locations as well.
void setExecutionDomain(MachineInstr &MI, unsigned Domain) const override
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg, MachineInstr::MIFlag Flags=MachineInstr::NoFlags) const override
const TargetRegisterClass * getRegClass(const MCInstrDesc &MCID, unsigned OpNum, const TargetRegisterInfo *TRI) const override
Given a machine instruction descriptor, returns the register class constraint for OpNum,...
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const override
ArrayRef< std::pair< unsigned, const char * > > getSerializableDirectMachineOperandTargetFlags() const override
Register isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex) const override
bool setExecutionDomainCustom(MachineInstr &MI, unsigned Domain) const
int getSPAdjust(const MachineInstr &MI) const override
getSPAdjust - This returns the stack pointer adjustment made by this instruction.
bool verifyInstruction(const MachineInstr &MI, StringRef &ErrInfo) const override
bool isReMaterializableImpl(const MachineInstr &MI) const override
Register getGlobalBaseReg(MachineFunction *MF) const
getGlobalBaseReg - Return a virtual register initialized with the the global base register value.
int getJumpTableIndex(const MachineInstr &MI) const override
void insertNoop(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const override
void setSpecialOperandAttr(MachineInstr &OldMI1, MachineInstr &OldMI2, MachineInstr &NewMI1, MachineInstr &NewMI2) const override
This is an architecture-specific helper function of reassociateOps.
std::pair< uint16_t, uint16_t > getExecutionDomain(const MachineInstr &MI) const override
bool isCoalescableExtInstr(const MachineInstr &MI, Register &SrcReg, Register &DstReg, unsigned &SubIdx) const override
isCoalescableExtInstr - Return true if the instruction is a "coalescable" extension instruction.
void loadStoreTileReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned Opc, Register Reg, int FrameIdx, bool isKill=false) const
void genAlternativeCodeSequence(MachineInstr &Root, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< Register, unsigned > &InstrIdxForVirtReg) const override
When getMachineCombinerPatterns() finds potential patterns, this function generates the instructions ...
bool hasReassociableOperands(const MachineInstr &Inst, const MachineBasicBlock *MBB) const override
bool analyzeBranchPredicate(MachineBasicBlock &MBB, TargetInstrInfo::MachineBranchPredicate &MBP, bool AllowModify=false) const override
static bool isDataInvariant(MachineInstr &MI)
Returns true if the instruction has no behavior (specified or otherwise) that is based on the value o...
unsigned getUndefRegClearance(const MachineInstr &MI, unsigned OpNum, const TargetRegisterInfo *TRI) const override
Inform the BreakFalseDeps pass how many idle instructions we would like before certain undef register...
void breakPartialRegDependency(MachineInstr &MI, unsigned OpNum, const TargetRegisterInfo *TRI) const override
void buildClearRegister(Register Reg, MachineBasicBlock &MBB, MachineBasicBlock::iterator Iter, DebugLoc &DL, bool AllowSideEffects=true) const override
Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override
int64_t getFrameAdjustment(const MachineInstr &I) const
Returns the stack pointer adjustment that happens inside the frame setup..destroy sequence (e....
bool hasHighOperandLatency(const TargetSchedModel &SchedModel, const MachineRegisterInfo *MRI, const MachineInstr &DefMI, unsigned DefIdx, const MachineInstr &UseMI, unsigned UseIdx) const override
bool isSafeToMoveRegClassDefs(const TargetRegisterClass *RC) const override
uint16_t getExecutionDomainCustom(const MachineInstr &MI) const
bool isHighLatencyDef(int opc) const override
void buildOutlinedFrame(MachineBasicBlock &MBB, MachineFunction &MF, const outliner::OutlinedFunction &OF) const override
bool foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg, MachineRegisterInfo *MRI) const override
foldImmediate - 'Reg' is known to be defined by a move immediate instruction, try to fold the immedia...
unsigned removeBranch(MachineBasicBlock &MBB, int *BytesRemoved=nullptr) const override
unsigned getFMA3OpcodeToCommuteOperands(const MachineInstr &MI, unsigned SrcOpIdx1, unsigned SrcOpIdx2, const X86InstrFMA3Group &FMA3Group) const
Returns an adjusted FMA opcode that must be used in FMA instruction that performs the same computatio...
bool preservesZeroValueInReg(const MachineInstr *MI, const Register NullValueReg, const TargetRegisterInfo *TRI) const override
unsigned getPartialRegUpdateClearance(const MachineInstr &MI, unsigned OpNum, const TargetRegisterInfo *TRI) const override
Inform the BreakFalseDeps pass how many idle instructions we would like before a partial register upd...
X86MachineFunctionInfo - This class is derived from MachineFunction and contains private X86 target-s...
Register getGlobalBaseReg() const
int getTCReturnAddrDelta() const
void setGlobalBaseReg(Register Reg)
unsigned getNumLocalDynamicTLSAccesses() const
bool getUsesRedZone() const
const TargetRegisterClass * constrainRegClassToNonRex2(const TargetRegisterClass *RC) const
bool isPICStyleGOT() const
const X86InstrInfo * getInstrInfo() const override
const X86RegisterInfo * getRegisterInfo() const override
const X86FrameLowering * getFrameLowering() const override
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Define
Register definition.
@ Kill
The last use of a register.
@ Undef
Value of the register doesn't matter.
@ X86
Windows x64, Windows Itanium (IA-64)
X86II - This namespace holds all of the target specific flags that instruction info tracks.
bool isKMergeMasked(uint64_t TSFlags)
bool hasNewDataDest(uint64_t TSFlags)
@ MO_GOT_ABSOLUTE_ADDRESS
MO_GOT_ABSOLUTE_ADDRESS - On a symbol operand, this represents a relocation of: SYMBOL_LABEL + [.
@ MO_INDNTPOFF
MO_INDNTPOFF - On a symbol operand this indicates that the immediate is the absolute address of the G...
@ MO_GOTNTPOFF
MO_GOTNTPOFF - On a symbol operand this indicates that the immediate is the offset of the GOT entry w...
@ MO_GOTTPOFF
MO_GOTTPOFF - On a symbol operand this indicates that the immediate is the offset of the GOT entry wi...
@ MO_PIC_BASE_OFFSET
MO_PIC_BASE_OFFSET - On a symbol operand this indicates that the immediate should get the value of th...
@ MO_GOTPCREL
MO_GOTPCREL - On a symbol operand this indicates that the immediate is offset to the GOT entry for th...
@ EVEX
EVEX - Specifies that this instruction use EVEX form which provides syntax support up to 32 512-bit r...
@ SSEDomainShift
Execution domain for SSE instructions.
bool canUseApxExtendedReg(const MCInstrDesc &Desc)
bool isPseudo(uint64_t TSFlags)
bool isKMasked(uint64_t TSFlags)
int getMemoryOperandNo(uint64_t TSFlags)
unsigned getOperandBias(const MCInstrDesc &Desc)
Compute whether all of the def operands are repeated in the uses and therefore should be skipped.
Define some predicates that are used for node matching.
CondCode getCondFromBranch(const MachineInstr &MI)
CondCode getCondFromCFCMov(const MachineInstr &MI)
CondCode getCondFromMI(const MachineInstr &MI)
Return the condition code of the instruction.
int getFirstAddrOperandIdx(const MachineInstr &MI)
Return the index of the instruction's first address operand, if it has a memory reference,...
unsigned getSwappedVCMPImm(unsigned Imm)
Get the VCMP immediate if the opcodes are swapped.
CondCode GetOppositeBranchCondition(CondCode CC)
GetOppositeBranchCondition - Return the inverse of the specified cond, e.g.
unsigned getSwappedVPCOMImm(unsigned Imm)
Get the VPCOM immediate if the opcodes are swapped.
bool isX87Instruction(MachineInstr &MI)
Check if the instruction is X87 instruction.
unsigned getNonNDVariant(unsigned Opc)
unsigned getVPCMPImmForCond(ISD::CondCode CC)
Get the VPCMP immediate for the given condition.
std::pair< CondCode, bool > getX86ConditionCode(CmpInst::Predicate Predicate)
Return a pair of condition code for the given predicate and whether the instruction operands should b...
CondCode getCondFromSETCC(const MachineInstr &MI)
unsigned getSwappedVPCMPImm(unsigned Imm)
Get the VPCMP immediate if the opcodes are swapped.
CondCode getCondFromCCMP(const MachineInstr &MI)
int getCCMPCondFlagsFromCondCode(CondCode CC)
int getCondSrcNoFromDesc(const MCInstrDesc &MCID)
Return the source operand # for condition code by MCID.
const Constant * getConstantFromPool(const MachineInstr &MI, unsigned OpNo)
Find any constant pool entry associated with a specific instruction operand.
unsigned getCMovOpcode(unsigned RegBytes, bool HasMemoryOperand=false, bool HasNDD=false)
Return a cmov opcode for the given register size in bytes, and operand type.
unsigned getNFVariant(unsigned Opc)
unsigned getVectorRegisterWidth(const MCOperandInfo &Info)
Get the width of the vector register operand.
CondCode getCondFromCMov(const MachineInstr &MI)
initializer< Ty > init(const Ty &Val)
InstrType
Represents how an instruction should be mapped by the outliner.
NodeAddr< NodeBase * > Node
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
static bool isAddMemInstrWithRelocation(const MachineInstr &MI)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
static bool isMem(const MachineInstr &MI, unsigned Op)
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
bool isAligned(Align Lhs, uint64_t SizeInBytes)
Checks that SizeInBytes is a multiple of the alignment.
MCRegister getX86SubSuperRegister(MCRegister Reg, unsigned Size, bool High=false)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
FunctionPass * createX86GlobalBaseRegPass()
This pass initializes a global base register for PIC on x86-32.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
static const MachineInstrBuilder & addRegReg(const MachineInstrBuilder &MIB, Register Reg1, bool isKill1, unsigned SubReg1, Register Reg2, bool isKill2, unsigned SubReg2)
addRegReg - This function is used to add a memory reference of the form: [Reg + Reg].
unsigned getDeadRegState(bool B)
static const MachineInstrBuilder & addFrameReference(const MachineInstrBuilder &MIB, int FI, int Offset=0, bool mem=true)
addFrameReference - This function is used to add a reference to the base of an abstract object on the...
FunctionPass * createCleanupLocalDynamicTLSPass()
This pass combines multiple accesses to local-dynamic TLS variables so that the TLS base address for ...
const X86FoldTableEntry * lookupBroadcastFoldTable(unsigned RegOp, unsigned OpNum)
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
const X86InstrFMA3Group * getFMA3Group(unsigned Opcode, uint64_t TSFlags)
Returns a reference to a group of FMA3 opcodes to where the given Opcode is included.
auto reverse(ContainerTy &&C)
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
const X86FoldTableEntry * lookupTwoAddrFoldTable(unsigned RegOp)
FunctionAddr VTableAddr Count
bool is_sorted(R &&Range, Compare C)
Wrapper function around std::is_sorted to check if elements in a range R are sorted with respect to a...
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
DomTreeNodeBase< MachineBasicBlock > MachineDomTreeNode
static bool isMemInstrWithGOTPCREL(const MachineInstr &MI)
static const MachineInstrBuilder & addOffset(const MachineInstrBuilder &MIB, int Offset)
unsigned getUndefRegState(bool B)
unsigned getRegState(const MachineOperand &RegOp)
Get all register state flags from machine operand RegOp.
unsigned getDefRegState(bool B)
auto lower_bound(R &&Range, T &&Value)
Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...
@ Sub
Subtraction of integers.
unsigned getKillRegState(bool B)
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
FunctionAddr VTableAddr Next
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
const X86FoldTableEntry * lookupUnfoldTable(unsigned MemOp)
constexpr unsigned BitWidth
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
bool matchBroadcastSize(const X86FoldTableEntry &Entry, unsigned BroadcastBits)
std::pair< MachineOperand, DIExpression * > ParamLoadedValue
auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.
const X86FoldTableEntry * lookupFoldTable(unsigned RegOp, unsigned OpNum)
static const MachineInstrBuilder & addRegOffset(const MachineInstrBuilder &MIB, Register Reg, bool isKill, int Offset)
addRegOffset - This function is used to add a memory reference of the form [Reg + Offset],...
int popcount(T Value) noexcept
Count the number of set bits in a value.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Used to describe addressing mode similar to ExtAddrMode in CodeGenPrepare.
This represents a simple continuous liveness interval for a value.
std::vector< MachineInstr * > Kills
Kills - List of MachineInstruction's which are the last use of this virtual register (kill it) in the...
static LLVM_ABI MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
X86AddressMode - This struct holds a generalized full x86 address mode.
enum llvm::X86AddressMode::@202116273335065351270200035056227005202106004277 BaseType
This class is used to group {132, 213, 231} forms of FMA opcodes together.
unsigned get213Opcode() const
Returns the 213 form of FMA opcode.
unsigned get231Opcode() const
Returns the 231 form of FMA opcode.
bool isIntrinsic() const
Returns true iff the group of FMA opcodes holds intrinsic opcodes.
unsigned get132Opcode() const
Returns the 132 form of FMA opcode.
An individual sequence of instructions to be replaced with a call to an outlined function.
The information necessary to create an outlined function for some class of candidate.