diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h index 0b6d155b6d161..69820aed2137b 100644 --- a/llvm/include/llvm/CodeGen/ISDOpcodes.h +++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h @@ -1490,17 +1490,6 @@ enum NodeType { BUILTIN_OP_END }; -/// FIRST_TARGET_STRICTFP_OPCODE - Target-specific pre-isel operations -/// which cannot raise FP exceptions should be less than this value. -/// Those that do must not be less than this value. -static const int FIRST_TARGET_STRICTFP_OPCODE = BUILTIN_OP_END + 400; - -/// FIRST_TARGET_MEMORY_OPCODE - Target-specific pre-isel operations -/// which do not reference a specific memory location should be less than -/// this value. Those that do must not be less than this value, and can -/// be used with SelectionDAG::getMemIntrinsicNode. -static const int FIRST_TARGET_MEMORY_OPCODE = BUILTIN_OP_END + 500; - /// Whether this is bitwise logic opcode. inline bool isBitwiseLogicOp(unsigned Opcode) { return Opcode == ISD::AND || Opcode == ISD::OR || Opcode == ISD::XOR; diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h index d63e1c559122b..ff7caec41855f 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -1330,8 +1330,8 @@ class SelectionDAG { /// Creates a MemIntrinsicNode that may produce a /// result and takes a list of operands. Opcode may be INTRINSIC_VOID, - /// INTRINSIC_W_CHAIN, or a target-specific opcode with a value not - /// less than FIRST_TARGET_MEMORY_OPCODE. + /// INTRINSIC_W_CHAIN, or a target-specific memory-referencing opcode + // (see `SelectionDAGTargetInfo::isTargetMemoryOpcode`). SDValue getMemIntrinsicNode( unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h index 5e2440c1b57e3..03899493847b3 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -210,7 +210,6 @@ class SDValue { inline const SDValue &getOperand(unsigned i) const; inline uint64_t getConstantOperandVal(unsigned i) const; inline const APInt &getConstantOperandAPInt(unsigned i) const; - inline bool isTargetMemoryOpcode() const; inline bool isTargetOpcode() const; inline bool isMachineOpcode() const; inline bool isUndef() const; @@ -691,22 +690,6 @@ END_TWO_BYTE_PACK() /// \ISD namespace). bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; } - /// Test if this node has a target-specific opcode that may raise - /// FP exceptions (in the \ISD namespace and greater than - /// FIRST_TARGET_STRICTFP_OPCODE). Note that all target memory - /// opcode are currently automatically considered to possibly raise - /// FP exceptions as well. - bool isTargetStrictFPOpcode() const { - return NodeType >= ISD::FIRST_TARGET_STRICTFP_OPCODE; - } - - /// Test if this node has a target-specific - /// memory-referencing opcode (in the \ISD namespace and - /// greater than FIRST_TARGET_MEMORY_OPCODE). - bool isTargetMemoryOpcode() const { - return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE; - } - /// Return true if the type of the node type undefined. bool isUndef() const { return NodeType == ISD::UNDEF; } @@ -1255,10 +1238,6 @@ inline bool SDValue::isTargetOpcode() const { return Node->isTargetOpcode(); } -inline bool SDValue::isTargetMemoryOpcode() const { - return Node->isTargetMemoryOpcode(); -} - inline bool SDValue::isMachineOpcode() const { return Node->isMachineOpcode(); } @@ -1615,10 +1594,10 @@ class AtomicSDNode : public MemSDNode { } }; -/// This SDNode is used for target intrinsics that touch -/// memory and need an associated MachineMemOperand. Its opcode may be -/// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode -/// with a value not less than FIRST_TARGET_MEMORY_OPCODE. +/// This SDNode is used for target intrinsics that touch memory and need +/// an associated MachineMemOperand. Its opcode may be INTRINSIC_VOID, +/// INTRINSIC_W_CHAIN, PREFETCH, or a target-specific memory-referencing +/// opcode (see `SelectionDAGTargetInfo::isTargetMemoryOpcode`). class MemIntrinsicSDNode : public MemSDNode { public: MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, diff --git a/llvm/include/llvm/CodeGen/SelectionDAGTargetInfo.h b/llvm/include/llvm/CodeGen/SelectionDAGTargetInfo.h index 720c9463867c3..ef5ae5dba58de 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGTargetInfo.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGTargetInfo.h @@ -35,6 +35,19 @@ class SelectionDAGTargetInfo { SelectionDAGTargetInfo &operator=(const SelectionDAGTargetInfo &) = delete; virtual ~SelectionDAGTargetInfo(); + /// Returns true if a node with the given target-specific opcode has + /// a memory operand. Nodes with such opcodes can only be created with + /// `SelectionDAG::getMemIntrinsicNode`. + virtual bool isTargetMemoryOpcode(unsigned Opcode) const { return false; } + + /// Returns true if a node with the given target-specific opcode has + /// strict floating-point semantics. + virtual bool isTargetStrictFPOpcode(unsigned Opcode) const { return false; } + + /// Returns true if a node with the given target-specific opcode + /// may raise a floating-point exception. + virtual bool mayRaiseFPException(unsigned Opcode) const; + /// Emit target-specific code that performs a memcpy. /// This can be used by targets to provide code sequences for cases /// that don't fit the target's parameters for simple loads/stores and can be diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 20119537b631f..10e8ba93359fb 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -9040,12 +9040,12 @@ SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef Ops, EVT MemVT, MachineMemOperand *MMO) { - assert((Opcode == ISD::INTRINSIC_VOID || - Opcode == ISD::INTRINSIC_W_CHAIN || - Opcode == ISD::PREFETCH || - (Opcode <= (unsigned)std::numeric_limits::max() && - (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) && - "Opcode is not a memory-accessing opcode!"); + assert( + (Opcode == ISD::INTRINSIC_VOID || Opcode == ISD::INTRINSIC_W_CHAIN || + Opcode == ISD::PREFETCH || + (Opcode <= (unsigned)std::numeric_limits::max() && + Opcode >= ISD::BUILTIN_OP_END && TSI->isTargetMemoryOpcode(Opcode))) && + "Opcode is not a memory-accessing opcode!"); // Memoize the node unless it returns a glue result. MemIntrinsicSDNode *N; diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index c1dabe05452fb..d64a90bcaae7d 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -51,6 +51,7 @@ #include "llvm/CodeGen/SchedulerRegistry.h" #include "llvm/CodeGen/SelectionDAG.h" #include "llvm/CodeGen/SelectionDAGNodes.h" +#include "llvm/CodeGen/SelectionDAGTargetInfo.h" #include "llvm/CodeGen/StackMaps.h" #include "llvm/CodeGen/StackProtector.h" #include "llvm/CodeGen/SwiftErrorValueTracking.h" @@ -4382,8 +4383,10 @@ bool SelectionDAGISel::mayRaiseFPException(SDNode *N) const { // For ISD opcodes, only StrictFP opcodes may raise an FP // exception. - if (N->isTargetOpcode()) - return N->isTargetStrictFPOpcode(); + if (N->isTargetOpcode()) { + const SelectionDAGTargetInfo &TSI = CurDAG->getSelectionDAGInfo(); + return TSI.mayRaiseFPException(N->getOpcode()); + } return N->isStrictFPOpcode(); } diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGTargetInfo.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGTargetInfo.cpp index 3a2df6f60593a..0f3b36658f10a 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGTargetInfo.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGTargetInfo.cpp @@ -15,3 +15,9 @@ using namespace llvm; SelectionDAGTargetInfo::~SelectionDAGTargetInfo() = default; + +bool SelectionDAGTargetInfo::mayRaiseFPException(unsigned Opcode) const { + // FIXME: All target memory opcodes are currently automatically considered + // to possibly raise FP exceptions. See rev. 63336795. + return isTargetStrictFPOpcode(Opcode) || isTargetMemoryOpcode(Opcode); +} diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h index 4ab8e0103fa2c..36d62ca69ca08 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h @@ -477,11 +477,14 @@ enum NodeType : unsigned { MSRR, // Strict (exception-raising) floating point comparison - STRICT_FCMP = ISD::FIRST_TARGET_STRICTFP_OPCODE, + FIRST_STRICTFP_OPCODE, + STRICT_FCMP = FIRST_STRICTFP_OPCODE, STRICT_FCMPE, + LAST_STRICTFP_OPCODE = STRICT_FCMPE, // NEON Load/Store with post-increment base updates - LD2post = ISD::FIRST_TARGET_MEMORY_OPCODE, + FIRST_MEMORY_OPCODE, + LD2post = FIRST_MEMORY_OPCODE, LD3post, LD4post, ST2post, @@ -516,6 +519,7 @@ enum NodeType : unsigned { STP, STILP, STNP, + LAST_MEMORY_OPCODE = STNP, // SME ZA loads and stores SME_ZA_LDR, diff --git a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp index c4d60a0cb4a11..17adda15d9fc8 100644 --- a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp @@ -23,6 +23,16 @@ static cl::opt "to lower to librt functions"), cl::init(true)); +bool AArch64SelectionDAGInfo::isTargetMemoryOpcode(unsigned Opcode) const { + return Opcode >= AArch64ISD::FIRST_MEMORY_OPCODE && + Opcode <= AArch64ISD::LAST_MEMORY_OPCODE; +} + +bool AArch64SelectionDAGInfo::isTargetStrictFPOpcode(unsigned Opcode) const { + return Opcode >= AArch64ISD::FIRST_STRICTFP_OPCODE && + Opcode <= AArch64ISD::LAST_STRICTFP_OPCODE; +} + SDValue AArch64SelectionDAGInfo::EmitMOPS(unsigned Opcode, SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue SrcOrValue, diff --git a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.h b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.h index 9d1f2e9cba846..7efe49c720655 100644 --- a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.h +++ b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.h @@ -19,6 +19,10 @@ namespace llvm { class AArch64SelectionDAGInfo : public SelectionDAGTargetInfo { public: + bool isTargetMemoryOpcode(unsigned Opcode) const override; + + bool isTargetStrictFPOpcode(unsigned Opcode) const override; + SDValue EmitMOPS(unsigned Opcode, SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue SrcOrValue, SDValue Size, Align Alignment, bool isVolatile, diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp index a716d185e392a..cca9fa72d0ca5 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp @@ -5555,7 +5555,6 @@ const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const { NODE_NAME_CASE(PC_ADD_REL_OFFSET) NODE_NAME_CASE(LDS) NODE_NAME_CASE(DUMMY_CHAIN) - case AMDGPUISD::FIRST_MEM_OPCODE_NUMBER: break; NODE_NAME_CASE(LOAD_D16_HI) NODE_NAME_CASE(LOAD_D16_LO) NODE_NAME_CASE(LOAD_D16_HI_I8) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h index 33991239a4120..c74dc7942f52c 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h @@ -546,8 +546,9 @@ enum NodeType : unsigned { LDS, DUMMY_CHAIN, - FIRST_MEM_OPCODE_NUMBER = ISD::FIRST_TARGET_MEMORY_OPCODE, - LOAD_D16_HI, + + FIRST_MEMORY_OPCODE, + LOAD_D16_HI = FIRST_MEMORY_OPCODE, LOAD_D16_LO, LOAD_D16_HI_I8, LOAD_D16_HI_U8, @@ -603,6 +604,7 @@ enum NodeType : unsigned { BUFFER_ATOMIC_FMIN, BUFFER_ATOMIC_FMAX, BUFFER_ATOMIC_COND_SUB_U32, + LAST_MEMORY_OPCODE = BUFFER_ATOMIC_COND_SUB_U32, }; } // End namespace AMDGPUISD diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSelectionDAGInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSelectionDAGInfo.cpp index 7bc651504e36d..2941a48c78d94 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUSelectionDAGInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUSelectionDAGInfo.cpp @@ -7,7 +7,13 @@ //===----------------------------------------------------------------------===// #include "AMDGPUSelectionDAGInfo.h" +#include "AMDGPUISelLowering.h" using namespace llvm; AMDGPUSelectionDAGInfo::~AMDGPUSelectionDAGInfo() = default; + +bool AMDGPUSelectionDAGInfo::isTargetMemoryOpcode(unsigned Opcode) const { + return Opcode >= AMDGPUISD::FIRST_MEMORY_OPCODE && + Opcode <= AMDGPUISD::LAST_MEMORY_OPCODE; +} diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSelectionDAGInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUSelectionDAGInfo.h index bb11a56da5259..3280be73b2fdf 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUSelectionDAGInfo.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUSelectionDAGInfo.h @@ -16,6 +16,8 @@ namespace llvm { class AMDGPUSelectionDAGInfo : public SelectionDAGTargetInfo { public: ~AMDGPUSelectionDAGInfo() override; + + bool isTargetMemoryOpcode(unsigned Opcode) const override; }; } // namespace llvm diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h index 49416e2c8b25e..3c1a414af8597 100644 --- a/llvm/lib/Target/ARM/ARMISelLowering.h +++ b/llvm/lib/Target/ARM/ARMISelLowering.h @@ -321,7 +321,8 @@ class VectorType; CSINC, // Conditional select increment. // Vector load N-element structure to all lanes: - VLD1DUP = ISD::FIRST_TARGET_MEMORY_OPCODE, + FIRST_MEMORY_OPCODE, + VLD1DUP = FIRST_MEMORY_OPCODE, VLD2DUP, VLD3DUP, VLD4DUP, @@ -356,7 +357,8 @@ class VectorType; // Load/Store of dual registers LDRD, - STRD + STRD, + LAST_MEMORY_OPCODE = STRD, }; } // end namespace ARMISD diff --git a/llvm/lib/Target/ARM/ARMSelectionDAGInfo.cpp b/llvm/lib/Target/ARM/ARMSelectionDAGInfo.cpp index e7ea10ff971a0..a39487c318f8e 100644 --- a/llvm/lib/Target/ARM/ARMSelectionDAGInfo.cpp +++ b/llvm/lib/Target/ARM/ARMSelectionDAGInfo.cpp @@ -30,6 +30,11 @@ cl::opt EnableMemtransferTPLoop( "Allow (may be subject to certain conditions) " "conversion of memcpy to TP loop."))); +bool ARMSelectionDAGInfo::isTargetMemoryOpcode(unsigned Opcode) const { + return Opcode >= ARMISD::FIRST_MEMORY_OPCODE && + Opcode <= ARMISD::LAST_MEMORY_OPCODE; +} + // Emit, if possible, a specialized version of the given Libcall. Typically this // means selecting the appropriately aligned version, but we also convert memset // of 0 into memclr. diff --git a/llvm/lib/Target/ARM/ARMSelectionDAGInfo.h b/llvm/lib/Target/ARM/ARMSelectionDAGInfo.h index 275b1c0f8dc01..d68150e66567c 100644 --- a/llvm/lib/Target/ARM/ARMSelectionDAGInfo.h +++ b/llvm/lib/Target/ARM/ARMSelectionDAGInfo.h @@ -37,6 +37,8 @@ namespace ARM_AM { class ARMSelectionDAGInfo : public SelectionDAGTargetInfo { public: + bool isTargetMemoryOpcode(unsigned Opcode) const override; + SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, diff --git a/llvm/lib/Target/Mips/MipsISelLowering.h b/llvm/lib/Target/Mips/MipsISelLowering.h index e245c056de649..655a347679ad7 100644 --- a/llvm/lib/Target/Mips/MipsISelLowering.h +++ b/llvm/lib/Target/Mips/MipsISelLowering.h @@ -247,14 +247,16 @@ class TargetRegisterClass; DOUBLE_SELECT_I64, // Load/Store Left/Right nodes. - LWL = ISD::FIRST_TARGET_MEMORY_OPCODE, + FIRST_MEMORY_OPCODE, + LWL = FIRST_MEMORY_OPCODE, LWR, SWL, SWR, LDL, LDR, SDL, - SDR + SDR, + LAST_MEMORY_OPCODE = SDR, }; } // ene namespace MipsISD diff --git a/llvm/lib/Target/Mips/MipsSelectionDAGInfo.cpp b/llvm/lib/Target/Mips/MipsSelectionDAGInfo.cpp index c24107bf63943..6497ac5bb2df6 100644 --- a/llvm/lib/Target/Mips/MipsSelectionDAGInfo.cpp +++ b/llvm/lib/Target/Mips/MipsSelectionDAGInfo.cpp @@ -7,7 +7,13 @@ //===----------------------------------------------------------------------===// #include "MipsSelectionDAGInfo.h" +#include "MipsISelLowering.h" using namespace llvm; MipsSelectionDAGInfo::~MipsSelectionDAGInfo() = default; + +bool MipsSelectionDAGInfo::isTargetMemoryOpcode(unsigned Opcode) const { + return Opcode >= MipsISD::FIRST_MEMORY_OPCODE && + Opcode <= MipsISD::LAST_MEMORY_OPCODE; +} diff --git a/llvm/lib/Target/Mips/MipsSelectionDAGInfo.h b/llvm/lib/Target/Mips/MipsSelectionDAGInfo.h index bccd924a30e71..934cd2e056595 100644 --- a/llvm/lib/Target/Mips/MipsSelectionDAGInfo.h +++ b/llvm/lib/Target/Mips/MipsSelectionDAGInfo.h @@ -16,6 +16,8 @@ namespace llvm { class MipsSelectionDAGInfo : public SelectionDAGTargetInfo { public: ~MipsSelectionDAGInfo() override; + + bool isTargetMemoryOpcode(unsigned Opcode) const override; }; } // namespace llvm diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h index 0244a0c5bec9d..4a98fe21b81dc 100644 --- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h +++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h @@ -70,7 +70,8 @@ enum NodeType : unsigned { BrxEnd, Dummy, - LoadV2 = ISD::FIRST_TARGET_MEMORY_OPCODE, + FIRST_MEMORY_OPCODE, + LoadV2 = FIRST_MEMORY_OPCODE, LoadV4, LDUV2, // LDU.v2 LDUV4, // LDU.v4 @@ -87,6 +88,7 @@ enum NodeType : unsigned { StoreRetval, StoreRetvalV2, StoreRetvalV4, + LAST_MEMORY_OPCODE = StoreRetvalV4, }; } diff --git a/llvm/lib/Target/NVPTX/NVPTXSelectionDAGInfo.cpp b/llvm/lib/Target/NVPTX/NVPTXSelectionDAGInfo.cpp index 9c26f310bbf65..d2035c6f8166f 100644 --- a/llvm/lib/Target/NVPTX/NVPTXSelectionDAGInfo.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXSelectionDAGInfo.cpp @@ -7,7 +7,13 @@ //===----------------------------------------------------------------------===// #include "NVPTXSelectionDAGInfo.h" +#include "NVPTXISelLowering.h" using namespace llvm; NVPTXSelectionDAGInfo::~NVPTXSelectionDAGInfo() = default; + +bool NVPTXSelectionDAGInfo::isTargetMemoryOpcode(unsigned Opcode) const { + return Opcode >= NVPTXISD::FIRST_MEMORY_OPCODE && + Opcode <= NVPTXISD::LAST_MEMORY_OPCODE; +} diff --git a/llvm/lib/Target/NVPTX/NVPTXSelectionDAGInfo.h b/llvm/lib/Target/NVPTX/NVPTXSelectionDAGInfo.h index 6b04d78ca9687..9d69f48026c79 100644 --- a/llvm/lib/Target/NVPTX/NVPTXSelectionDAGInfo.h +++ b/llvm/lib/Target/NVPTX/NVPTXSelectionDAGInfo.h @@ -16,6 +16,8 @@ namespace llvm { class NVPTXSelectionDAGInfo : public SelectionDAGTargetInfo { public: ~NVPTXSelectionDAGInfo() override; + + bool isTargetMemoryOpcode(unsigned Opcode) const override; }; } // namespace llvm diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.h b/llvm/lib/Target/PowerPC/PPCISelLowering.h index 8f41fc107a691..5d692e3fcae92 100644 --- a/llvm/lib/Target/PowerPC/PPCISelLowering.h +++ b/llvm/lib/Target/PowerPC/PPCISelLowering.h @@ -36,14 +36,11 @@ namespace llvm { namespace PPCISD { - // When adding a NEW PPCISD node please add it to the correct position in - // the enum. The order of elements in this enum matters! - // Values that are added after this entry: - // STBRX = ISD::FIRST_TARGET_MEMORY_OPCODE - // are considered memory opcodes and are treated differently than entries - // that come before it. For example, ADD or MUL should be placed before - // the ISD::FIRST_TARGET_MEMORY_OPCODE while a LOAD or STORE should come - // after it. + // When adding a NEW PPCISD node please add it to the correct position in + // the enum. The order of elements in this enum matters! + // Values that are added between FIRST_MEMORY_OPCODE and LAST_MEMORY_OPCODE + // are considered memory opcodes and are treated differently than other + // entries. enum NodeType : unsigned { // Start the numbering where the builtin ops and target ops leave off. FIRST_NUMBER = ISD::BUILTIN_OP_END, @@ -487,7 +484,8 @@ namespace llvm { XXMFACC, // Constrained conversion from floating point to int - STRICT_FCTIDZ = ISD::FIRST_TARGET_STRICTFP_OPCODE, + FIRST_STRICTFP_OPCODE, + STRICT_FCTIDZ = FIRST_STRICTFP_OPCODE, STRICT_FCTIWZ, STRICT_FCTIDUZ, STRICT_FCTIWUZ, @@ -500,6 +498,7 @@ namespace llvm { /// Constrained floating point add in round-to-zero mode. STRICT_FADDRTZ, + LAST_STRICTFP_OPCODE = STRICT_FADDRTZ, /// SETBC - The ISA 3.1 (P10) SETBC instruction. SETBC, @@ -516,7 +515,8 @@ namespace llvm { /// byte-swapping store instruction. It byte-swaps the low "Type" bits of /// the GPRC input, then stores it through Ptr. Type can be either i16 or /// i32. - STBRX = ISD::FIRST_TARGET_MEMORY_OPCODE, + FIRST_MEMORY_OPCODE, + STBRX = FIRST_MEMORY_OPCODE, /// GPRC, CHAIN = LBRX CHAIN, Ptr, Type - This is a /// byte-swapping load instruction. It loads "Type" bits, byte swaps it, @@ -607,7 +607,8 @@ namespace llvm { /// GPRC = TOC_ENTRY GA, TOC /// Loads the entry for GA from the TOC, where the TOC base is given by /// the last operand. - TOC_ENTRY + TOC_ENTRY, + LAST_MEMORY_OPCODE = TOC_ENTRY, }; } // end namespace PPCISD diff --git a/llvm/lib/Target/PowerPC/PPCSelectionDAGInfo.cpp b/llvm/lib/Target/PowerPC/PPCSelectionDAGInfo.cpp index 211aaff3cfa0e..95de9f39b86e8 100644 --- a/llvm/lib/Target/PowerPC/PPCSelectionDAGInfo.cpp +++ b/llvm/lib/Target/PowerPC/PPCSelectionDAGInfo.cpp @@ -7,7 +7,18 @@ //===----------------------------------------------------------------------===// #include "PPCSelectionDAGInfo.h" +#include "PPCISelLowering.h" using namespace llvm; PPCSelectionDAGInfo::~PPCSelectionDAGInfo() = default; + +bool PPCSelectionDAGInfo::isTargetMemoryOpcode(unsigned Opcode) const { + return Opcode >= PPCISD::FIRST_MEMORY_OPCODE && + Opcode <= PPCISD::LAST_MEMORY_OPCODE; +} + +bool PPCSelectionDAGInfo::isTargetStrictFPOpcode(unsigned Opcode) const { + return Opcode >= PPCISD::FIRST_STRICTFP_OPCODE && + Opcode <= PPCISD::LAST_STRICTFP_OPCODE; +} diff --git a/llvm/lib/Target/PowerPC/PPCSelectionDAGInfo.h b/llvm/lib/Target/PowerPC/PPCSelectionDAGInfo.h index cc14e9b0a6904..08e2ddbf1c4ca 100644 --- a/llvm/lib/Target/PowerPC/PPCSelectionDAGInfo.h +++ b/llvm/lib/Target/PowerPC/PPCSelectionDAGInfo.h @@ -16,6 +16,10 @@ namespace llvm { class PPCSelectionDAGInfo : public SelectionDAGTargetInfo { public: ~PPCSelectionDAGInfo() override; + + bool isTargetMemoryOpcode(unsigned Opcode) const override; + + bool isTargetStrictFPOpcode(unsigned Opcode) const override; }; } // namespace llvm diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index ea8814aa2b4fc..04dd23d9cdaa2 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -17,6 +17,7 @@ #include "RISCVConstantPoolValue.h" #include "RISCVMachineFunctionInfo.h" #include "RISCVRegisterInfo.h" +#include "RISCVSelectionDAGInfo.h" #include "RISCVSubtarget.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/Statistic.h" @@ -6394,14 +6395,12 @@ static unsigned getRISCVVLOp(SDValue Op) { /// Return true if a RISC-V target specified op has a passthru operand. static bool hasPassthruOp(unsigned Opcode) { assert(Opcode > RISCVISD::FIRST_NUMBER && - Opcode <= RISCVISD::LAST_RISCV_STRICTFP_OPCODE && + Opcode <= RISCVISD::LAST_STRICTFP_OPCODE && "not a RISC-V target specific op"); - static_assert(RISCVISD::LAST_VL_VECTOR_OP - RISCVISD::FIRST_VL_VECTOR_OP == - 127 && - RISCVISD::LAST_RISCV_STRICTFP_OPCODE - - ISD::FIRST_TARGET_STRICTFP_OPCODE == - 21 && - "adding target specific op should update this function"); + static_assert( + RISCVISD::LAST_VL_VECTOR_OP - RISCVISD::FIRST_VL_VECTOR_OP == 127 && + RISCVISD::LAST_STRICTFP_OPCODE - RISCVISD::FIRST_STRICTFP_OPCODE == 21 && + "adding target specific op should update this function"); if (Opcode >= RISCVISD::ADD_VL && Opcode <= RISCVISD::VFMAX_VL) return true; if (Opcode == RISCVISD::FCOPYSIGN_VL) @@ -6420,14 +6419,12 @@ static bool hasPassthruOp(unsigned Opcode) { /// Return true if a RISC-V target specified op has a mask operand. static bool hasMaskOp(unsigned Opcode) { assert(Opcode > RISCVISD::FIRST_NUMBER && - Opcode <= RISCVISD::LAST_RISCV_STRICTFP_OPCODE && + Opcode <= RISCVISD::LAST_STRICTFP_OPCODE && "not a RISC-V target specific op"); - static_assert(RISCVISD::LAST_VL_VECTOR_OP - RISCVISD::FIRST_VL_VECTOR_OP == - 127 && - RISCVISD::LAST_RISCV_STRICTFP_OPCODE - - ISD::FIRST_TARGET_STRICTFP_OPCODE == - 21 && - "adding target specific op should update this function"); + static_assert( + RISCVISD::LAST_VL_VECTOR_OP - RISCVISD::FIRST_VL_VECTOR_OP == 127 && + RISCVISD::LAST_STRICTFP_OPCODE - RISCVISD::FIRST_STRICTFP_OPCODE == 21 && + "adding target specific op should update this function"); if (Opcode >= RISCVISD::TRUNCATE_VECTOR_VL && Opcode <= RISCVISD::SETCC_VL) return true; if (Opcode >= RISCVISD::VRGATHER_VX_VL && Opcode <= RISCVISD::VFIRST_VL) @@ -15980,7 +15977,7 @@ static SDValue performFP_TO_INTCombine(SDNode *N, SDValue Src = N->getOperand(0); // Don't do this for strict-fp Src. - if (Src->isStrictFPOpcode() || Src->isTargetStrictFPOpcode()) + if (Src->isStrictFPOpcode()) return SDValue(); // Ensure the FP type is legal. @@ -16085,7 +16082,7 @@ static SDValue performFP_TO_INT_SATCombine(SDNode *N, SDValue Src = N->getOperand(0); // Don't do this for strict-fp Src. - if (Src->isStrictFPOpcode() || Src->isTargetStrictFPOpcode()) + if (Src->isStrictFPOpcode()) return SDValue(); // Ensure the FP type is also legal. @@ -16193,7 +16190,9 @@ static unsigned negateFMAOpcode(unsigned Opcode, bool NegMul, bool NegAcc) { static SDValue combineVFMADD_VLWithVFNEG_VL(SDNode *N, SelectionDAG &DAG) { // Fold FNEG_VL into FMA opcodes. // The first operand of strict-fp is chain. - unsigned Offset = N->isTargetStrictFPOpcode(); + bool IsStrict = + DAG.getSelectionDAGInfo().isTargetStrictFPOpcode(N->getOpcode()); + unsigned Offset = IsStrict ? 1 : 0; SDValue A = N->getOperand(0 + Offset); SDValue B = N->getOperand(1 + Offset); SDValue C = N->getOperand(2 + Offset); @@ -16220,7 +16219,7 @@ static SDValue combineVFMADD_VLWithVFNEG_VL(SDNode *N, SelectionDAG &DAG) { return SDValue(); unsigned NewOpcode = negateFMAOpcode(N->getOpcode(), NegA != NegB, NegC); - if (N->isTargetStrictFPOpcode()) + if (IsStrict) return DAG.getNode(NewOpcode, SDLoc(N), N->getVTList(), {N->getOperand(0), A, B, C, Mask, VL}); return DAG.getNode(NewOpcode, SDLoc(N), N->getValueType(0), A, B, C, Mask, @@ -16236,7 +16235,7 @@ static SDValue performVFMADD_VLCombine(SDNode *N, return V; // FIXME: Ignore strict opcodes for now. - if (N->isTargetStrictFPOpcode()) + if (DAG.getSelectionDAGInfo().isTargetStrictFPOpcode(N->getOpcode())) return SDValue(); return combineOp_VLToVWOp_VL(N, DCI, Subtarget); diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h index 4c78fd784a3c8..ea077c7d2d23a 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.h +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h @@ -467,7 +467,8 @@ enum NodeType : unsigned { // FP to 32 bit int conversions for RV64. These are used to keep track of the // result being sign extended to 64 bit. These saturate out of range inputs. - STRICT_FCVT_W_RV64 = ISD::FIRST_TARGET_STRICTFP_OPCODE, + FIRST_STRICTFP_OPCODE, + STRICT_FCVT_W_RV64 = FIRST_STRICTFP_OPCODE, STRICT_FCVT_WU_RV64, STRICT_FADD_VL, STRICT_FSUB_VL, @@ -489,17 +490,15 @@ enum NodeType : unsigned { STRICT_FSETCC_VL, STRICT_FSETCCS_VL, STRICT_VFROUND_NOEXCEPT_VL, - LAST_RISCV_STRICTFP_OPCODE = STRICT_VFROUND_NOEXCEPT_VL, + LAST_STRICTFP_OPCODE = STRICT_VFROUND_NOEXCEPT_VL, - // WARNING: Do not add anything in the end unless you want the node to - // have memop! In fact, starting from FIRST_TARGET_MEMORY_OPCODE all - // opcodes will be thought as target memory ops! - - TH_LWD = ISD::FIRST_TARGET_MEMORY_OPCODE, + FIRST_MEMORY_OPCODE, + TH_LWD = FIRST_MEMORY_OPCODE, TH_LWUD, TH_LDD, TH_SWD, TH_SDD, + LAST_MEMORY_OPCODE = TH_SDD, }; // clang-format on } // namespace RISCVISD diff --git a/llvm/lib/Target/RISCV/RISCVSelectionDAGInfo.cpp b/llvm/lib/Target/RISCV/RISCVSelectionDAGInfo.cpp index 19d6138609a2b..ab1ade89a76d1 100644 --- a/llvm/lib/Target/RISCV/RISCVSelectionDAGInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVSelectionDAGInfo.cpp @@ -7,7 +7,18 @@ //===----------------------------------------------------------------------===// #include "RISCVSelectionDAGInfo.h" +#include "RISCVISelLowering.h" using namespace llvm; RISCVSelectionDAGInfo::~RISCVSelectionDAGInfo() = default; + +bool RISCVSelectionDAGInfo::isTargetMemoryOpcode(unsigned Opcode) const { + return Opcode >= RISCVISD::FIRST_MEMORY_OPCODE && + Opcode <= RISCVISD::LAST_MEMORY_OPCODE; +} + +bool RISCVSelectionDAGInfo::isTargetStrictFPOpcode(unsigned Opcode) const { + return Opcode >= RISCVISD::FIRST_STRICTFP_OPCODE && + Opcode <= RISCVISD::LAST_STRICTFP_OPCODE; +} diff --git a/llvm/lib/Target/RISCV/RISCVSelectionDAGInfo.h b/llvm/lib/Target/RISCV/RISCVSelectionDAGInfo.h index 7543d8b493ceb..6977d8507a960 100644 --- a/llvm/lib/Target/RISCV/RISCVSelectionDAGInfo.h +++ b/llvm/lib/Target/RISCV/RISCVSelectionDAGInfo.h @@ -16,6 +16,10 @@ namespace llvm { class RISCVSelectionDAGInfo : public SelectionDAGTargetInfo { public: ~RISCVSelectionDAGInfo() override; + + bool isTargetMemoryOpcode(unsigned Opcode) const override; + + bool isTargetStrictFPOpcode(unsigned Opcode) const override; }; } // namespace llvm diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h index 0a899e861c73b..d663e4abfb4e3 100644 --- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h +++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h @@ -307,7 +307,8 @@ enum NodeType : unsigned { // Strict variants of scalar floating-point comparisons. // Quiet and signaling versions. - STRICT_FCMP = ISD::FIRST_TARGET_STRICTFP_OPCODE, + FIRST_STRICTFP_OPCODE, + STRICT_FCMP = FIRST_STRICTFP_OPCODE, STRICT_FCMPS, // Strict variants of vector floating-point comparisons. @@ -322,6 +323,7 @@ enum NodeType : unsigned { // Strict variants of VEXTEND and VROUND. STRICT_VEXTEND, STRICT_VROUND, + LAST_STRICTFP_OPCODE = STRICT_VROUND, // Wrappers around the inner loop of an 8- or 16-bit ATOMIC_SWAP or // ATOMIC_LOAD_. @@ -333,7 +335,8 @@ enum NodeType : unsigned { // operand into the high bits // Operand 3: the negative of operand 2, for rotating the other way // Operand 4: the width of the field in bits (8 or 16) - ATOMIC_SWAPW = ISD::FIRST_TARGET_MEMORY_OPCODE, + FIRST_MEMORY_OPCODE, + ATOMIC_SWAPW = FIRST_MEMORY_OPCODE, ATOMIC_LOADW_ADD, ATOMIC_LOADW_SUB, ATOMIC_LOADW_AND, @@ -384,7 +387,8 @@ enum NodeType : unsigned { // Prefetch from the second operand using the 4-bit control code in // the first operand. The code is 1 for a load prefetch and 2 for // a store prefetch. - PREFETCH + PREFETCH, + LAST_MEMORY_OPCODE = PREFETCH, }; // Return true if OPCODE is some kind of PC-relative address. diff --git a/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp b/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp index c182c9890509f..d76babec73dd4 100644 --- a/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp +++ b/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp @@ -17,6 +17,16 @@ using namespace llvm; #define DEBUG_TYPE "systemz-selectiondag-info" +bool SystemZSelectionDAGInfo::isTargetMemoryOpcode(unsigned Opcode) const { + return Opcode >= SystemZISD::FIRST_MEMORY_OPCODE && + Opcode <= SystemZISD::LAST_MEMORY_OPCODE; +} + +bool SystemZSelectionDAGInfo::isTargetStrictFPOpcode(unsigned Opcode) const { + return Opcode >= SystemZISD::FIRST_STRICTFP_OPCODE && + Opcode <= SystemZISD::LAST_STRICTFP_OPCODE; +} + static unsigned getMemMemLenAdj(unsigned Op) { return Op == SystemZISD::MEMSET_MVC ? 2 : 1; } diff --git a/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.h b/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.h index 6ac5bf8c6c1a3..c928f343e5710 100644 --- a/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.h +++ b/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.h @@ -21,6 +21,10 @@ class SystemZSelectionDAGInfo : public SelectionDAGTargetInfo { public: explicit SystemZSelectionDAGInfo() = default; + bool isTargetMemoryOpcode(unsigned Opcode) const override; + + bool isTargetStrictFPOpcode(unsigned Opcode) const override; + SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISD.def b/llvm/lib/Target/WebAssembly/WebAssemblyISD.def index 3502c47016c6b..1cf0d13df1ff6 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyISD.def +++ b/llvm/lib/Target/WebAssembly/WebAssemblyISD.def @@ -48,10 +48,10 @@ HANDLE_NODETYPE(I64_MUL_WIDE_S) HANDLE_NODETYPE(I64_MUL_WIDE_U) // Memory intrinsics -HANDLE_MEM_NODETYPE(GLOBAL_GET) -HANDLE_MEM_NODETYPE(GLOBAL_SET) -HANDLE_MEM_NODETYPE(TABLE_GET) -HANDLE_MEM_NODETYPE(TABLE_SET) +HANDLE_NODETYPE(GLOBAL_GET) +HANDLE_NODETYPE(GLOBAL_SET) +HANDLE_NODETYPE(TABLE_GET) +HANDLE_NODETYPE(TABLE_SET) // Bulk memory instructions. These follow LLVM's expected semantics of // supporting out-of-bounds pointers if the length is zero, by inserting diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp index 7712570869ff6..084aed6eed46d 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp @@ -871,14 +871,11 @@ const char * WebAssemblyTargetLowering::getTargetNodeName(unsigned Opcode) const { switch (static_cast(Opcode)) { case WebAssemblyISD::FIRST_NUMBER: - case WebAssemblyISD::FIRST_MEM_OPCODE: break; #define HANDLE_NODETYPE(NODE) \ case WebAssemblyISD::NODE: \ return "WebAssemblyISD::" #NODE; -#define HANDLE_MEM_NODETYPE(NODE) HANDLE_NODETYPE(NODE) #include "WebAssemblyISD.def" -#undef HANDLE_MEM_NODETYPE #undef HANDLE_NODETYPE } return nullptr; diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h index 82b33b6d1933d..454432728ca87 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h +++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h @@ -24,16 +24,8 @@ namespace WebAssemblyISD { enum NodeType : unsigned { FIRST_NUMBER = ISD::BUILTIN_OP_END, #define HANDLE_NODETYPE(NODE) NODE, -#define HANDLE_MEM_NODETYPE(NODE) #include "WebAssemblyISD.def" - FIRST_MEM_OPCODE = ISD::FIRST_TARGET_MEMORY_OPCODE, #undef HANDLE_NODETYPE -#undef HANDLE_MEM_NODETYPE -#define HANDLE_NODETYPE(NODE) -#define HANDLE_MEM_NODETYPE(NODE) NODE, -#include "WebAssemblyISD.def" -#undef HANDLE_NODETYPE -#undef HANDLE_MEM_NODETYPE }; } // end namespace WebAssemblyISD diff --git a/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp b/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp index 6f37dab409534..2673c81eae40b 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp @@ -18,6 +18,18 @@ using namespace llvm; WebAssemblySelectionDAGInfo::~WebAssemblySelectionDAGInfo() = default; // anchor +bool WebAssemblySelectionDAGInfo::isTargetMemoryOpcode(unsigned Opcode) const { + switch (static_cast(Opcode)) { + default: + return false; + case WebAssemblyISD::GLOBAL_GET: + case WebAssemblyISD::GLOBAL_SET: + case WebAssemblyISD::TABLE_GET: + case WebAssemblyISD::TABLE_SET: + return true; + } +} + SDValue WebAssemblySelectionDAGInfo::EmitTargetCodeForMemcpy( SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool IsVolatile, bool AlwaysInline, diff --git a/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.h b/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.h index fd517b238715b..69c9af0966308 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.h +++ b/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.h @@ -22,6 +22,9 @@ namespace llvm { class WebAssemblySelectionDAGInfo final : public SelectionDAGTargetInfo { public: ~WebAssemblySelectionDAGInfo() override; + + bool isTargetMemoryOpcode(unsigned Opcode) const override; + SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, Align Alignment, bool isVolatile, diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 2528ca553d3e9..b50e0c60fadb6 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -54481,7 +54481,8 @@ static SDValue combineX86INT_TO_FP(SDNode *N, SelectionDAG &DAG, static SDValue combineCVTP2I_CVTTP2I(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) { - bool IsStrict = N->isTargetStrictFPOpcode(); + const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); + bool IsStrict = TSI.isTargetStrictFPOpcode(N->getOpcode()); EVT VT = N->getValueType(0); // Convert a full vector load into vzload when not all bits are needed. @@ -55102,7 +55103,10 @@ static SDValue combineFMA(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) { SDLoc dl(N); EVT VT = N->getValueType(0); - bool IsStrict = N->isStrictFPOpcode() || N->isTargetStrictFPOpcode(); + const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); + bool IsStrict = N->isTargetOpcode() + ? TSI.isTargetStrictFPOpcode(N->getOpcode()) + : N->isStrictFPOpcode(); // Let legalize expand this if it isn't a legal type yet. const TargetLowering &TLI = DAG.getTargetLoweringInfo(); diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h index d154ee9745b97..2b7a8eaf249d8 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.h +++ b/llvm/lib/Target/X86/X86ISelLowering.h @@ -809,7 +809,8 @@ namespace llvm { CTEST, /// X86 strict FP compare instructions. - STRICT_FCMP = ISD::FIRST_TARGET_STRICTFP_OPCODE, + FIRST_STRICTFP_OPCODE, + STRICT_FCMP = FIRST_STRICTFP_OPCODE, STRICT_FCMPS, // Vector packed double/float comparison. @@ -853,12 +854,11 @@ namespace llvm { /// Floating point max and min. STRICT_FMAX, STRICT_FMIN, - - // WARNING: Only add nodes here if they are strict FP nodes. Non-memory and - // non-strict FP nodes should be above FIRST_TARGET_STRICTFP_OPCODE. + LAST_STRICTFP_OPCODE = STRICT_FMIN, // Compare and swap. - LCMPXCHG_DAG = ISD::FIRST_TARGET_MEMORY_OPCODE, + FIRST_MEMORY_OPCODE, + LCMPXCHG_DAG = FIRST_MEMORY_OPCODE, LCMPXCHG8_DAG, LCMPXCHG16_DAG, LCMPXCHG16_SAVE_RBX_DAG, @@ -979,10 +979,7 @@ namespace llvm { // Conditional load/store instructions CLOAD, CSTORE, - - // WARNING: Do not add anything in the end unless you want the node to - // have memop! In fact, starting from FIRST_TARGET_MEMORY_OPCODE all - // opcodes will be thought as target memory ops! + LAST_MEMORY_OPCODE = CSTORE, }; } // end namespace X86ISD diff --git a/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp b/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp index 3f88bcf9ce5ec..aba62c36546f9 100644 --- a/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp +++ b/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp @@ -27,6 +27,16 @@ static cl::opt UseFSRMForMemcpy("x86-use-fsrm-for-memcpy", cl::Hidden, cl::init(false), cl::desc("Use fast short rep mov in memcpy lowering")); +bool X86SelectionDAGInfo::isTargetMemoryOpcode(unsigned Opcode) const { + return Opcode >= X86ISD::FIRST_MEMORY_OPCODE && + Opcode <= X86ISD::LAST_MEMORY_OPCODE; +} + +bool X86SelectionDAGInfo::isTargetStrictFPOpcode(unsigned Opcode) const { + return Opcode >= X86ISD::FIRST_STRICTFP_OPCODE && + Opcode <= X86ISD::LAST_STRICTFP_OPCODE; +} + /// Returns the best type to use with repmovs/repstos depending on alignment. static MVT getOptimalRepType(const X86Subtarget &Subtarget, Align Alignment) { uint64_t Align = Alignment.value(); diff --git a/llvm/lib/Target/X86/X86SelectionDAGInfo.h b/llvm/lib/Target/X86/X86SelectionDAGInfo.h index 19136ca4f6f58..e77e16bab830d 100644 --- a/llvm/lib/Target/X86/X86SelectionDAGInfo.h +++ b/llvm/lib/Target/X86/X86SelectionDAGInfo.h @@ -26,6 +26,10 @@ class X86SelectionDAGInfo : public SelectionDAGTargetInfo { public: explicit X86SelectionDAGInfo() = default; + bool isTargetMemoryOpcode(unsigned Opcode) const override; + + bool isTargetStrictFPOpcode(unsigned Opcode) const override; + SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, Align Alignment,