Thanks to visit codestin.com
Credit goes to llvm.org

LLVM 22.0.0git
TargetLowering.h
Go to the documentation of this file.
1//===- llvm/CodeGen/TargetLowering.h - Target Lowering Info -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file describes how to lower LLVM code to machine code. This has two
11/// main components:
12///
13/// 1. Which ValueTypes are natively supported by the target.
14/// 2. Which operations are supported for supported ValueTypes.
15/// 3. Cost thresholds for alternative implementations of certain operations.
16///
17/// In addition it has a few other components, like information about FP
18/// immediates.
19///
20//===----------------------------------------------------------------------===//
21
22#ifndef LLVM_CODEGEN_TARGETLOWERING_H
23#define LLVM_CODEGEN_TARGETLOWERING_H
24
25#include "llvm/ADT/APInt.h"
26#include "llvm/ADT/ArrayRef.h"
27#include "llvm/ADT/DenseMap.h"
29#include "llvm/ADT/StringRef.h"
40#include "llvm/IR/Attributes.h"
41#include "llvm/IR/CallingConv.h"
42#include "llvm/IR/DataLayout.h"
44#include "llvm/IR/Function.h"
45#include "llvm/IR/InlineAsm.h"
46#include "llvm/IR/Instruction.h"
49#include "llvm/IR/Type.h"
56#include <algorithm>
57#include <cassert>
58#include <climits>
59#include <cstdint>
60#include <iterator>
61#include <map>
62#include <string>
63#include <utility>
64#include <vector>
65
66namespace llvm {
67
68class AssumptionCache;
69class CCState;
70class CCValAssign;
73class Constant;
74class FastISel;
76class GlobalValue;
77class Loop;
79class IntrinsicInst;
80class IRBuilderBase;
81struct KnownBits;
82class LLVMContext;
84class MachineFunction;
85class MachineInstr;
87class MachineLoop;
89class MCContext;
90class MCExpr;
91class Module;
94class TargetMachine;
98class Value;
99class VPIntrinsic;
100
101namespace Sched {
102
104 None, // No preference
105 Source, // Follow source order.
106 RegPressure, // Scheduling for lowest register pressure.
107 Hybrid, // Scheduling for both latency and register pressure.
108 ILP, // Scheduling for ILP in low register pressure mode.
109 VLIW, // Scheduling for VLIW targets.
110 Fast, // Fast suboptimal list scheduling
111 Linearize, // Linearize DAG, no scheduling
112 Last = Linearize // Marker for the last Sched::Preference
113};
114
115} // end namespace Sched
116
117// MemOp models a memory operation, either memset or memcpy/memmove.
118struct MemOp {
119private:
120 // Shared
121 uint64_t Size;
122 bool DstAlignCanChange; // true if destination alignment can satisfy any
123 // constraint.
124 Align DstAlign; // Specified alignment of the memory operation.
125
126 bool AllowOverlap;
127 // memset only
128 bool IsMemset; // If setthis memory operation is a memset.
129 bool ZeroMemset; // If set clears out memory with zeros.
130 // memcpy only
131 bool MemcpyStrSrc; // Indicates whether the memcpy source is an in-register
132 // constant so it does not need to be loaded.
133 Align SrcAlign; // Inferred alignment of the source or default value if the
134 // memory operation does not need to load the value.
135public:
136 static MemOp Copy(uint64_t Size, bool DstAlignCanChange, Align DstAlign,
137 Align SrcAlign, bool IsVolatile,
138 bool MemcpyStrSrc = false) {
139 MemOp Op;
140 Op.Size = Size;
141 Op.DstAlignCanChange = DstAlignCanChange;
142 Op.DstAlign = DstAlign;
143 Op.AllowOverlap = !IsVolatile;
144 Op.IsMemset = false;
145 Op.ZeroMemset = false;
146 Op.MemcpyStrSrc = MemcpyStrSrc;
147 Op.SrcAlign = SrcAlign;
148 return Op;
149 }
150
151 static MemOp Set(uint64_t Size, bool DstAlignCanChange, Align DstAlign,
152 bool IsZeroMemset, bool IsVolatile) {
153 MemOp Op;
154 Op.Size = Size;
155 Op.DstAlignCanChange = DstAlignCanChange;
156 Op.DstAlign = DstAlign;
157 Op.AllowOverlap = !IsVolatile;
158 Op.IsMemset = true;
159 Op.ZeroMemset = IsZeroMemset;
160 Op.MemcpyStrSrc = false;
161 return Op;
162 }
163
164 uint64_t size() const { return Size; }
166 assert(!DstAlignCanChange);
167 return DstAlign;
168 }
169 bool isFixedDstAlign() const { return !DstAlignCanChange; }
170 bool allowOverlap() const { return AllowOverlap; }
171 bool isMemset() const { return IsMemset; }
172 bool isMemcpy() const { return !IsMemset; }
174 return isMemcpy() && !DstAlignCanChange;
175 }
176 bool isZeroMemset() const { return isMemset() && ZeroMemset; }
177 bool isMemcpyStrSrc() const {
178 assert(isMemcpy() && "Must be a memcpy");
179 return MemcpyStrSrc;
180 }
182 assert(isMemcpy() && "Must be a memcpy");
183 return SrcAlign;
184 }
185 bool isSrcAligned(Align AlignCheck) const {
186 return isMemset() || llvm::isAligned(AlignCheck, SrcAlign.value());
187 }
188 bool isDstAligned(Align AlignCheck) const {
189 return DstAlignCanChange || llvm::isAligned(AlignCheck, DstAlign.value());
190 }
191 bool isAligned(Align AlignCheck) const {
192 return isSrcAligned(AlignCheck) && isDstAligned(AlignCheck);
193 }
194};
195
196/// This base class for TargetLowering contains the SelectionDAG-independent
197/// parts that can be used from the rest of CodeGen.
199public:
200 /// This enum indicates whether operations are valid for a target, and if not,
201 /// what action should be used to make them valid.
203 Legal, // The target natively supports this operation.
204 Promote, // This operation should be executed in a larger type.
205 Expand, // Try to expand this to other ops, otherwise use a libcall.
206 LibCall, // Don't try to expand this to other ops, always use a libcall.
207 Custom // Use the LowerOperation hook to implement custom lowering.
208 };
209
210 /// This enum indicates whether a types are legal for a target, and if not,
211 /// what action should be used to make them valid.
213 TypeLegal, // The target natively supports this type.
214 TypePromoteInteger, // Replace this integer with a larger one.
215 TypeExpandInteger, // Split this integer into two of half the size.
216 TypeSoftenFloat, // Convert this float to a same size integer type.
217 TypeExpandFloat, // Split this float into two of half the size.
218 TypeScalarizeVector, // Replace this one-element vector with its element.
219 TypeSplitVector, // Split this vector into two of half the size.
220 TypeWidenVector, // This vector should be widened into a larger vector.
221 TypePromoteFloat, // Replace this float with a larger one.
222 TypeSoftPromoteHalf, // Soften half to i16 and use float to do arithmetic.
223 TypeScalarizeScalableVector, // This action is explicitly left unimplemented.
224 // While it is theoretically possible to
225 // legalize operations on scalable types with a
226 // loop that handles the vscale * #lanes of the
227 // vector, this is non-trivial at SelectionDAG
228 // level and these types are better to be
229 // widened or promoted.
230 };
231
232 /// LegalizeKind holds the legalization kind that needs to happen to EVT
233 /// in order to type-legalize it.
234 using LegalizeKind = std::pair<LegalizeTypeAction, EVT>;
235
236 /// Enum that describes how the target represents true/false values.
238 UndefinedBooleanContent, // Only bit 0 counts, the rest can hold garbage.
239 ZeroOrOneBooleanContent, // All bits zero except for bit 0.
240 ZeroOrNegativeOneBooleanContent // All bits equal to bit 0.
241 };
242
243 /// Enum that describes what type of support for selects the target has.
245 ScalarValSelect, // The target supports scalar selects (ex: cmov).
246 ScalarCondVectorVal, // The target supports selects with a scalar condition
247 // and vector values (ex: cmov).
248 VectorMaskSelect // The target supports vector selects with a vector
249 // mask (ex: x86 blends).
250 };
251
252 /// Enum that specifies what an atomic load/AtomicRMWInst is expanded
253 /// to, if at all. Exists because different targets have different levels of
254 /// support for these atomic instructions, and also have different options
255 /// w.r.t. what they should expand to.
257 None, // Don't expand the instruction.
258 CastToInteger, // Cast the atomic instruction to another type, e.g. from
259 // floating-point to integer type.
260 LLSC, // Expand the instruction into loadlinked/storeconditional; used
261 // by ARM/AArch64/PowerPC.
262 LLOnly, // Expand the (load) instruction into just a load-linked, which has
263 // greater atomic guarantees than a normal load.
264 CmpXChg, // Expand the instruction into cmpxchg; used by at least X86.
265 MaskedIntrinsic, // Use a target-specific intrinsic for the LL/SC loop.
266 BitTestIntrinsic, // Use a target-specific intrinsic for special bit
267 // operations; used by X86.
268 CmpArithIntrinsic, // Use a target-specific intrinsic for special compare
269 // operations; used by X86.
270 Expand, // Generic expansion in terms of other atomic operations.
271 CustomExpand, // Custom target-specific expansion using TLI hooks.
272
273 // Rewrite to a non-atomic form for use in a known non-preemptible
274 // environment.
276 };
277
278 /// Enum that specifies when a multiplication should be expanded.
279 enum class MulExpansionKind {
280 Always, // Always expand the instruction.
281 OnlyLegalOrCustom, // Only expand when the resulting instructions are legal
282 // or custom.
283 };
284
285 /// Enum that specifies when a float negation is beneficial.
286 enum class NegatibleCost {
287 Cheaper = 0, // Negated expression is cheaper.
288 Neutral = 1, // Negated expression has the same cost.
289 Expensive = 2 // Negated expression is more expensive.
290 };
291
292 /// Enum of different potentially desirable ways to fold (and/or (setcc ...),
293 /// (setcc ...)).
295 None = 0, // No fold is preferable.
296 AddAnd = 1, // Fold with `Add` op and `And` op is preferable.
297 NotAnd = 2, // Fold with `Not` op and `And` op is preferable.
298 ABS = 4, // Fold with `llvm.abs` op is preferable.
299 };
300
302 public:
305 /// Original unlegalized argument type.
307 /// Same as OrigTy, or partially legalized for soft float libcalls.
309 bool IsSExt : 1;
310 bool IsZExt : 1;
311 bool IsNoExt : 1;
312 bool IsInReg : 1;
313 bool IsSRet : 1;
314 bool IsNest : 1;
315 bool IsByVal : 1;
316 bool IsByRef : 1;
317 bool IsInAlloca : 1;
319 bool IsReturned : 1;
320 bool IsSwiftSelf : 1;
321 bool IsSwiftAsync : 1;
322 bool IsSwiftError : 1;
324 MaybeAlign Alignment = std::nullopt;
325 Type *IndirectType = nullptr;
326
333
336
338
339 LLVM_ABI void setAttributes(const CallBase *Call, unsigned ArgIdx);
340 };
341 using ArgListTy = std::vector<ArgListEntry>;
342
344 switch (Content) {
346 // Extend by adding rubbish bits.
347 return ISD::ANY_EXTEND;
349 // Extend by adding zero bits.
350 return ISD::ZERO_EXTEND;
352 // Extend by copying the sign bit.
353 return ISD::SIGN_EXTEND;
354 }
355 llvm_unreachable("Invalid content kind");
356 }
357
358 explicit TargetLoweringBase(const TargetMachine &TM);
362
363 /// Return true if the target support strict float operation
364 bool isStrictFPEnabled() const {
365 return IsStrictFPEnabled;
366 }
367
368protected:
369 /// Initialize all of the actions to default values.
370 void initActions();
371
372public:
373 const TargetMachine &getTargetMachine() const { return TM; }
374
375 virtual bool useSoftFloat() const { return false; }
376
377 /// Return the pointer type for the given address space, defaults to
378 /// the pointer type from the data layout.
379 /// FIXME: The default needs to be removed once all the code is updated.
380 virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS = 0) const {
381 return MVT::getIntegerVT(DL.getPointerSizeInBits(AS));
382 }
383
384 /// Return the in-memory pointer type for the given address space, defaults to
385 /// the pointer type from the data layout.
386 /// FIXME: The default needs to be removed once all the code is updated.
387 virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS = 0) const {
388 return MVT::getIntegerVT(DL.getPointerSizeInBits(AS));
389 }
390
391 /// Return the type for frame index, which is determined by
392 /// the alloca address space specified through the data layout.
394 return getPointerTy(DL, DL.getAllocaAddrSpace());
395 }
396
397 /// Return the type for code pointers, which is determined by the program
398 /// address space specified through the data layout.
400 return getPointerTy(DL, DL.getProgramAddressSpace());
401 }
402
403 /// Return the type for operands of fence.
404 /// TODO: Let fence operands be of i32 type and remove this.
405 virtual MVT getFenceOperandTy(const DataLayout &DL) const {
406 return getPointerTy(DL);
407 }
408
409 /// Return the type to use for a scalar shift opcode, given the shifted amount
410 /// type. Targets should return a legal type if the input type is legal.
411 /// Targets can return a type that is too small if the input type is illegal.
412 virtual MVT getScalarShiftAmountTy(const DataLayout &, EVT) const;
413
414 /// Returns the type for the shift amount of a shift opcode. For vectors,
415 /// returns the input type. For scalars, calls getScalarShiftAmountTy.
416 /// If getScalarShiftAmountTy type cannot represent all possible shift
417 /// amounts, returns MVT::i32.
418 EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const;
419
420 /// Return the preferred type to use for a shift opcode, given the shifted
421 /// amount type is \p ShiftValueTy.
423 virtual LLT getPreferredShiftAmountTy(LLT ShiftValueTy) const {
424 return ShiftValueTy;
425 }
426
427 /// Returns the type to be used for the index operand vector operations. By
428 /// default we assume it will have the same size as an address space 0
429 /// pointer.
430 virtual unsigned getVectorIdxWidth(const DataLayout &DL) const {
431 return DL.getPointerSizeInBits(0);
432 }
433
434 /// Returns the type to be used for the index operand of:
435 /// ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
436 /// ISD::INSERT_SUBVECTOR, and ISD::EXTRACT_SUBVECTOR
440
441 /// Returns the type to be used for the index operand of:
442 /// G_INSERT_VECTOR_ELT, G_EXTRACT_VECTOR_ELT,
443 /// G_INSERT_SUBVECTOR, and G_EXTRACT_SUBVECTOR
446 }
447
448 /// Returns the type to be used for the EVL/AVL operand of VP nodes:
449 /// ISD::VP_ADD, ISD::VP_SUB, etc. It must be a legal scalar integer type,
450 /// and must be at least as large as i32. The EVL is implicitly zero-extended
451 /// to any larger type.
452 virtual MVT getVPExplicitVectorLengthTy() const { return MVT::i32; }
453
454 /// This callback is used to inspect load/store instructions and add
455 /// target-specific MachineMemOperand flags to them. The default
456 /// implementation does nothing.
460
461 /// This callback is used to inspect load/store SDNode.
462 /// The default implementation does nothing.
467
469 getLoadMemOperandFlags(const LoadInst &LI, const DataLayout &DL,
470 AssumptionCache *AC = nullptr,
471 const TargetLibraryInfo *LibInfo = nullptr) const;
472 MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI,
473 const DataLayout &DL) const;
474 MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI,
475 const DataLayout &DL) const;
477 getVPIntrinsicMemOperandFlags(const VPIntrinsic &VPIntrin) const;
478
479 virtual bool isSelectSupported(SelectSupportKind /*kind*/) const {
480 return true;
481 }
482
483 /// Return true if the @llvm.get.active.lane.mask intrinsic should be expanded
484 /// using generic code in SelectionDAGBuilder.
485 virtual bool shouldExpandGetActiveLaneMask(EVT VT, EVT OpVT) const {
486 return true;
487 }
488
489 virtual bool shouldExpandGetVectorLength(EVT CountVT, unsigned VF,
490 bool IsScalable) const {
491 return true;
492 }
493
494 /// Return true if the @llvm.experimental.cttz.elts intrinsic should be
495 /// expanded using generic code in SelectionDAGBuilder.
496 virtual bool shouldExpandCttzElements(EVT VT) const { return true; }
497
498 /// Return the minimum number of bits required to hold the maximum possible
499 /// number of trailing zero vector elements.
500 unsigned getBitWidthForCttzElements(Type *RetTy, ElementCount EC,
501 bool ZeroIsPoison,
502 const ConstantRange *VScaleRange) const;
503
504 /// Return true if the @llvm.experimental.vector.match intrinsic should be
505 /// expanded for vector type `VT' and search size `SearchSize' using generic
506 /// code in SelectionDAGBuilder.
507 virtual bool shouldExpandVectorMatch(EVT VT, unsigned SearchSize) const {
508 return true;
509 }
510
511 // Return true if op(vecreduce(x), vecreduce(y)) should be reassociated to
512 // vecreduce(op(x, y)) for the reduction opcode RedOpc.
513 virtual bool shouldReassociateReduction(unsigned RedOpc, EVT VT) const {
514 return true;
515 }
516
517 /// Return true if it is profitable to convert a select of FP constants into
518 /// a constant pool load whose address depends on the select condition. The
519 /// parameter may be used to differentiate a select with FP compare from
520 /// integer compare.
521 virtual bool reduceSelectOfFPConstantLoads(EVT CmpOpVT) const {
522 return true;
523 }
524
525 /// Does the target have multiple (allocatable) condition registers that
526 /// can be used to store the results of comparisons for use by selects
527 /// and conditional branches. With multiple condition registers, the code
528 /// generator will not aggressively sink comparisons into the blocks of their
529 /// users.
530 virtual bool hasMultipleConditionRegisters(EVT VT) const { return false; }
531
532 /// Return true if the target has BitExtract instructions.
533 bool hasExtractBitsInsn() const { return HasExtractBitsInsn; }
534
535 /// Return the preferred vector type legalization action.
538 // The default action for one element vectors is to scalarize
540 return TypeScalarizeVector;
541 // The default action for an odd-width vector is to widen.
542 if (!VT.isPow2VectorType())
543 return TypeWidenVector;
544 // The default action for other vectors is to promote
545 return TypePromoteInteger;
546 }
547
548 // Return true if the half type should be promoted using soft promotion rules
549 // where each operation is promoted to f32 individually, then converted to
550 // fp16. The default behavior is to promote chains of operations, keeping
551 // intermediate results in f32 precision and range.
552 virtual bool softPromoteHalfType() const { return false; }
553
554 // Return true if, for soft-promoted half, the half type should be passed to
555 // and returned from functions as f32. The default behavior is to pass as
556 // i16. If soft-promoted half is not used, this function is ignored and
557 // values are always passed and returned as f32.
558 virtual bool useFPRegsForHalfType() const { return false; }
559
560 // There are two general methods for expanding a BUILD_VECTOR node:
561 // 1. Use SCALAR_TO_VECTOR on the defined scalar values and then shuffle
562 // them together.
563 // 2. Build the vector on the stack and then load it.
564 // If this function returns true, then method (1) will be used, subject to
565 // the constraint that all of the necessary shuffles are legal (as determined
566 // by isShuffleMaskLegal). If this function returns false, then method (2) is
567 // always used. The vector type, and the number of defined values, are
568 // provided.
569 virtual bool
571 unsigned DefinedValues) const {
572 return DefinedValues < 3;
573 }
574
575 /// Return true if integer divide is usually cheaper than a sequence of
576 /// several shifts, adds, and multiplies for this target.
577 /// The definition of "cheaper" may depend on whether we're optimizing
578 /// for speed or for size.
579 virtual bool isIntDivCheap(EVT VT, AttributeList Attr) const { return false; }
580
581 /// Return true if the target can handle a standalone remainder operation.
582 virtual bool hasStandaloneRem(EVT VT) const {
583 return true;
584 }
585
586 /// Return true if SQRT(X) shouldn't be replaced with X*RSQRT(X).
587 virtual bool isFsqrtCheap(SDValue X, SelectionDAG &DAG) const {
588 // Default behavior is to replace SQRT(X) with X*RSQRT(X).
589 return false;
590 }
591
592 /// Reciprocal estimate status values used by the functions below.
597 };
598
599 /// Return a ReciprocalEstimate enum value for a square root of the given type
600 /// based on the function's attributes. If the operation is not overridden by
601 /// the function's attributes, "Unspecified" is returned and target defaults
602 /// are expected to be used for instruction selection.
603 int getRecipEstimateSqrtEnabled(EVT VT, MachineFunction &MF) const;
604
605 /// Return a ReciprocalEstimate enum value for a division of the given type
606 /// based on the function's attributes. If the operation is not overridden by
607 /// the function's attributes, "Unspecified" is returned and target defaults
608 /// are expected to be used for instruction selection.
609 int getRecipEstimateDivEnabled(EVT VT, MachineFunction &MF) const;
610
611 /// Return the refinement step count for a square root of the given type based
612 /// on the function's attributes. If the operation is not overridden by
613 /// the function's attributes, "Unspecified" is returned and target defaults
614 /// are expected to be used for instruction selection.
615 int getSqrtRefinementSteps(EVT VT, MachineFunction &MF) const;
616
617 /// Return the refinement step count for a division of the given type based
618 /// on the function's attributes. If the operation is not overridden by
619 /// the function's attributes, "Unspecified" is returned and target defaults
620 /// are expected to be used for instruction selection.
621 int getDivRefinementSteps(EVT VT, MachineFunction &MF) const;
622
623 /// Returns true if target has indicated at least one type should be bypassed.
624 bool isSlowDivBypassed() const { return !BypassSlowDivWidths.empty(); }
625
626 /// Returns map of slow types for division or remainder with corresponding
627 /// fast types
629 return BypassSlowDivWidths;
630 }
631
632 /// Return true only if vscale must be a power of two.
633 virtual bool isVScaleKnownToBeAPowerOfTwo() const { return false; }
634
635 /// Return true if Flow Control is an expensive operation that should be
636 /// avoided.
637 bool isJumpExpensive() const { return JumpIsExpensive; }
638
639 // Costs parameters used by
640 // SelectionDAGBuilder::shouldKeepJumpConditionsTogether.
641 // shouldKeepJumpConditionsTogether will use these parameter value to
642 // determine if two conditions in the form `br (and/or cond1, cond2)` should
643 // be split into two branches or left as one.
644 //
645 // BaseCost is the cost threshold (in latency). If the estimated latency of
646 // computing both `cond1` and `cond2` is below the cost of just computing
647 // `cond1` + BaseCost, the two conditions will be kept together. Otherwise
648 // they will be split.
649 //
650 // LikelyBias increases BaseCost if branch probability info indicates that it
651 // is likely that both `cond1` and `cond2` will be computed.
652 //
653 // UnlikelyBias decreases BaseCost if branch probability info indicates that
654 // it is likely that both `cond1` and `cond2` will be computed.
655 //
656 // Set any field to -1 to make it ignored (setting BaseCost to -1 results in
657 // `shouldKeepJumpConditionsTogether` always returning false).
663 // Return params for deciding if we should keep two branch conditions merged
664 // or split them into two separate branches.
665 // Arg0: The binary op joining the two conditions (and/or).
666 // Arg1: The first condition (cond1)
667 // Arg2: The second condition (cond2)
668 virtual CondMergingParams
670 const Value *) const {
671 // -1 will always result in splitting.
672 return {-1, -1, -1};
673 }
674
675 /// Return true if selects are only cheaper than branches if the branch is
676 /// unlikely to be predicted right.
680
681 virtual bool fallBackToDAGISel(const Instruction &Inst) const {
682 return false;
683 }
684
685 /// Return true if the following transform is beneficial:
686 /// fold (conv (load x)) -> (load (conv*)x)
687 /// On architectures that don't natively support some vector loads
688 /// efficiently, casting the load to a smaller vector of larger types and
689 /// loading is more efficient, however, this can be undone by optimizations in
690 /// dag combiner.
691 virtual bool isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT,
692 const SelectionDAG &DAG,
693 const MachineMemOperand &MMO) const;
694
695 /// Return true if the following transform is beneficial:
696 /// (store (y (conv x)), y*)) -> (store x, (x*))
697 virtual bool isStoreBitCastBeneficial(EVT StoreVT, EVT BitcastVT,
698 const SelectionDAG &DAG,
699 const MachineMemOperand &MMO) const {
700 // Default to the same logic as loads.
701 return isLoadBitCastBeneficial(StoreVT, BitcastVT, DAG, MMO);
702 }
703
704 /// Return true if it is expected to be cheaper to do a store of vector
705 /// constant with the given size and type for the address space than to
706 /// store the individual scalar element constants.
707 virtual bool storeOfVectorConstantIsCheap(bool IsZero, EVT MemVT,
708 unsigned NumElem,
709 unsigned AddrSpace) const {
710 return IsZero;
711 }
712
713 /// Allow store merging for the specified type after legalization in addition
714 /// to before legalization. This may transform stores that do not exist
715 /// earlier (for example, stores created from intrinsics).
716 virtual bool mergeStoresAfterLegalization(EVT MemVT) const {
717 return true;
718 }
719
720 /// Returns if it's reasonable to merge stores to MemVT size.
721 virtual bool canMergeStoresTo(unsigned AS, EVT MemVT,
722 const MachineFunction &MF) const {
723 return true;
724 }
725
726 /// Return true if it is cheap to speculate a call to intrinsic cttz.
727 virtual bool isCheapToSpeculateCttz(Type *Ty) const {
728 return false;
729 }
730
731 /// Return true if it is cheap to speculate a call to intrinsic ctlz.
732 virtual bool isCheapToSpeculateCtlz(Type *Ty) const {
733 return false;
734 }
735
736 /// Return true if ctlz instruction is fast.
737 virtual bool isCtlzFast() const {
738 return false;
739 }
740
741 /// Return true if ctpop instruction is fast.
742 virtual bool isCtpopFast(EVT VT) const {
743 return isOperationLegal(ISD::CTPOP, VT);
744 }
745
746 /// Return the maximum number of "x & (x - 1)" operations that can be done
747 /// instead of deferring to a custom CTPOP.
748 virtual unsigned getCustomCtpopCost(EVT VT, ISD::CondCode Cond) const {
749 return 1;
750 }
751
752 /// Return true if instruction generated for equality comparison is folded
753 /// with instruction generated for signed comparison.
754 virtual bool isEqualityCmpFoldedWithSignedCmp() const { return true; }
755
756 /// Return true if the heuristic to prefer icmp eq zero should be used in code
757 /// gen prepare.
758 virtual bool preferZeroCompareBranch() const { return false; }
759
760 /// Return true if it is cheaper to split the store of a merged int val
761 /// from a pair of smaller values into multiple stores.
762 virtual bool isMultiStoresCheaperThanBitsMerge(EVT LTy, EVT HTy) const {
763 return false;
764 }
765
766 /// Return if the target supports combining a
767 /// chain like:
768 /// \code
769 /// %andResult = and %val1, #mask
770 /// %icmpResult = icmp %andResult, 0
771 /// \endcode
772 /// into a single machine instruction of a form like:
773 /// \code
774 /// cc = test %register, #mask
775 /// \endcode
776 virtual bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const {
777 return false;
778 }
779
780 /// Return true if it is valid to merge the TargetMMOFlags in two SDNodes.
781 virtual bool
783 const MemSDNode &NodeY) const {
784 return true;
785 }
786
787 /// Use bitwise logic to make pairs of compares more efficient. For example:
788 /// and (seteq A, B), (seteq C, D) --> seteq (or (xor A, B), (xor C, D)), 0
789 /// This should be true when it takes more than one instruction to lower
790 /// setcc (cmp+set on x86 scalar), when bitwise ops are faster than logic on
791 /// condition bits (crand on PowerPC), and/or when reducing cmp+br is a win.
792 virtual bool convertSetCCLogicToBitwiseLogic(EVT VT) const {
793 return false;
794 }
795
796 /// Return the preferred operand type if the target has a quick way to compare
797 /// integer values of the given size. Assume that any legal integer type can
798 /// be compared efficiently. Targets may override this to allow illegal wide
799 /// types to return a vector type if there is support to compare that type.
800 virtual MVT hasFastEqualityCompare(unsigned NumBits) const {
801 MVT VT = MVT::getIntegerVT(NumBits);
803 }
804
805 /// Return true if the target should transform:
806 /// (X & Y) == Y ---> (~X & Y) == 0
807 /// (X & Y) != Y ---> (~X & Y) != 0
808 ///
809 /// This may be profitable if the target has a bitwise and-not operation that
810 /// sets comparison flags. A target may want to limit the transformation based
811 /// on the type of Y or if Y is a constant.
812 ///
813 /// Note that the transform will not occur if Y is known to be a power-of-2
814 /// because a mask and compare of a single bit can be handled by inverting the
815 /// predicate, for example:
816 /// (X & 8) == 8 ---> (X & 8) != 0
817 virtual bool hasAndNotCompare(SDValue Y) const {
818 return false;
819 }
820
821 /// Return true if the target has a bitwise and-not operation:
822 /// X = ~A & B
823 /// This can be used to simplify select or other instructions.
824 virtual bool hasAndNot(SDValue X) const {
825 // If the target has the more complex version of this operation, assume that
826 // it has this operation too.
827 return hasAndNotCompare(X);
828 }
829
830 /// Return true if the target has a bit-test instruction:
831 /// (X & (1 << Y)) ==/!= 0
832 /// This knowledge can be used to prevent breaking the pattern,
833 /// or creating it if it could be recognized.
834 virtual bool hasBitTest(SDValue X, SDValue Y) const { return false; }
835
836 /// There are two ways to clear extreme bits (either low or high):
837 /// Mask: x & (-1 << y) (the instcombine canonical form)
838 /// Shifts: x >> y << y
839 /// Return true if the variant with 2 variable shifts is preferred.
840 /// Return false if there is no preference.
842 // By default, let's assume that no one prefers shifts.
843 return false;
844 }
845
846 /// Return true if it is profitable to fold a pair of shifts into a mask.
847 /// This is usually true on most targets. But some targets, like Thumb1,
848 /// have immediate shift instructions, but no immediate "and" instruction;
849 /// this makes the fold unprofitable.
851 CombineLevel Level) const {
852 return true;
853 }
854
855 /// Should we tranform the IR-optimal check for whether given truncation
856 /// down into KeptBits would be truncating or not:
857 /// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)
858 /// Into it's more traditional form:
859 /// ((%x << C) a>> C) dstcond %x
860 /// Return true if we should transform.
861 /// Return false if there is no preference.
863 unsigned KeptBits) const {
864 // By default, let's assume that no one prefers shifts.
865 return false;
866 }
867
868 /// Given the pattern
869 /// (X & (C l>>/<< Y)) ==/!= 0
870 /// return true if it should be transformed into:
871 /// ((X <</l>> Y) & C) ==/!= 0
872 /// WARNING: if 'X' is a constant, the fold may deadlock!
873 /// FIXME: we could avoid passing XC, but we can't use isConstOrConstSplat()
874 /// here because it can end up being not linked in.
877 unsigned OldShiftOpcode, unsigned NewShiftOpcode,
878 SelectionDAG &DAG) const {
879 if (hasBitTest(X, Y)) {
880 // One interesting pattern that we'd want to form is 'bit test':
881 // ((1 << Y) & C) ==/!= 0
882 // But we also need to be careful not to try to reverse that fold.
883
884 // Is this '1 << Y' ?
885 if (OldShiftOpcode == ISD::SHL && CC->isOne())
886 return false; // Keep the 'bit test' pattern.
887
888 // Will it be '1 << Y' after the transform ?
889 if (XC && NewShiftOpcode == ISD::SHL && XC->isOne())
890 return true; // Do form the 'bit test' pattern.
891 }
892
893 // If 'X' is a constant, and we transform, then we will immediately
894 // try to undo the fold, thus causing endless combine loop.
895 // So by default, let's assume everyone prefers the fold
896 // iff 'X' is not a constant.
897 return !XC;
898 }
899
900 // Return true if its desirable to perform the following transform:
901 // (fmul C, (uitofp Pow2))
902 // -> (bitcast_to_FP (add (bitcast_to_INT C), Log2(Pow2) << mantissa))
903 // (fdiv C, (uitofp Pow2))
904 // -> (bitcast_to_FP (sub (bitcast_to_INT C), Log2(Pow2) << mantissa))
905 //
906 // This is only queried after we have verified the transform will be bitwise
907 // equals.
908 //
909 // SDNode *N : The FDiv/FMul node we want to transform.
910 // SDValue FPConst: The Float constant operand in `N`.
911 // SDValue IntPow2: The Integer power of 2 operand in `N`.
913 SDValue IntPow2) const {
914 // Default to avoiding fdiv which is often very expensive.
915 return N->getOpcode() == ISD::FDIV;
916 }
917
918 // Given:
919 // (icmp eq/ne (and X, C0), (shift X, C1))
920 // or
921 // (icmp eq/ne X, (rotate X, CPow2))
922
923 // If C0 is a mask or shifted mask and the shift amt (C1) isolates the
924 // remaining bits (i.e something like `(x64 & UINT32_MAX) == (x64 >> 32)`)
925 // Do we prefer the shift to be shift-right, shift-left, or rotate.
926 // Note: Its only valid to convert the rotate version to the shift version iff
927 // the shift-amt (`C1`) is a power of 2 (including 0).
928 // If ShiftOpc (current Opcode) is returned, do nothing.
930 EVT VT, unsigned ShiftOpc, bool MayTransformRotate,
931 const APInt &ShiftOrRotateAmt,
932 const std::optional<APInt> &AndMask) const {
933 return ShiftOpc;
934 }
935
936 /// These two forms are equivalent:
937 /// sub %y, (xor %x, -1)
938 /// add (add %x, 1), %y
939 /// The variant with two add's is IR-canonical.
940 /// Some targets may prefer one to the other.
941 virtual bool preferIncOfAddToSubOfNot(EVT VT) const {
942 // By default, let's assume that everyone prefers the form with two add's.
943 return true;
944 }
945
946 // By default prefer folding (abs (sub nsw x, y)) -> abds(x, y). Some targets
947 // may want to avoid this to prevent loss of sub_nsw pattern.
948 virtual bool preferABDSToABSWithNSW(EVT VT) const {
949 return true;
950 }
951
952 // Return true if the target wants to transform Op(Splat(X)) -> Splat(Op(X))
953 virtual bool preferScalarizeSplat(SDNode *N) const { return true; }
954
955 // Return true if the target wants to transform:
956 // (TruncVT truncate(sext_in_reg(VT X, ExtVT))
957 // -> (TruncVT sext_in_reg(truncate(VT X), ExtVT))
958 // Some targets might prefer pre-sextinreg to improve truncation/saturation.
959 virtual bool preferSextInRegOfTruncate(EVT TruncVT, EVT VT, EVT ExtVT) const {
960 return true;
961 }
962
963 /// Return true if the target wants to use the optimization that
964 /// turns ext(promotableInst1(...(promotableInstN(load)))) into
965 /// promotedInst1(...(promotedInstN(ext(load)))).
967
968 /// Return true if the target can combine store(extractelement VectorTy,
969 /// Idx).
970 /// \p Cost[out] gives the cost of that transformation when this is true.
971 virtual bool canCombineStoreAndExtract(Type *VectorTy, Value *Idx,
972 unsigned &Cost) const {
973 return false;
974 }
975
976 /// Return true if the target shall perform extract vector element and store
977 /// given that the vector is known to be splat of constant.
978 /// \p Index[out] gives the index of the vector element to be extracted when
979 /// this is true.
981 Type *VectorTy, unsigned ElemSizeInBits, unsigned &Index) const {
982 return false;
983 }
984
985 /// Return true if inserting a scalar into a variable element of an undef
986 /// vector is more efficiently handled by splatting the scalar instead.
987 virtual bool shouldSplatInsEltVarIndex(EVT) const {
988 return false;
989 }
990
991 /// Return true if target always benefits from combining into FMA for a
992 /// given value type. This must typically return false on targets where FMA
993 /// takes more cycles to execute than FADD.
994 virtual bool enableAggressiveFMAFusion(EVT VT) const { return false; }
995
996 /// Return true if target always benefits from combining into FMA for a
997 /// given value type. This must typically return false on targets where FMA
998 /// takes more cycles to execute than FADD.
999 virtual bool enableAggressiveFMAFusion(LLT Ty) const { return false; }
1000
1001 /// Return the ValueType of the result of SETCC operations.
1002 virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
1003 EVT VT) const;
1004
1005 /// Return the ValueType for comparison libcalls. Comparison libcalls include
1006 /// floating point comparison calls, and Ordered/Unordered check calls on
1007 /// floating point numbers.
1008 virtual
1009 MVT::SimpleValueType getCmpLibcallReturnType() const;
1010
1011 /// For targets without i1 registers, this gives the nature of the high-bits
1012 /// of boolean values held in types wider than i1.
1013 ///
1014 /// "Boolean values" are special true/false values produced by nodes like
1015 /// SETCC and consumed (as the condition) by nodes like SELECT and BRCOND.
1016 /// Not to be confused with general values promoted from i1. Some cpus
1017 /// distinguish between vectors of boolean and scalars; the isVec parameter
1018 /// selects between the two kinds. For example on X86 a scalar boolean should
1019 /// be zero extended from i1, while the elements of a vector of booleans
1020 /// should be sign extended from i1.
1021 ///
1022 /// Some cpus also treat floating point types the same way as they treat
1023 /// vectors instead of the way they treat scalars.
1024 BooleanContent getBooleanContents(bool isVec, bool isFloat) const {
1025 if (isVec)
1026 return BooleanVectorContents;
1027 return isFloat ? BooleanFloatContents : BooleanContents;
1028 }
1029
1031 return getBooleanContents(Type.isVector(), Type.isFloatingPoint());
1032 }
1033
1034 /// Promote the given target boolean to a target boolean of the given type.
1035 /// A target boolean is an integer value, not necessarily of type i1, the bits
1036 /// of which conform to getBooleanContents.
1037 ///
1038 /// ValVT is the type of values that produced the boolean.
1040 EVT ValVT) const {
1041 SDLoc dl(Bool);
1042 EVT BoolVT =
1043 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ValVT);
1045 return DAG.getNode(ExtendCode, dl, BoolVT, Bool);
1046 }
1047
1048 /// Return target scheduling preference.
1050 return SchedPreferenceInfo;
1051 }
1052
1053 /// Some scheduler, e.g. hybrid, can switch to different scheduling heuristics
1054 /// for different nodes. This function returns the preference (or none) for
1055 /// the given node.
1057 return Sched::None;
1058 }
1059
1060 /// Return the register class that should be used for the specified value
1061 /// type.
1062 virtual const TargetRegisterClass *getRegClassFor(MVT VT, bool isDivergent = false) const {
1063 (void)isDivergent;
1064 const TargetRegisterClass *RC = RegClassForVT[VT.SimpleTy];
1065 assert(RC && "This value type is not natively supported!");
1066 return RC;
1067 }
1068
1069 /// Allows target to decide about the register class of the
1070 /// specific value that is live outside the defining block.
1071 /// Returns true if the value needs uniform register class.
1073 const Value *) const {
1074 return false;
1075 }
1076
1077 /// Return the 'representative' register class for the specified value
1078 /// type.
1079 ///
1080 /// The 'representative' register class is the largest legal super-reg
1081 /// register class for the register class of the value type. For example, on
1082 /// i386 the rep register class for i8, i16, and i32 are GR32; while the rep
1083 /// register class is GR64 on x86_64.
1084 virtual const TargetRegisterClass *getRepRegClassFor(MVT VT) const {
1085 const TargetRegisterClass *RC = RepRegClassForVT[VT.SimpleTy];
1086 return RC;
1087 }
1088
1089 /// Return the cost of the 'representative' register class for the specified
1090 /// value type.
1092 return RepRegClassCostForVT[VT.SimpleTy];
1093 }
1094
1095 /// Return the preferred strategy to legalize tihs SHIFT instruction, with
1096 /// \p ExpansionFactor being the recursion depth - how many expansion needed.
1102 virtual ShiftLegalizationStrategy
1104 unsigned ExpansionFactor) const {
1105 if (ExpansionFactor == 1)
1108 }
1109
1110 /// Return true if the target has native support for the specified value type.
1111 /// This means that it has a register that directly holds it without
1112 /// promotions or expansions.
1113 bool isTypeLegal(EVT VT) const {
1114 assert(!VT.isSimple() ||
1115 (unsigned)VT.getSimpleVT().SimpleTy < std::size(RegClassForVT));
1116 return VT.isSimple() && RegClassForVT[VT.getSimpleVT().SimpleTy] != nullptr;
1117 }
1118
1120 /// ValueTypeActions - For each value type, keep a LegalizeTypeAction enum
1121 /// that indicates how instruction selection should deal with the type.
1122 LegalizeTypeAction ValueTypeActions[MVT::VALUETYPE_SIZE];
1123
1124 public:
1125 ValueTypeActionImpl() { llvm::fill(ValueTypeActions, TypeLegal); }
1126
1128 return ValueTypeActions[VT.SimpleTy];
1129 }
1130
1132 ValueTypeActions[VT.SimpleTy] = Action;
1133 }
1134 };
1135
1137 return ValueTypeActions;
1138 }
1139
1140 /// Return pair that represents the legalization kind (first) that needs to
1141 /// happen to EVT (second) in order to type-legalize it.
1142 ///
1143 /// First: how we should legalize values of this type, either it is already
1144 /// legal (return 'Legal') or we need to promote it to a larger type (return
1145 /// 'Promote'), or we need to expand it into multiple registers of smaller
1146 /// integer type (return 'Expand'). 'Custom' is not an option.
1147 ///
1148 /// Second: for types supported by the target, this is an identity function.
1149 /// For types that must be promoted to larger types, this returns the larger
1150 /// type to promote to. For integer types that are larger than the largest
1151 /// integer register, this contains one step in the expansion to get to the
1152 /// smaller register. For illegal floating point types, this returns the
1153 /// integer type to transform to.
1154 LegalizeKind getTypeConversion(LLVMContext &Context, EVT VT) const;
1155
1156 /// Return how we should legalize values of this type, either it is already
1157 /// legal (return 'Legal') or we need to promote it to a larger type (return
1158 /// 'Promote'), or we need to expand it into multiple registers of smaller
1159 /// integer type (return 'Expand'). 'Custom' is not an option.
1161 return getTypeConversion(Context, VT).first;
1162 }
1164 return ValueTypeActions.getTypeAction(VT);
1165 }
1166
1167 /// For types supported by the target, this is an identity function. For
1168 /// types that must be promoted to larger types, this returns the larger type
1169 /// to promote to. For integer types that are larger than the largest integer
1170 /// register, this contains one step in the expansion to get to the smaller
1171 /// register. For illegal floating point types, this returns the integer type
1172 /// to transform to.
1173 virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const {
1174 return getTypeConversion(Context, VT).second;
1175 }
1176
1177 /// For types supported by the target, this is an identity function. For
1178 /// types that must be expanded (i.e. integer types that are larger than the
1179 /// largest integer register or illegal floating point types), this returns
1180 /// the largest legal type it will be expanded to.
1181 EVT getTypeToExpandTo(LLVMContext &Context, EVT VT) const {
1182 assert(!VT.isVector());
1183 while (true) {
1184 switch (getTypeAction(Context, VT)) {
1185 case TypeLegal:
1186 return VT;
1187 case TypeExpandInteger:
1188 VT = getTypeToTransformTo(Context, VT);
1189 break;
1190 default:
1191 llvm_unreachable("Type is not legal nor is it to be expanded!");
1192 }
1193 }
1194 }
1195
1196 /// Vector types are broken down into some number of legal first class types.
1197 /// For example, EVT::v8f32 maps to 2 EVT::v4f32 with Altivec or SSE1, or 8
1198 /// promoted EVT::f64 values with the X86 FP stack. Similarly, EVT::v2i64
1199 /// turns into 4 EVT::i32 values with both PPC and X86.
1200 ///
1201 /// This method returns the number of registers needed, and the VT for each
1202 /// register. It also returns the VT and quantity of the intermediate values
1203 /// before they are promoted/expanded.
1204 unsigned getVectorTypeBreakdown(LLVMContext &Context, EVT VT,
1205 EVT &IntermediateVT,
1206 unsigned &NumIntermediates,
1207 MVT &RegisterVT) const;
1208
1209 /// Certain targets such as MIPS require that some types such as vectors are
1210 /// always broken down into scalars in some contexts. This occurs even if the
1211 /// vector type is legal.
1213 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
1214 unsigned &NumIntermediates, MVT &RegisterVT) const {
1215 return getVectorTypeBreakdown(Context, VT, IntermediateVT, NumIntermediates,
1216 RegisterVT);
1217 }
1218
1220 unsigned opc = 0; // target opcode
1221 EVT memVT; // memory VT
1222
1223 // value representing memory location
1225
1226 // Fallback address space for use if ptrVal is nullptr. std::nullopt means
1227 // unknown address space.
1228 std::optional<unsigned> fallbackAddressSpace;
1229
1230 int offset = 0; // offset off of ptrVal
1231 uint64_t size = 0; // the size of the memory location
1232 // (taken from memVT if zero)
1233 MaybeAlign align = Align(1); // alignment
1234
1239 IntrinsicInfo() = default;
1240 };
1241
1242 /// Given an intrinsic, checks if on the target the intrinsic will need to map
1243 /// to a MemIntrinsicNode (touches memory). If this is the case, it returns
1244 /// true and store the intrinsic information into the IntrinsicInfo that was
1245 /// passed to the function.
1248 unsigned /*Intrinsic*/) const {
1249 return false;
1250 }
1251
1252 /// Returns true if the target can instruction select the specified FP
1253 /// immediate natively. If false, the legalizer will materialize the FP
1254 /// immediate as a load from a constant pool.
1255 virtual bool isFPImmLegal(const APFloat & /*Imm*/, EVT /*VT*/,
1256 bool ForCodeSize = false) const {
1257 return false;
1258 }
1259
1260 /// Targets can use this to indicate that they only support *some*
1261 /// VECTOR_SHUFFLE operations, those with specific masks. By default, if a
1262 /// target supports the VECTOR_SHUFFLE node, all mask values are assumed to be
1263 /// legal.
1264 virtual bool isShuffleMaskLegal(ArrayRef<int> /*Mask*/, EVT /*VT*/) const {
1265 return true;
1266 }
1267
1268 /// Returns true if the operation can trap for the value type.
1269 ///
1270 /// VT must be a legal type. By default, we optimistically assume most
1271 /// operations don't trap except for integer divide and remainder.
1272 virtual bool canOpTrap(unsigned Op, EVT VT) const;
1273
1274 /// Similar to isShuffleMaskLegal. Targets can use this to indicate if there
1275 /// is a suitable VECTOR_SHUFFLE that can be used to replace a VAND with a
1276 /// constant pool entry.
1278 EVT /*VT*/) const {
1279 return false;
1280 }
1281
1282 /// How to legalize this custom operation?
1284 return Legal;
1285 }
1286
1287 /// Return how this operation should be treated: either it is legal, needs to
1288 /// be promoted to a larger size, needs to be expanded to some other code
1289 /// sequence, or the target has a custom expander for it.
1291 // If a target-specific SDNode requires legalization, require the target
1292 // to provide custom legalization for it.
1293 if (Op >= std::size(OpActions[0]))
1294 return Custom;
1295 if (VT.isExtended())
1296 return Expand;
1297 return OpActions[(unsigned)VT.getSimpleVT().SimpleTy][Op];
1298 }
1299
1300 /// Custom method defined by each target to indicate if an operation which
1301 /// may require a scale is supported natively by the target.
1302 /// If not, the operation is illegal.
1303 virtual bool isSupportedFixedPointOperation(unsigned Op, EVT VT,
1304 unsigned Scale) const {
1305 return false;
1306 }
1307
1308 /// Some fixed point operations may be natively supported by the target but
1309 /// only for specific scales. This method allows for checking
1310 /// if the width is supported by the target for a given operation that may
1311 /// depend on scale.
1313 unsigned Scale) const {
1314 auto Action = getOperationAction(Op, VT);
1315 if (Action != Legal)
1316 return Action;
1317
1318 // This operation is supported in this type but may only work on specific
1319 // scales.
1320 bool Supported;
1321 switch (Op) {
1322 default:
1323 llvm_unreachable("Unexpected fixed point operation.");
1324 case ISD::SMULFIX:
1325 case ISD::SMULFIXSAT:
1326 case ISD::UMULFIX:
1327 case ISD::UMULFIXSAT:
1328 case ISD::SDIVFIX:
1329 case ISD::SDIVFIXSAT:
1330 case ISD::UDIVFIX:
1331 case ISD::UDIVFIXSAT:
1332 Supported = isSupportedFixedPointOperation(Op, VT, Scale);
1333 break;
1334 }
1335
1336 return Supported ? Action : Expand;
1337 }
1338
1339 // If Op is a strict floating-point operation, return the result
1340 // of getOperationAction for the equivalent non-strict operation.
1342 unsigned EqOpc;
1343 switch (Op) {
1344 default: llvm_unreachable("Unexpected FP pseudo-opcode");
1345#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1346 case ISD::STRICT_##DAGN: EqOpc = ISD::DAGN; break;
1347#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1348 case ISD::STRICT_##DAGN: EqOpc = ISD::SETCC; break;
1349#include "llvm/IR/ConstrainedOps.def"
1350 }
1351
1352 return getOperationAction(EqOpc, VT);
1353 }
1354
1355 /// Return true if the specified operation is legal on this target or can be
1356 /// made legal with custom lowering. This is used to help guide high-level
1357 /// lowering decisions. LegalOnly is an optional convenience for code paths
1358 /// traversed pre and post legalisation.
1360 bool LegalOnly = false) const {
1361 if (LegalOnly)
1362 return isOperationLegal(Op, VT);
1363
1364 return (VT == MVT::Other || isTypeLegal(VT)) &&
1365 (getOperationAction(Op, VT) == Legal ||
1366 getOperationAction(Op, VT) == Custom);
1367 }
1368
1369 /// Return true if the specified operation is legal on this target or can be
1370 /// made legal using promotion. This is used to help guide high-level lowering
1371 /// decisions. LegalOnly is an optional convenience for code paths traversed
1372 /// pre and post legalisation.
1374 bool LegalOnly = false) const {
1375 if (LegalOnly)
1376 return isOperationLegal(Op, VT);
1377
1378 return (VT == MVT::Other || isTypeLegal(VT)) &&
1379 (getOperationAction(Op, VT) == Legal ||
1380 getOperationAction(Op, VT) == Promote);
1381 }
1382
1383 /// Return true if the specified operation is legal on this target or can be
1384 /// made legal with custom lowering or using promotion. This is used to help
1385 /// guide high-level lowering decisions. LegalOnly is an optional convenience
1386 /// for code paths traversed pre and post legalisation.
1388 bool LegalOnly = false) const {
1389 if (LegalOnly)
1390 return isOperationLegal(Op, VT);
1391
1392 return (VT == MVT::Other || isTypeLegal(VT)) &&
1393 (getOperationAction(Op, VT) == Legal ||
1394 getOperationAction(Op, VT) == Custom ||
1395 getOperationAction(Op, VT) == Promote);
1396 }
1397
1398 /// Return true if the operation uses custom lowering, regardless of whether
1399 /// the type is legal or not.
1400 bool isOperationCustom(unsigned Op, EVT VT) const {
1401 return getOperationAction(Op, VT) == Custom;
1402 }
1403
1404 /// Return true if lowering to a jump table is allowed.
1405 virtual bool areJTsAllowed(const Function *Fn) const {
1406 if (Fn->getFnAttribute("no-jump-tables").getValueAsBool())
1407 return false;
1408
1409 return isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) ||
1410 isOperationLegalOrCustom(ISD::BRIND, MVT::Other);
1411 }
1412
1413 /// Check whether the range [Low,High] fits in a machine word.
1414 bool rangeFitsInWord(const APInt &Low, const APInt &High,
1415 const DataLayout &DL) const {
1416 // FIXME: Using the pointer type doesn't seem ideal.
1417 uint64_t BW = DL.getIndexSizeInBits(0u);
1418 uint64_t Range = (High - Low).getLimitedValue(UINT64_MAX - 1) + 1;
1419 return Range <= BW;
1420 }
1421
1422 /// Return true if lowering to a jump table is suitable for a set of case
1423 /// clusters which may contain \p NumCases cases, \p Range range of values.
1424 virtual bool isSuitableForJumpTable(const SwitchInst *SI, uint64_t NumCases,
1426 BlockFrequencyInfo *BFI) const;
1427
1428 /// Returns preferred type for switch condition.
1429 virtual MVT getPreferredSwitchConditionType(LLVMContext &Context,
1430 EVT ConditionVT) const;
1431
1432 /// Return true if lowering to a bit test is suitable for a set of case
1433 /// clusters which contains \p NumDests unique destinations, \p Low and
1434 /// \p High as its lowest and highest case values, and expects \p NumCmps
1435 /// case value comparisons. Check if the number of destinations, comparison
1436 /// metric, and range are all suitable.
1437 bool isSuitableForBitTests(unsigned NumDests, unsigned NumCmps,
1438 const APInt &Low, const APInt &High,
1439 const DataLayout &DL) const {
1440 // FIXME: I don't think NumCmps is the correct metric: a single case and a
1441 // range of cases both require only one branch to lower. Just looking at the
1442 // number of clusters and destinations should be enough to decide whether to
1443 // build bit tests.
1444
1445 // To lower a range with bit tests, the range must fit the bitwidth of a
1446 // machine word.
1447 if (!rangeFitsInWord(Low, High, DL))
1448 return false;
1449
1450 // Decide whether it's profitable to lower this range with bit tests. Each
1451 // destination requires a bit test and branch, and there is an overall range
1452 // check branch. For a small number of clusters, separate comparisons might
1453 // be cheaper, and for many destinations, splitting the range might be
1454 // better.
1455 return (NumDests == 1 && NumCmps >= 3) || (NumDests == 2 && NumCmps >= 5) ||
1456 (NumDests == 3 && NumCmps >= 6);
1457 }
1458
1459 /// Return true if the specified operation is illegal on this target or
1460 /// unlikely to be made legal with custom lowering. This is used to help guide
1461 /// high-level lowering decisions.
1462 bool isOperationExpand(unsigned Op, EVT VT) const {
1463 return (!isTypeLegal(VT) || getOperationAction(Op, VT) == Expand);
1464 }
1465
1466 /// Return true if the specified operation is legal on this target.
1467 bool isOperationLegal(unsigned Op, EVT VT) const {
1468 return (VT == MVT::Other || isTypeLegal(VT)) &&
1469 getOperationAction(Op, VT) == Legal;
1470 }
1471
1472 /// Return how this load with extension should be treated: either it is legal,
1473 /// needs to be promoted to a larger size, needs to be expanded to some other
1474 /// code sequence, or the target has a custom expander for it.
1475 LegalizeAction getLoadExtAction(unsigned ExtType, EVT ValVT,
1476 EVT MemVT) const {
1477 if (ValVT.isExtended() || MemVT.isExtended()) return Expand;
1478 unsigned ValI = (unsigned) ValVT.getSimpleVT().SimpleTy;
1479 unsigned MemI = (unsigned) MemVT.getSimpleVT().SimpleTy;
1481 MemI < MVT::VALUETYPE_SIZE && "Table isn't big enough!");
1482 unsigned Shift = 4 * ExtType;
1483 return (LegalizeAction)((LoadExtActions[ValI][MemI] >> Shift) & 0xf);
1484 }
1485
1486 /// Return true if the specified load with extension is legal on this target.
1487 bool isLoadExtLegal(unsigned ExtType, EVT ValVT, EVT MemVT) const {
1488 return getLoadExtAction(ExtType, ValVT, MemVT) == Legal;
1489 }
1490
1491 /// Return true if the specified load with extension is legal or custom
1492 /// on this target.
1493 bool isLoadExtLegalOrCustom(unsigned ExtType, EVT ValVT, EVT MemVT) const {
1494 return getLoadExtAction(ExtType, ValVT, MemVT) == Legal ||
1495 getLoadExtAction(ExtType, ValVT, MemVT) == Custom;
1496 }
1497
1498 /// Same as getLoadExtAction, but for atomic loads.
1500 EVT MemVT) const {
1501 if (ValVT.isExtended() || MemVT.isExtended()) return Expand;
1502 unsigned ValI = (unsigned)ValVT.getSimpleVT().SimpleTy;
1503 unsigned MemI = (unsigned)MemVT.getSimpleVT().SimpleTy;
1505 MemI < MVT::VALUETYPE_SIZE && "Table isn't big enough!");
1506 unsigned Shift = 4 * ExtType;
1507 LegalizeAction Action =
1508 (LegalizeAction)((AtomicLoadExtActions[ValI][MemI] >> Shift) & 0xf);
1509 assert((Action == Legal || Action == Expand) &&
1510 "Unsupported atomic load extension action.");
1511 return Action;
1512 }
1513
1514 /// Return true if the specified atomic load with extension is legal on
1515 /// this target.
1516 bool isAtomicLoadExtLegal(unsigned ExtType, EVT ValVT, EVT MemVT) const {
1517 return getAtomicLoadExtAction(ExtType, ValVT, MemVT) == Legal;
1518 }
1519
1520 /// Return how this store with truncation should be treated: either it is
1521 /// legal, needs to be promoted to a larger size, needs to be expanded to some
1522 /// other code sequence, or the target has a custom expander for it.
1524 if (ValVT.isExtended() || MemVT.isExtended()) return Expand;
1525 unsigned ValI = (unsigned) ValVT.getSimpleVT().SimpleTy;
1526 unsigned MemI = (unsigned) MemVT.getSimpleVT().SimpleTy;
1528 "Table isn't big enough!");
1529 return TruncStoreActions[ValI][MemI];
1530 }
1531
1532 /// Return true if the specified store with truncation is legal on this
1533 /// target.
1534 bool isTruncStoreLegal(EVT ValVT, EVT MemVT) const {
1535 return isTypeLegal(ValVT) && getTruncStoreAction(ValVT, MemVT) == Legal;
1536 }
1537
1538 /// Return true if the specified store with truncation has solution on this
1539 /// target.
1540 bool isTruncStoreLegalOrCustom(EVT ValVT, EVT MemVT) const {
1541 return isTypeLegal(ValVT) &&
1542 (getTruncStoreAction(ValVT, MemVT) == Legal ||
1543 getTruncStoreAction(ValVT, MemVT) == Custom);
1544 }
1545
1546 virtual bool canCombineTruncStore(EVT ValVT, EVT MemVT,
1547 bool LegalOnly) const {
1548 if (LegalOnly)
1549 return isTruncStoreLegal(ValVT, MemVT);
1550
1551 return isTruncStoreLegalOrCustom(ValVT, MemVT);
1552 }
1553
1554 /// Return how the indexed load should be treated: either it is legal, needs
1555 /// to be promoted to a larger size, needs to be expanded to some other code
1556 /// sequence, or the target has a custom expander for it.
1557 LegalizeAction getIndexedLoadAction(unsigned IdxMode, MVT VT) const {
1558 return getIndexedModeAction(IdxMode, VT, IMAB_Load);
1559 }
1560
1561 /// Return true if the specified indexed load is legal on this target.
1562 bool isIndexedLoadLegal(unsigned IdxMode, EVT VT) const {
1563 return VT.isSimple() &&
1564 (getIndexedLoadAction(IdxMode, VT.getSimpleVT()) == Legal ||
1565 getIndexedLoadAction(IdxMode, VT.getSimpleVT()) == Custom);
1566 }
1567
1568 /// Return how the indexed store should be treated: either it is legal, needs
1569 /// to be promoted to a larger size, needs to be expanded to some other code
1570 /// sequence, or the target has a custom expander for it.
1571 LegalizeAction getIndexedStoreAction(unsigned IdxMode, MVT VT) const {
1572 return getIndexedModeAction(IdxMode, VT, IMAB_Store);
1573 }
1574
1575 /// Return true if the specified indexed load is legal on this target.
1576 bool isIndexedStoreLegal(unsigned IdxMode, EVT VT) const {
1577 return VT.isSimple() &&
1578 (getIndexedStoreAction(IdxMode, VT.getSimpleVT()) == Legal ||
1579 getIndexedStoreAction(IdxMode, VT.getSimpleVT()) == Custom);
1580 }
1581
1582 /// Return how the indexed load should be treated: either it is legal, needs
1583 /// to be promoted to a larger size, needs to be expanded to some other code
1584 /// sequence, or the target has a custom expander for it.
1585 LegalizeAction getIndexedMaskedLoadAction(unsigned IdxMode, MVT VT) const {
1586 return getIndexedModeAction(IdxMode, VT, IMAB_MaskedLoad);
1587 }
1588
1589 /// Return true if the specified indexed load is legal on this target.
1590 bool isIndexedMaskedLoadLegal(unsigned IdxMode, EVT VT) const {
1591 return VT.isSimple() &&
1592 (getIndexedMaskedLoadAction(IdxMode, VT.getSimpleVT()) == Legal ||
1594 }
1595
1596 /// Return how the indexed store should be treated: either it is legal, needs
1597 /// to be promoted to a larger size, needs to be expanded to some other code
1598 /// sequence, or the target has a custom expander for it.
1599 LegalizeAction getIndexedMaskedStoreAction(unsigned IdxMode, MVT VT) const {
1600 return getIndexedModeAction(IdxMode, VT, IMAB_MaskedStore);
1601 }
1602
1603 /// Return true if the specified indexed load is legal on this target.
1604 bool isIndexedMaskedStoreLegal(unsigned IdxMode, EVT VT) const {
1605 return VT.isSimple() &&
1606 (getIndexedMaskedStoreAction(IdxMode, VT.getSimpleVT()) == Legal ||
1608 }
1609
1610 /// Returns true if the index type for a masked gather/scatter requires
1611 /// extending
1612 virtual bool shouldExtendGSIndex(EVT VT, EVT &EltTy) const { return false; }
1613
1614 // Returns true if Extend can be folded into the index of a masked gathers/scatters
1615 // on this target.
1616 virtual bool shouldRemoveExtendFromGSIndex(SDValue Extend, EVT DataVT) const {
1617 return false;
1618 }
1619
1620 // Return true if the target supports a scatter/gather instruction with
1621 // indices which are scaled by the particular value. Note that all targets
1622 // must by definition support scale of 1.
1624 uint64_t ElemSize) const {
1625 // MGATHER/MSCATTER are only required to support scaling by one or by the
1626 // element size.
1627 if (Scale != ElemSize && Scale != 1)
1628 return false;
1629 return true;
1630 }
1631
1632 /// Return how the condition code should be treated: either it is legal, needs
1633 /// to be expanded to some other code sequence, or the target has a custom
1634 /// expander for it.
1637 assert((unsigned)CC < std::size(CondCodeActions) &&
1638 ((unsigned)VT.SimpleTy >> 3) < std::size(CondCodeActions[0]) &&
1639 "Table isn't big enough!");
1640 // See setCondCodeAction for how this is encoded.
1641 uint32_t Shift = 4 * (VT.SimpleTy & 0x7);
1642 uint32_t Value = CondCodeActions[CC][VT.SimpleTy >> 3];
1643 LegalizeAction Action = (LegalizeAction) ((Value >> Shift) & 0xF);
1644 assert(Action != Promote && "Can't promote condition code!");
1645 return Action;
1646 }
1647
1648 /// Return true if the specified condition code is legal for a comparison of
1649 /// the specified types on this target.
1650 bool isCondCodeLegal(ISD::CondCode CC, MVT VT) const {
1651 return getCondCodeAction(CC, VT) == Legal;
1652 }
1653
1654 /// Return true if the specified condition code is legal or custom for a
1655 /// comparison of the specified types on this target.
1657 return getCondCodeAction(CC, VT) == Legal ||
1658 getCondCodeAction(CC, VT) == Custom;
1659 }
1660
1661 /// Return how a PARTIAL_REDUCE_U/SMLA node with Acc type AccVT and Input type
1662 /// InputVT should be treated. Either it's legal, needs to be promoted to a
1663 /// larger size, needs to be expanded to some other code sequence, or the
1664 /// target has a custom expander for it.
1666 EVT InputVT) const {
1667 assert(Opc == ISD::PARTIAL_REDUCE_SMLA || Opc == ISD::PARTIAL_REDUCE_UMLA ||
1668 Opc == ISD::PARTIAL_REDUCE_SUMLA);
1669 PartialReduceActionTypes Key = {Opc, AccVT.getSimpleVT().SimpleTy,
1670 InputVT.getSimpleVT().SimpleTy};
1671 auto It = PartialReduceMLAActions.find(Key);
1672 return It != PartialReduceMLAActions.end() ? It->second : Expand;
1673 }
1674
1675 /// Return true if a PARTIAL_REDUCE_U/SMLA node with the specified types is
1676 /// legal or custom for this target.
1678 EVT InputVT) const {
1679 LegalizeAction Action = getPartialReduceMLAAction(Opc, AccVT, InputVT);
1680 return Action == Legal || Action == Custom;
1681 }
1682
1683 /// If the action for this operation is to promote, this method returns the
1684 /// ValueType to promote to.
1685 MVT getTypeToPromoteTo(unsigned Op, MVT VT) const {
1687 "This operation isn't promoted!");
1688
1689 // See if this has an explicit type specified.
1690 std::map<std::pair<unsigned, MVT::SimpleValueType>,
1692 PromoteToType.find(std::make_pair(Op, VT.SimpleTy));
1693 if (PTTI != PromoteToType.end()) return PTTI->second;
1694
1695 assert((VT.isInteger() || VT.isFloatingPoint()) &&
1696 "Cannot autopromote this type, add it with AddPromotedToType.");
1697
1698 uint64_t VTBits = VT.getScalarSizeInBits();
1699 MVT NVT = VT;
1700 do {
1701 NVT = (MVT::SimpleValueType)(NVT.SimpleTy+1);
1702 assert(NVT.isInteger() == VT.isInteger() &&
1703 NVT.isFloatingPoint() == VT.isFloatingPoint() &&
1704 "Didn't find type to promote to!");
1705 } while (VTBits >= NVT.getScalarSizeInBits() || !isTypeLegal(NVT) ||
1706 getOperationAction(Op, NVT) == Promote);
1707 return NVT;
1708 }
1709
1711 bool AllowUnknown = false) const {
1712 return getValueType(DL, Ty, AllowUnknown);
1713 }
1714
1715 /// Return the EVT corresponding to this LLVM type. This is fixed by the LLVM
1716 /// operations except for the pointer size. If AllowUnknown is true, this
1717 /// will return MVT::Other for types with no EVT counterpart (e.g. structs),
1718 /// otherwise it will assert.
1720 bool AllowUnknown = false) const {
1721 // Lower scalar pointers to native pointer types.
1722 if (auto *PTy = dyn_cast<PointerType>(Ty))
1723 return getPointerTy(DL, PTy->getAddressSpace());
1724
1725 if (auto *VTy = dyn_cast<VectorType>(Ty)) {
1726 Type *EltTy = VTy->getElementType();
1727 // Lower vectors of pointers to native pointer types.
1728 if (auto *PTy = dyn_cast<PointerType>(EltTy)) {
1729 EVT PointerTy(getPointerTy(DL, PTy->getAddressSpace()));
1730 EltTy = PointerTy.getTypeForEVT(Ty->getContext());
1731 }
1732 return EVT::getVectorVT(Ty->getContext(), EVT::getEVT(EltTy, false),
1733 VTy->getElementCount());
1734 }
1735
1736 return EVT::getEVT(Ty, AllowUnknown);
1737 }
1738
1740 bool AllowUnknown = false) const {
1741 // Lower scalar pointers to native pointer types.
1742 if (auto *PTy = dyn_cast<PointerType>(Ty))
1743 return getPointerMemTy(DL, PTy->getAddressSpace());
1744
1745 if (auto *VTy = dyn_cast<VectorType>(Ty)) {
1746 Type *EltTy = VTy->getElementType();
1747 if (auto *PTy = dyn_cast<PointerType>(EltTy)) {
1748 EVT PointerTy(getPointerMemTy(DL, PTy->getAddressSpace()));
1749 EltTy = PointerTy.getTypeForEVT(Ty->getContext());
1750 }
1751 return EVT::getVectorVT(Ty->getContext(), EVT::getEVT(EltTy, false),
1752 VTy->getElementCount());
1753 }
1754
1755 return getValueType(DL, Ty, AllowUnknown);
1756 }
1757
1758
1759 /// Return the MVT corresponding to this LLVM type. See getValueType.
1761 bool AllowUnknown = false) const {
1762 return getValueType(DL, Ty, AllowUnknown).getSimpleVT();
1763 }
1764
1765 /// Returns the desired alignment for ByVal or InAlloca aggregate function
1766 /// arguments in the caller parameter area.
1767 virtual Align getByValTypeAlignment(Type *Ty, const DataLayout &DL) const;
1768
1769 /// Return the type of registers that this ValueType will eventually require.
1771 assert((unsigned)VT.SimpleTy < std::size(RegisterTypeForVT));
1772 return RegisterTypeForVT[VT.SimpleTy];
1773 }
1774
1775 /// Return the type of registers that this ValueType will eventually require.
1776 MVT getRegisterType(LLVMContext &Context, EVT VT) const {
1777 if (VT.isSimple())
1778 return getRegisterType(VT.getSimpleVT());
1779 if (VT.isVector()) {
1780 EVT VT1;
1781 MVT RegisterVT;
1782 unsigned NumIntermediates;
1783 (void)getVectorTypeBreakdown(Context, VT, VT1,
1784 NumIntermediates, RegisterVT);
1785 return RegisterVT;
1786 }
1787 if (VT.isInteger()) {
1788 return getRegisterType(Context, getTypeToTransformTo(Context, VT));
1789 }
1790 llvm_unreachable("Unsupported extended type!");
1791 }
1792
1793 /// Return the number of registers that this ValueType will eventually
1794 /// require.
1795 ///
1796 /// This is one for any types promoted to live in larger registers, but may be
1797 /// more than one for types (like i64) that are split into pieces. For types
1798 /// like i140, which are first promoted then expanded, it is the number of
1799 /// registers needed to hold all the bits of the original type. For an i140
1800 /// on a 32 bit machine this means 5 registers.
1801 ///
1802 /// RegisterVT may be passed as a way to override the default settings, for
1803 /// instance with i128 inline assembly operands on SystemZ.
1804 virtual unsigned
1806 std::optional<MVT> RegisterVT = std::nullopt) const {
1807 if (VT.isSimple()) {
1808 assert((unsigned)VT.getSimpleVT().SimpleTy <
1809 std::size(NumRegistersForVT));
1810 return NumRegistersForVT[VT.getSimpleVT().SimpleTy];
1811 }
1812 if (VT.isVector()) {
1813 EVT VT1;
1814 MVT VT2;
1815 unsigned NumIntermediates;
1816 return getVectorTypeBreakdown(Context, VT, VT1, NumIntermediates, VT2);
1817 }
1818 if (VT.isInteger()) {
1819 unsigned BitWidth = VT.getSizeInBits();
1820 unsigned RegWidth = getRegisterType(Context, VT).getSizeInBits();
1821 return (BitWidth + RegWidth - 1) / RegWidth;
1822 }
1823 llvm_unreachable("Unsupported extended type!");
1824 }
1825
1826 /// Certain combinations of ABIs, Targets and features require that types
1827 /// are legal for some operations and not for other operations.
1828 /// For MIPS all vector types must be passed through the integer register set.
1830 CallingConv::ID CC, EVT VT) const {
1831 return getRegisterType(Context, VT);
1832 }
1833
1834 /// Certain targets require unusual breakdowns of certain types. For MIPS,
1835 /// this occurs when a vector type is used, as vector are passed through the
1836 /// integer register set.
1838 CallingConv::ID CC,
1839 EVT VT) const {
1840 return getNumRegisters(Context, VT);
1841 }
1842
1843 /// Certain targets have context sensitive alignment requirements, where one
1844 /// type has the alignment requirement of another type.
1846 const DataLayout &DL) const {
1847 return DL.getABITypeAlign(ArgTy);
1848 }
1849
1850 /// If true, then instruction selection should seek to shrink the FP constant
1851 /// of the specified type to a smaller type in order to save space and / or
1852 /// reduce runtime.
1853 virtual bool ShouldShrinkFPConstant(EVT) const { return true; }
1854
1855 /// Return true if it is profitable to reduce a load to a smaller type.
1856 /// \p ByteOffset is only set if we know the pointer offset at compile time
1857 /// otherwise we should assume that additional pointer math is required.
1858 /// Example: (i16 (trunc (i32 (load x))) -> i16 load x
1859 /// Example: (i16 (trunc (srl (i32 (load x)), 16)) -> i16 load x+2
1861 SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT,
1862 std::optional<unsigned> ByteOffset = std::nullopt) const {
1863 // By default, assume that it is cheaper to extract a subvector from a wide
1864 // vector load rather than creating multiple narrow vector loads.
1865 if (NewVT.isVector() && !SDValue(Load, 0).hasOneUse())
1866 return false;
1867
1868 return true;
1869 }
1870
1871 /// Return true (the default) if it is profitable to remove a sext_inreg(x)
1872 /// where the sext is redundant, and use x directly.
1873 virtual bool shouldRemoveRedundantExtend(SDValue Op) const { return true; }
1874
1875 /// Indicates if any padding is guaranteed to go at the most significant bits
1876 /// when storing the type to memory and the type size isn't equal to the store
1877 /// size.
1879 return VT.isScalarInteger() && !VT.isByteSized();
1880 }
1881
1882 /// When splitting a value of the specified type into parts, does the Lo
1883 /// or Hi part come first? This usually follows the endianness, except
1884 /// for ppcf128, where the Hi part always comes first.
1886 return DL.isBigEndian() || VT == MVT::ppcf128;
1887 }
1888
1889 /// If true, the target has custom DAG combine transformations that it can
1890 /// perform for the specified node.
1892 assert(unsigned(NT >> 3) < std::size(TargetDAGCombineArray));
1893 return TargetDAGCombineArray[NT >> 3] & (1 << (NT&7));
1894 }
1895
1898 }
1899
1900 /// Returns the size of the platform's va_list object.
1901 virtual unsigned getVaListSizeInBits(const DataLayout &DL) const {
1902 return getPointerTy(DL).getSizeInBits();
1903 }
1904
1905 /// Get maximum # of store operations permitted for llvm.memset
1906 ///
1907 /// This function returns the maximum number of store operations permitted
1908 /// to replace a call to llvm.memset. The value is set by the target at the
1909 /// performance threshold for such a replacement. If OptSize is true,
1910 /// return the limit for functions that have OptSize attribute.
1911 unsigned getMaxStoresPerMemset(bool OptSize) const {
1913 }
1914
1915 /// Get maximum # of store operations permitted for llvm.memcpy
1916 ///
1917 /// This function returns the maximum number of store operations permitted
1918 /// to replace a call to llvm.memcpy. The value is set by the target at the
1919 /// performance threshold for such a replacement. If OptSize is true,
1920 /// return the limit for functions that have OptSize attribute.
1921 unsigned getMaxStoresPerMemcpy(bool OptSize) const {
1923 }
1924
1925 /// \brief Get maximum # of store operations to be glued together
1926 ///
1927 /// This function returns the maximum number of store operations permitted
1928 /// to glue together during lowering of llvm.memcpy. The value is set by
1929 // the target at the performance threshold for such a replacement.
1930 virtual unsigned getMaxGluedStoresPerMemcpy() const {
1932 }
1933
1934 /// Get maximum # of load operations permitted for memcmp
1935 ///
1936 /// This function returns the maximum number of load operations permitted
1937 /// to replace a call to memcmp. The value is set by the target at the
1938 /// performance threshold for such a replacement. If OptSize is true,
1939 /// return the limit for functions that have OptSize attribute.
1940 unsigned getMaxExpandSizeMemcmp(bool OptSize) const {
1942 }
1943
1944 /// Get maximum # of store operations permitted for llvm.memmove
1945 ///
1946 /// This function returns the maximum number of store operations permitted
1947 /// to replace a call to llvm.memmove. The value is set by the target at the
1948 /// performance threshold for such a replacement. If OptSize is true,
1949 /// return the limit for functions that have OptSize attribute.
1950 unsigned getMaxStoresPerMemmove(bool OptSize) const {
1952 }
1953
1954 /// Determine if the target supports unaligned memory accesses.
1955 ///
1956 /// This function returns true if the target allows unaligned memory accesses
1957 /// of the specified type in the given address space. If true, it also returns
1958 /// a relative speed of the unaligned memory access in the last argument by
1959 /// reference. The higher the speed number the faster the operation comparing
1960 /// to a number returned by another such call. This is used, for example, in
1961 /// situations where an array copy/move/set is converted to a sequence of
1962 /// store operations. Its use helps to ensure that such replacements don't
1963 /// generate code that causes an alignment error (trap) on the target machine.
1965 EVT, unsigned AddrSpace = 0, Align Alignment = Align(1),
1967 unsigned * /*Fast*/ = nullptr) const {
1968 return false;
1969 }
1970
1971 /// LLT handling variant.
1973 LLT, unsigned AddrSpace = 0, Align Alignment = Align(1),
1975 unsigned * /*Fast*/ = nullptr) const {
1976 return false;
1977 }
1978
1979 /// This function returns true if the memory access is aligned or if the
1980 /// target allows this specific unaligned memory access. If the access is
1981 /// allowed, the optional final parameter returns a relative speed of the
1982 /// access (as defined by the target).
1983 bool allowsMemoryAccessForAlignment(
1984 LLVMContext &Context, const DataLayout &DL, EVT VT,
1985 unsigned AddrSpace = 0, Align Alignment = Align(1),
1987 unsigned *Fast = nullptr) const;
1988
1989 /// Return true if the memory access of this type is aligned or if the target
1990 /// allows this specific unaligned access for the given MachineMemOperand.
1991 /// If the access is allowed, the optional final parameter returns a relative
1992 /// speed of the access (as defined by the target).
1993 bool allowsMemoryAccessForAlignment(LLVMContext &Context,
1994 const DataLayout &DL, EVT VT,
1995 const MachineMemOperand &MMO,
1996 unsigned *Fast = nullptr) const;
1997
1998 /// Return true if the target supports a memory access of this type for the
1999 /// given address space and alignment. If the access is allowed, the optional
2000 /// final parameter returns the relative speed of the access (as defined by
2001 /// the target).
2002 virtual bool
2003 allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT,
2004 unsigned AddrSpace = 0, Align Alignment = Align(1),
2006 unsigned *Fast = nullptr) const;
2007
2008 /// Return true if the target supports a memory access of this type for the
2009 /// given MachineMemOperand. If the access is allowed, the optional
2010 /// final parameter returns the relative access speed (as defined by the
2011 /// target).
2012 bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT,
2013 const MachineMemOperand &MMO,
2014 unsigned *Fast = nullptr) const;
2015
2016 /// LLT handling variant.
2017 bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, LLT Ty,
2018 const MachineMemOperand &MMO,
2019 unsigned *Fast = nullptr) const;
2020
2021 /// Returns the target specific optimal type for load and store operations as
2022 /// a result of memset, memcpy, and memmove lowering.
2023 /// It returns EVT::Other if the type should be determined using generic
2024 /// target-independent logic.
2025 virtual EVT
2027 const AttributeList & /*FuncAttributes*/) const {
2028 return MVT::Other;
2029 }
2030
2031 /// LLT returning variant.
2032 virtual LLT
2034 const AttributeList & /*FuncAttributes*/) const {
2035 return LLT();
2036 }
2037
2038 /// Returns true if it's safe to use load / store of the specified type to
2039 /// expand memcpy / memset inline.
2040 ///
2041 /// This is mostly true for all types except for some special cases. For
2042 /// example, on X86 targets without SSE2 f64 load / store are done with fldl /
2043 /// fstpl which also does type conversion. Note the specified type doesn't
2044 /// have to be legal as the hook is used before type legalization.
2045 virtual bool isSafeMemOpType(MVT /*VT*/) const { return true; }
2046
2047 /// Return lower limit for number of blocks in a jump table.
2048 virtual unsigned getMinimumJumpTableEntries() const;
2049
2050 /// Return lower limit of the density in a jump table.
2051 unsigned getMinimumJumpTableDensity(bool OptForSize) const;
2052
2053 /// Return upper limit for number of entries in a jump table.
2054 /// Zero if no limit.
2055 unsigned getMaximumJumpTableSize() const;
2056
2057 virtual bool isJumpTableRelative() const;
2058
2059 /// If a physical register, this specifies the register that
2060 /// llvm.savestack/llvm.restorestack should save and restore.
2062 return StackPointerRegisterToSaveRestore;
2063 }
2064
2065 /// If a physical register, this returns the register that receives the
2066 /// exception address on entry to an EH pad.
2067 virtual Register
2068 getExceptionPointerRegister(const Constant *PersonalityFn) const {
2069 return Register();
2070 }
2071
2072 /// If a physical register, this returns the register that receives the
2073 /// exception typeid on entry to a landing pad.
2074 virtual Register
2075 getExceptionSelectorRegister(const Constant *PersonalityFn) const {
2076 return Register();
2077 }
2078
2079 virtual bool needsFixedCatchObjects() const {
2080 report_fatal_error("Funclet EH is not implemented for this target");
2081 }
2082
2083 /// Return the minimum stack alignment of an argument.
2085 return MinStackArgumentAlignment;
2086 }
2087
2088 /// Return the minimum function alignment.
2089 Align getMinFunctionAlignment() const { return MinFunctionAlignment; }
2090
2091 /// Return the preferred function alignment.
2092 Align getPrefFunctionAlignment() const { return PrefFunctionAlignment; }
2093
2094 /// Return the preferred loop alignment.
2095 virtual Align getPrefLoopAlignment(MachineLoop *ML = nullptr) const;
2096
2097 /// Return the maximum amount of bytes allowed to be emitted when padding for
2098 /// alignment
2099 virtual unsigned
2100 getMaxPermittedBytesForAlignment(MachineBasicBlock *MBB) const;
2101
2102 /// Should loops be aligned even when the function is marked OptSize (but not
2103 /// MinSize).
2104 virtual bool alignLoopsWithOptSize() const { return false; }
2105
2106 /// If the target has a standard location for the stack protector guard,
2107 /// returns the address of that location. Otherwise, returns nullptr.
2108 /// DEPRECATED: please override useLoadStackGuardNode and customize
2109 /// LOAD_STACK_GUARD, or customize \@llvm.stackguard().
2110 virtual Value *getIRStackGuard(IRBuilderBase &IRB) const;
2111
2112 /// Inserts necessary declarations for SSP (stack protection) purpose.
2113 /// Should be used only when getIRStackGuard returns nullptr.
2114 virtual void insertSSPDeclarations(Module &M) const;
2115
2116 /// Return the variable that's previously inserted by insertSSPDeclarations,
2117 /// if any, otherwise return nullptr. Should be used only when
2118 /// getIRStackGuard returns nullptr.
2119 virtual Value *getSDagStackGuard(const Module &M) const;
2120
2121 /// If this function returns true, stack protection checks should XOR the
2122 /// frame pointer (or whichever pointer is used to address locals) into the
2123 /// stack guard value before checking it. getIRStackGuard must return nullptr
2124 /// if this returns true.
2125 virtual bool useStackGuardXorFP() const { return false; }
2126
2127 /// If the target has a standard stack protection check function that
2128 /// performs validation and error handling, returns the function. Otherwise,
2129 /// returns nullptr. Must be previously inserted by insertSSPDeclarations.
2130 /// Should be used only when getIRStackGuard returns nullptr.
2131 virtual Function *getSSPStackGuardCheck(const Module &M) const;
2132
2133protected:
2134 Value *getDefaultSafeStackPointerLocation(IRBuilderBase &IRB,
2135 bool UseTLS) const;
2136
2137public:
2138 /// Returns the target-specific address of the unsafe stack pointer.
2139 virtual Value *getSafeStackPointerLocation(IRBuilderBase &IRB) const;
2140
2141 /// Returns the name of the symbol used to emit stack probes or the empty
2142 /// string if not applicable.
2143 virtual bool hasStackProbeSymbol(const MachineFunction &MF) const { return false; }
2144
2145 virtual bool hasInlineStackProbe(const MachineFunction &MF) const { return false; }
2146
2148 return "";
2149 }
2150
2151 /// Returns true if a cast from SrcAS to DestAS is "cheap", such that e.g. we
2152 /// are happy to sink it into basic blocks. A cast may be free, but not
2153 /// necessarily a no-op. e.g. a free truncate from a 64-bit to 32-bit pointer.
2154 virtual bool isFreeAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const;
2155
2156 /// Return true if the pointer arguments to CI should be aligned by aligning
2157 /// the object whose address is being passed. If so then MinSize is set to the
2158 /// minimum size the object must be to be aligned and PrefAlign is set to the
2159 /// preferred alignment.
2160 virtual bool shouldAlignPointerArgs(CallInst * /*CI*/, unsigned & /*MinSize*/,
2161 Align & /*PrefAlign*/) const {
2162 return false;
2163 }
2164
2165 //===--------------------------------------------------------------------===//
2166 /// \name Helpers for TargetTransformInfo implementations
2167 /// @{
2168
2169 /// Get the ISD node that corresponds to the Instruction class opcode.
2170 int InstructionOpcodeToISD(unsigned Opcode) const;
2171
2172 /// Get the ISD node that corresponds to the Intrinsic ID. Returns
2173 /// ISD::DELETED_NODE by default for an unsupported Intrinsic ID.
2174 int IntrinsicIDToISD(Intrinsic::ID ID) const;
2175
2176 /// @}
2177
2178 //===--------------------------------------------------------------------===//
2179 /// \name Helpers for atomic expansion.
2180 /// @{
2181
2182 /// Returns the maximum atomic operation size (in bits) supported by
2183 /// the backend. Atomic operations greater than this size (as well
2184 /// as ones that are not naturally aligned), will be expanded by
2185 /// AtomicExpandPass into an __atomic_* library call.
2187 return MaxAtomicSizeInBitsSupported;
2188 }
2189
2190 /// Returns the size in bits of the maximum div/rem the backend supports.
2191 /// Larger operations will be expanded by ExpandLargeDivRem.
2193 return MaxDivRemBitWidthSupported;
2194 }
2195
2196 /// Returns the size in bits of the maximum fp to/from int conversion the
2197 /// backend supports. Larger operations will be expanded by ExpandFp.
2199 return MaxLargeFPConvertBitWidthSupported;
2200 }
2201
2202 /// Returns the size of the smallest cmpxchg or ll/sc instruction
2203 /// the backend supports. Any smaller operations are widened in
2204 /// AtomicExpandPass.
2205 ///
2206 /// Note that *unlike* operations above the maximum size, atomic ops
2207 /// are still natively supported below the minimum; they just
2208 /// require a more complex expansion.
2209 unsigned getMinCmpXchgSizeInBits() const { return MinCmpXchgSizeInBits; }
2210
2211 /// Whether the target supports unaligned atomic operations.
2212 bool supportsUnalignedAtomics() const { return SupportsUnalignedAtomics; }
2213
2214 /// Whether AtomicExpandPass should automatically insert fences and reduce
2215 /// ordering for this atomic. This should be true for most architectures with
2216 /// weak memory ordering. Defaults to false.
2217 virtual bool shouldInsertFencesForAtomic(const Instruction *I) const {
2218 return false;
2219 }
2220
2221 // The memory ordering that AtomicExpandPass should assign to a atomic
2222 // instruction that it has lowered by adding fences. This can be used
2223 // to "fold" one of the fences into the atomic instruction.
2224 virtual AtomicOrdering
2228
2229 /// Whether AtomicExpandPass should automatically insert a trailing fence
2230 /// without reducing the ordering for this atomic. Defaults to false.
2231 virtual bool
2233 return false;
2234 }
2235
2236 /// Perform a load-linked operation on Addr, returning a "Value *" with the
2237 /// corresponding pointee type. This may entail some non-trivial operations to
2238 /// truncate or reconstruct types that will be illegal in the backend. See
2239 /// ARMISelLowering for an example implementation.
2240 virtual Value *emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy,
2241 Value *Addr, AtomicOrdering Ord) const {
2242 llvm_unreachable("Load linked unimplemented on this target");
2243 }
2244
2245 /// Perform a store-conditional operation to Addr. Return the status of the
2246 /// store. This should be 0 if the store succeeded, non-zero otherwise.
2248 Value *Addr, AtomicOrdering Ord) const {
2249 llvm_unreachable("Store conditional unimplemented on this target");
2250 }
2251
2252 /// Perform a masked atomicrmw using a target-specific intrinsic. This
2253 /// represents the core LL/SC loop which will be lowered at a late stage by
2254 /// the backend. The target-specific intrinsic returns the loaded value and
2255 /// is not responsible for masking and shifting the result.
2257 AtomicRMWInst *AI,
2258 Value *AlignedAddr, Value *Incr,
2259 Value *Mask, Value *ShiftAmt,
2260 AtomicOrdering Ord) const {
2261 llvm_unreachable("Masked atomicrmw expansion unimplemented on this target");
2262 }
2263
2264 /// Perform a atomicrmw expansion using a target-specific way. This is
2265 /// expected to be called when masked atomicrmw and bit test atomicrmw don't
2266 /// work, and the target supports another way to lower atomicrmw.
2267 virtual void emitExpandAtomicRMW(AtomicRMWInst *AI) const {
2269 "Generic atomicrmw expansion unimplemented on this target");
2270 }
2271
2272 /// Perform a atomic store using a target-specific way.
2273 virtual void emitExpandAtomicStore(StoreInst *SI) const {
2275 "Generic atomic store expansion unimplemented on this target");
2276 }
2277
2278 /// Perform a atomic load using a target-specific way.
2279 virtual void emitExpandAtomicLoad(LoadInst *LI) const {
2281 "Generic atomic load expansion unimplemented on this target");
2282 }
2283
2284 /// Perform a cmpxchg expansion using a target-specific method.
2286 llvm_unreachable("Generic cmpxchg expansion unimplemented on this target");
2287 }
2288
2289 /// Perform a bit test atomicrmw using a target-specific intrinsic. This
2290 /// represents the combined bit test intrinsic which will be lowered at a late
2291 /// stage by the backend.
2294 "Bit test atomicrmw expansion unimplemented on this target");
2295 }
2296
2297 /// Perform a atomicrmw which the result is only used by comparison, using a
2298 /// target-specific intrinsic. This represents the combined atomic and compare
2299 /// intrinsic which will be lowered at a late stage by the backend.
2302 "Compare arith atomicrmw expansion unimplemented on this target");
2303 }
2304
2305 /// Perform a masked cmpxchg using a target-specific intrinsic. This
2306 /// represents the core LL/SC loop which will be lowered at a late stage by
2307 /// the backend. The target-specific intrinsic returns the loaded value and
2308 /// is not responsible for masking and shifting the result.
2310 IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
2311 Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
2312 llvm_unreachable("Masked cmpxchg expansion unimplemented on this target");
2313 }
2314
2315 //===--------------------------------------------------------------------===//
2316 /// \name KCFI check lowering.
2317 /// @{
2318
2321 const TargetInstrInfo *TII) const {
2322 llvm_unreachable("KCFI is not supported on this target");
2323 }
2324
2325 /// @}
2326
2327 /// Inserts in the IR a target-specific intrinsic specifying a fence.
2328 /// It is called by AtomicExpandPass before expanding an
2329 /// AtomicRMW/AtomicCmpXchg/AtomicStore/AtomicLoad
2330 /// if shouldInsertFencesForAtomic returns true.
2331 ///
2332 /// Inst is the original atomic instruction, prior to other expansions that
2333 /// may be performed.
2334 ///
2335 /// This function should either return a nullptr, or a pointer to an IR-level
2336 /// Instruction*. Even complex fence sequences can be represented by a
2337 /// single Instruction* through an intrinsic to be lowered later.
2338 ///
2339 /// The default implementation emits an IR fence before any release (or
2340 /// stronger) operation that stores, and after any acquire (or stronger)
2341 /// operation. This is generally a correct implementation, but backends may
2342 /// override if they wish to use alternative schemes (e.g. the PowerPC
2343 /// standard ABI uses a fence before a seq_cst load instead of after a
2344 /// seq_cst store).
2345 /// @{
2346 virtual Instruction *emitLeadingFence(IRBuilderBase &Builder,
2347 Instruction *Inst,
2348 AtomicOrdering Ord) const;
2349
2350 virtual Instruction *emitTrailingFence(IRBuilderBase &Builder,
2351 Instruction *Inst,
2352 AtomicOrdering Ord) const;
2353 /// @}
2354
2355 // Emits code that executes when the comparison result in the ll/sc
2356 // expansion of a cmpxchg instruction is such that the store-conditional will
2357 // not execute. This makes it possible to balance out the load-linked with
2358 // a dedicated instruction, if desired.
2359 // E.g., on ARM, if ldrex isn't followed by strex, the exclusive monitor would
2360 // be unnecessarily held, except if clrex, inserted by this hook, is executed.
2361 virtual void emitAtomicCmpXchgNoStoreLLBalance(IRBuilderBase &Builder) const {}
2362
2363 /// Returns true if arguments should be sign-extended in lib calls.
2364 virtual bool shouldSignExtendTypeInLibCall(Type *Ty, bool IsSigned) const {
2365 return IsSigned;
2366 }
2367
2368 /// Returns true if arguments should be extended in lib calls.
2369 virtual bool shouldExtendTypeInLibCall(EVT Type) const {
2370 return true;
2371 }
2372
2373 /// Returns how the given (atomic) load should be expanded by the
2374 /// IR-level AtomicExpand pass.
2378
2379 /// Returns how the given (atomic) load should be cast by the IR-level
2380 /// AtomicExpand pass.
2386
2387 /// Returns how the given (atomic) store should be expanded by the IR-level
2388 /// AtomicExpand pass into. For instance AtomicExpansionKind::CustomExpand
2389 /// will try to use an atomicrmw xchg.
2393
2394 /// Returns how the given (atomic) store should be cast by the IR-level
2395 /// AtomicExpand pass into. For instance AtomicExpansionKind::CastToInteger
2396 /// will try to cast the operands to integer values.
2398 if (SI->getValueOperand()->getType()->isFloatingPointTy())
2401 }
2402
2403 /// Returns how the given atomic cmpxchg should be expanded by the IR-level
2404 /// AtomicExpand pass.
2405 virtual AtomicExpansionKind
2409
2410 /// Returns how the IR-level AtomicExpand pass should expand the given
2411 /// AtomicRMW, if at all. Default is to never expand.
2416
2417 /// Returns how the given atomic atomicrmw should be cast by the IR-level
2418 /// AtomicExpand pass.
2419 virtual AtomicExpansionKind
2428
2429 /// On some platforms, an AtomicRMW that never actually modifies the value
2430 /// (such as fetch_add of 0) can be turned into a fence followed by an
2431 /// atomic load. This may sound useless, but it makes it possible for the
2432 /// processor to keep the cacheline shared, dramatically improving
2433 /// performance. And such idempotent RMWs are useful for implementing some
2434 /// kinds of locks, see for example (justification + benchmarks):
2435 /// http://www.hpl.hp.com/techreports/2012/HPL-2012-68.pdf
2436 /// This method tries doing that transformation, returning the atomic load if
2437 /// it succeeds, and nullptr otherwise.
2438 /// If shouldExpandAtomicLoadInIR returns true on that load, it will undergo
2439 /// another round of expansion.
2440 virtual LoadInst *
2442 return nullptr;
2443 }
2444
2445 /// Returns how the platform's atomic operations are extended (ZERO_EXTEND,
2446 /// SIGN_EXTEND, or ANY_EXTEND).
2448 return ISD::ZERO_EXTEND;
2449 }
2450
2451 /// Returns how the platform's atomic compare and swap expects its comparison
2452 /// value to be extended (ZERO_EXTEND, SIGN_EXTEND, or ANY_EXTEND). This is
2453 /// separate from getExtendForAtomicOps, which is concerned with the
2454 /// sign-extension of the instruction's output, whereas here we are concerned
2455 /// with the sign-extension of the input. For targets with compare-and-swap
2456 /// instructions (or sub-word comparisons in their LL/SC loop expansions),
2457 /// the input can be ANY_EXTEND, but the output will still have a specific
2458 /// extension.
2460 return ISD::ANY_EXTEND;
2461 }
2462
2463 /// @}
2464
2465 /// Returns true if we should normalize
2466 /// select(N0&N1, X, Y) => select(N0, select(N1, X, Y), Y) and
2467 /// select(N0|N1, X, Y) => select(N0, select(N1, X, Y, Y)) if it is likely
2468 /// that it saves us from materializing N0 and N1 in an integer register.
2469 /// Targets that are able to perform and/or on flags should return false here.
2471 EVT VT) const {
2472 // If a target has multiple condition registers, then it likely has logical
2473 // operations on those registers.
2475 return false;
2476 // Only do the transform if the value won't be split into multiple
2477 // registers.
2478 LegalizeTypeAction Action = getTypeAction(Context, VT);
2479 return Action != TypeExpandInteger && Action != TypeExpandFloat &&
2480 Action != TypeSplitVector;
2481 }
2482
2483 virtual bool isProfitableToCombineMinNumMaxNum(EVT VT) const { return true; }
2484
2485 /// Return true if a select of constants (select Cond, C1, C2) should be
2486 /// transformed into simple math ops with the condition value. For example:
2487 /// select Cond, C1, C1-1 --> add (zext Cond), C1-1
2488 virtual bool convertSelectOfConstantsToMath(EVT VT) const {
2489 return false;
2490 }
2491
2492 /// Return true if it is profitable to transform an integer
2493 /// multiplication-by-constant into simpler operations like shifts and adds.
2494 /// This may be true if the target does not directly support the
2495 /// multiplication operation for the specified type or the sequence of simpler
2496 /// ops is faster than the multiply.
2498 EVT VT, SDValue C) const {
2499 return false;
2500 }
2501
2502 /// Return true if it may be profitable to transform
2503 /// (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2).
2504 /// This may not be true if c1 and c2 can be represented as immediates but
2505 /// c1*c2 cannot, for example.
2506 /// The target should check if c1, c2 and c1*c2 can be represented as
2507 /// immediates, or have to be materialized into registers. If it is not sure
2508 /// about some cases, a default true can be returned to let the DAGCombiner
2509 /// decide.
2510 /// AddNode is (add x, c1), and ConstNode is c2.
2512 SDValue ConstNode) const {
2513 return true;
2514 }
2515
2516 /// Return true if it is more correct/profitable to use strict FP_TO_INT
2517 /// conversion operations - canonicalizing the FP source value instead of
2518 /// converting all cases and then selecting based on value.
2519 /// This may be true if the target throws exceptions for out of bounds
2520 /// conversions or has fast FP CMOV.
2521 virtual bool shouldUseStrictFP_TO_INT(EVT FpVT, EVT IntVT,
2522 bool IsSigned) const {
2523 return false;
2524 }
2525
2526 /// Return true if it is beneficial to expand an @llvm.powi.* intrinsic.
2527 /// If not optimizing for size, expanding @llvm.powi.* intrinsics is always
2528 /// considered beneficial.
2529 /// If optimizing for size, expansion is only considered beneficial for upto
2530 /// 5 multiplies and a divide (if the exponent is negative).
2531 bool isBeneficialToExpandPowI(int64_t Exponent, bool OptForSize) const {
2532 if (Exponent < 0)
2533 Exponent = -Exponent;
2534 uint64_t E = static_cast<uint64_t>(Exponent);
2535 return !OptForSize || (llvm::popcount(E) + Log2_64(E) < 7);
2536 }
2537
2538 //===--------------------------------------------------------------------===//
2539 // TargetLowering Configuration Methods - These methods should be invoked by
2540 // the derived class constructor to configure this object for the target.
2541 //
2542protected:
2543 /// Specify how the target extends the result of integer and floating point
2544 /// boolean values from i1 to a wider type. See getBooleanContents.
2546 BooleanContents = Ty;
2547 BooleanFloatContents = Ty;
2548 }
2549
2550 /// Specify how the target extends the result of integer and floating point
2551 /// boolean values from i1 to a wider type. See getBooleanContents.
2553 BooleanContents = IntTy;
2554 BooleanFloatContents = FloatTy;
2555 }
2556
2557 /// Specify how the target extends the result of a vector boolean value from a
2558 /// vector of i1 to a wider type. See getBooleanContents.
2560 BooleanVectorContents = Ty;
2561 }
2562
2563 /// Specify the target scheduling preference.
2565 SchedPreferenceInfo = Pref;
2566 }
2567
2568 /// Indicate the minimum number of blocks to generate jump tables.
2569 void setMinimumJumpTableEntries(unsigned Val);
2570
2571 /// Indicate the maximum number of entries in jump tables.
2572 /// Set to zero to generate unlimited jump tables.
2573 void setMaximumJumpTableSize(unsigned);
2574
2575 /// If set to a physical register, this specifies the register that
2576 /// llvm.savestack/llvm.restorestack should save and restore.
2578 StackPointerRegisterToSaveRestore = R;
2579 }
2580
2581 /// Tells the code generator that the target has BitExtract instructions.
2582 /// The code generator will aggressively sink "shift"s into the blocks of
2583 /// their users if the users will generate "and" instructions which can be
2584 /// combined with "shift" to BitExtract instructions.
2585 void setHasExtractBitsInsn(bool hasExtractInsn = true) {
2586 HasExtractBitsInsn = hasExtractInsn;
2587 }
2588
2589 /// Tells the code generator not to expand logic operations on comparison
2590 /// predicates into separate sequences that increase the amount of flow
2591 /// control.
2592 void setJumpIsExpensive(bool isExpensive = true);
2593
2594 /// Tells the code generator which bitwidths to bypass.
2595 void addBypassSlowDiv(unsigned int SlowBitWidth, unsigned int FastBitWidth) {
2596 BypassSlowDivWidths[SlowBitWidth] = FastBitWidth;
2597 }
2598
2599 /// Add the specified register class as an available regclass for the
2600 /// specified value type. This indicates the selector can handle values of
2601 /// that class natively.
2603 assert((unsigned)VT.SimpleTy < std::size(RegClassForVT));
2604 RegClassForVT[VT.SimpleTy] = RC;
2605 }
2606
2607 /// Return the largest legal super-reg register class of the register class
2608 /// for the specified type and its associated "cost".
2609 virtual std::pair<const TargetRegisterClass *, uint8_t>
2610 findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const;
2611
2612 /// Once all of the register classes are added, this allows us to compute
2613 /// derived properties we expose.
2614 void computeRegisterProperties(const TargetRegisterInfo *TRI);
2615
2616 /// Indicate that the specified operation does not work with the specified
2617 /// type and indicate what to do about it. Note that VT may refer to either
2618 /// the type of a result or that of an operand of Op.
2619 void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action) {
2620 assert(Op < std::size(OpActions[0]) && "Table isn't big enough!");
2621 OpActions[(unsigned)VT.SimpleTy][Op] = Action;
2622 }
2624 LegalizeAction Action) {
2625 for (auto Op : Ops)
2626 setOperationAction(Op, VT, Action);
2627 }
2629 LegalizeAction Action) {
2630 for (auto VT : VTs)
2631 setOperationAction(Ops, VT, Action);
2632 }
2633
2634 /// Indicate that the specified load with extension does not work with the
2635 /// specified type and indicate what to do about it.
2636 void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT,
2637 LegalizeAction Action) {
2638 assert(ExtType < ISD::LAST_LOADEXT_TYPE && ValVT.isValid() &&
2639 MemVT.isValid() && "Table isn't big enough!");
2640 assert((unsigned)Action < 0x10 && "too many bits for bitfield array");
2641 unsigned Shift = 4 * ExtType;
2642 LoadExtActions[ValVT.SimpleTy][MemVT.SimpleTy] &= ~((uint16_t)0xF << Shift);
2643 LoadExtActions[ValVT.SimpleTy][MemVT.SimpleTy] |= (uint16_t)Action << Shift;
2644 }
2645 void setLoadExtAction(ArrayRef<unsigned> ExtTypes, MVT ValVT, MVT MemVT,
2646 LegalizeAction Action) {
2647 for (auto ExtType : ExtTypes)
2648 setLoadExtAction(ExtType, ValVT, MemVT, Action);
2649 }
2651 ArrayRef<MVT> MemVTs, LegalizeAction Action) {
2652 for (auto MemVT : MemVTs)
2653 setLoadExtAction(ExtTypes, ValVT, MemVT, Action);
2654 }
2655
2656 /// Let target indicate that an extending atomic load of the specified type
2657 /// is legal.
2658 void setAtomicLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT,
2659 LegalizeAction Action) {
2660 assert(ExtType < ISD::LAST_LOADEXT_TYPE && ValVT.isValid() &&
2661 MemVT.isValid() && "Table isn't big enough!");
2662 assert((unsigned)Action < 0x10 && "too many bits for bitfield array");
2663 unsigned Shift = 4 * ExtType;
2664 AtomicLoadExtActions[ValVT.SimpleTy][MemVT.SimpleTy] &=
2665 ~((uint16_t)0xF << Shift);
2666 AtomicLoadExtActions[ValVT.SimpleTy][MemVT.SimpleTy] |=
2667 ((uint16_t)Action << Shift);
2668 }
2670 LegalizeAction Action) {
2671 for (auto ExtType : ExtTypes)
2672 setAtomicLoadExtAction(ExtType, ValVT, MemVT, Action);
2673 }
2675 ArrayRef<MVT> MemVTs, LegalizeAction Action) {
2676 for (auto MemVT : MemVTs)
2677 setAtomicLoadExtAction(ExtTypes, ValVT, MemVT, Action);
2678 }
2679
2680 /// Indicate that the specified truncating store does not work with the
2681 /// specified type and indicate what to do about it.
2682 void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action) {
2683 assert(ValVT.isValid() && MemVT.isValid() && "Table isn't big enough!");
2684 TruncStoreActions[(unsigned)ValVT.SimpleTy][MemVT.SimpleTy] = Action;
2685 }
2686
2687 /// Indicate that the specified indexed load does or does not work with the
2688 /// specified type and indicate what to do abort it.
2689 ///
2690 /// NOTE: All indexed mode loads are initialized to Expand in
2691 /// TargetLowering.cpp
2693 LegalizeAction Action) {
2694 for (auto IdxMode : IdxModes)
2695 setIndexedModeAction(IdxMode, VT, IMAB_Load, Action);
2696 }
2697
2699 LegalizeAction Action) {
2700 for (auto VT : VTs)
2701 setIndexedLoadAction(IdxModes, VT, Action);
2702 }
2703
2704 /// Indicate that the specified indexed store does or does not work with the
2705 /// specified type and indicate what to do about it.
2706 ///
2707 /// NOTE: All indexed mode stores are initialized to Expand in
2708 /// TargetLowering.cpp
2710 LegalizeAction Action) {
2711 for (auto IdxMode : IdxModes)
2712 setIndexedModeAction(IdxMode, VT, IMAB_Store, Action);
2713 }
2714
2716 LegalizeAction Action) {
2717 for (auto VT : VTs)
2718 setIndexedStoreAction(IdxModes, VT, Action);
2719 }
2720
2721 /// Indicate that the specified indexed masked load does or does not work with
2722 /// the specified type and indicate what to do about it.
2723 ///
2724 /// NOTE: All indexed mode masked loads are initialized to Expand in
2725 /// TargetLowering.cpp
2726 void setIndexedMaskedLoadAction(unsigned IdxMode, MVT VT,
2727 LegalizeAction Action) {
2728 setIndexedModeAction(IdxMode, VT, IMAB_MaskedLoad, Action);
2729 }
2730
2731 /// Indicate that the specified indexed masked store does or does not work
2732 /// with the specified type and indicate what to do about it.
2733 ///
2734 /// NOTE: All indexed mode masked stores are initialized to Expand in
2735 /// TargetLowering.cpp
2736 void setIndexedMaskedStoreAction(unsigned IdxMode, MVT VT,
2737 LegalizeAction Action) {
2738 setIndexedModeAction(IdxMode, VT, IMAB_MaskedStore, Action);
2739 }
2740
2741 /// Indicate that the specified condition code is or isn't supported on the
2742 /// target and indicate what to do about it.
2744 LegalizeAction Action) {
2745 for (auto CC : CCs) {
2746 assert(VT.isValid() && (unsigned)CC < std::size(CondCodeActions) &&
2747 "Table isn't big enough!");
2748 assert((unsigned)Action < 0x10 && "too many bits for bitfield array");
2749 /// The lower 3 bits of the SimpleTy index into Nth 4bit set from the
2750 /// 32-bit value and the upper 29 bits index into the second dimension of
2751 /// the array to select what 32-bit value to use.
2752 uint32_t Shift = 4 * (VT.SimpleTy & 0x7);
2753 CondCodeActions[CC][VT.SimpleTy >> 3] &= ~((uint32_t)0xF << Shift);
2754 CondCodeActions[CC][VT.SimpleTy >> 3] |= (uint32_t)Action << Shift;
2755 }
2756 }
2758 LegalizeAction Action) {
2759 for (auto VT : VTs)
2760 setCondCodeAction(CCs, VT, Action);
2761 }
2762
2763 /// Indicate how a PARTIAL_REDUCE_U/SMLA node with Acc type AccVT and Input
2764 /// type InputVT should be treated by the target. Either it's legal, needs to
2765 /// be promoted to a larger size, needs to be expanded to some other code
2766 /// sequence, or the target has a custom expander for it.
2767 void setPartialReduceMLAAction(unsigned Opc, MVT AccVT, MVT InputVT,
2768 LegalizeAction Action) {
2769 assert(Opc == ISD::PARTIAL_REDUCE_SMLA || Opc == ISD::PARTIAL_REDUCE_UMLA ||
2770 Opc == ISD::PARTIAL_REDUCE_SUMLA);
2771 assert(AccVT.isValid() && InputVT.isValid() &&
2772 "setPartialReduceMLAAction types aren't valid");
2773 PartialReduceActionTypes Key = {Opc, AccVT.SimpleTy, InputVT.SimpleTy};
2774 PartialReduceMLAActions[Key] = Action;
2775 }
2777 MVT InputVT, LegalizeAction Action) {
2778 for (unsigned Opc : Opcodes)
2779 setPartialReduceMLAAction(Opc, AccVT, InputVT, Action);
2780 }
2781
2782 /// If Opc/OrigVT is specified as being promoted, the promotion code defaults
2783 /// to trying a larger integer/fp until it can find one that works. If that
2784 /// default is insufficient, this method can be used by the target to override
2785 /// the default.
2786 void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT) {
2787 PromoteToType[std::make_pair(Opc, OrigVT.SimpleTy)] = DestVT.SimpleTy;
2788 }
2789
2790 /// Convenience method to set an operation to Promote and specify the type
2791 /// in a single call.
2792 void setOperationPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT) {
2793 setOperationAction(Opc, OrigVT, Promote);
2794 AddPromotedToType(Opc, OrigVT, DestVT);
2795 }
2797 MVT DestVT) {
2798 for (auto Op : Ops) {
2799 setOperationAction(Op, OrigVT, Promote);
2800 AddPromotedToType(Op, OrigVT, DestVT);
2801 }
2802 }
2803
2804 /// Targets should invoke this method for each target independent node that
2805 /// they want to provide a custom DAG combiner for by implementing the
2806 /// PerformDAGCombine virtual method.
2808 for (auto NT : NTs) {
2809 assert(unsigned(NT >> 3) < std::size(TargetDAGCombineArray));
2810 TargetDAGCombineArray[NT >> 3] |= 1 << (NT & 7);
2811 }
2812 }
2813
2814 /// Set the target's minimum function alignment.
2816 MinFunctionAlignment = Alignment;
2817 }
2818
2819 /// Set the target's preferred function alignment. This should be set if
2820 /// there is a performance benefit to higher-than-minimum alignment
2822 PrefFunctionAlignment = Alignment;
2823 }
2824
2825 /// Set the target's preferred loop alignment. Default alignment is one, it
2826 /// means the target does not care about loop alignment. The target may also
2827 /// override getPrefLoopAlignment to provide per-loop values.
2828 void setPrefLoopAlignment(Align Alignment) { PrefLoopAlignment = Alignment; }
2829 void setMaxBytesForAlignment(unsigned MaxBytes) {
2830 MaxBytesForAlignment = MaxBytes;
2831 }
2832
2833 /// Set the minimum stack alignment of an argument.
2835 MinStackArgumentAlignment = Alignment;
2836 }
2837
2838 /// Set the maximum atomic operation size supported by the
2839 /// backend. Atomic operations greater than this size (as well as
2840 /// ones that are not naturally aligned), will be expanded by
2841 /// AtomicExpandPass into an __atomic_* library call.
2842 void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits) {
2843 MaxAtomicSizeInBitsSupported = SizeInBits;
2844 }
2845
2846 /// Set the size in bits of the maximum div/rem the backend supports.
2847 /// Larger operations will be expanded by ExpandLargeDivRem.
2848 void setMaxDivRemBitWidthSupported(unsigned SizeInBits) {
2849 MaxDivRemBitWidthSupported = SizeInBits;
2850 }
2851
2852 /// Set the size in bits of the maximum fp to/from int conversion the backend
2853 /// supports. Larger operations will be expanded by ExpandFp.
2854 void setMaxLargeFPConvertBitWidthSupported(unsigned SizeInBits) {
2855 MaxLargeFPConvertBitWidthSupported = SizeInBits;
2856 }
2857
2858 /// Sets the minimum cmpxchg or ll/sc size supported by the backend.
2859 void setMinCmpXchgSizeInBits(unsigned SizeInBits) {
2860 MinCmpXchgSizeInBits = SizeInBits;
2861 }
2862
2863 /// Sets whether unaligned atomic operations are supported.
2864 void setSupportsUnalignedAtomics(bool UnalignedSupported) {
2865 SupportsUnalignedAtomics = UnalignedSupported;
2866 }
2867
2868public:
2869 //===--------------------------------------------------------------------===//
2870 // Addressing mode description hooks (used by LSR etc).
2871 //
2872
2873 /// CodeGenPrepare sinks address calculations into the same BB as Load/Store
2874 /// instructions reading the address. This allows as much computation as
2875 /// possible to be done in the address mode for that operand. This hook lets
2876 /// targets also pass back when this should be done on intrinsics which
2877 /// load/store.
2878 virtual bool getAddrModeArguments(const IntrinsicInst * /*I*/,
2879 SmallVectorImpl<Value *> & /*Ops*/,
2880 Type *& /*AccessTy*/) const {
2881 return false;
2882 }
2883
2884 /// This represents an addressing mode of:
2885 /// BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*vscale
2886 /// If BaseGV is null, there is no BaseGV.
2887 /// If BaseOffs is zero, there is no base offset.
2888 /// If HasBaseReg is false, there is no base register.
2889 /// If Scale is zero, there is no ScaleReg. Scale of 1 indicates a reg with
2890 /// no scale.
2891 /// If ScalableOffset is zero, there is no scalable offset.
2892 struct AddrMode {
2894 int64_t BaseOffs = 0;
2895 bool HasBaseReg = false;
2896 int64_t Scale = 0;
2897 int64_t ScalableOffset = 0;
2898 AddrMode() = default;
2899 };
2900
2901 /// Return true if the addressing mode represented by AM is legal for this
2902 /// target, for a load/store of the specified type.
2903 ///
2904 /// The type may be VoidTy, in which case only return true if the addressing
2905 /// mode is legal for a load/store of any legal type. TODO: Handle
2906 /// pre/postinc as well.
2907 ///
2908 /// If the address space cannot be determined, it will be -1.
2909 ///
2910 /// TODO: Remove default argument
2911 virtual bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM,
2912 Type *Ty, unsigned AddrSpace,
2913 Instruction *I = nullptr) const;
2914
2915 /// Returns true if the targets addressing mode can target thread local
2916 /// storage (TLS).
2917 virtual bool addressingModeSupportsTLS(const GlobalValue &) const {
2918 return false;
2919 }
2920
2921 /// Return the prefered common base offset.
2922 virtual int64_t getPreferredLargeGEPBaseOffset(int64_t MinOffset,
2923 int64_t MaxOffset) const {
2924 return 0;
2925 }
2926
2927 /// Return true if the specified immediate is legal icmp immediate, that is
2928 /// the target has icmp instructions which can compare a register against the
2929 /// immediate without having to materialize the immediate into a register.
2930 virtual bool isLegalICmpImmediate(int64_t) const {
2931 return true;
2932 }
2933
2934 /// Return true if the specified immediate is legal add immediate, that is the
2935 /// target has add instructions which can add a register with the immediate
2936 /// without having to materialize the immediate into a register.
2937 virtual bool isLegalAddImmediate(int64_t) const {
2938 return true;
2939 }
2940
2941 /// Return true if adding the specified scalable immediate is legal, that is
2942 /// the target has add instructions which can add a register with the
2943 /// immediate (multiplied by vscale) without having to materialize the
2944 /// immediate into a register.
2945 virtual bool isLegalAddScalableImmediate(int64_t) const { return false; }
2946
2947 /// Return true if the specified immediate is legal for the value input of a
2948 /// store instruction.
2949 virtual bool isLegalStoreImmediate(int64_t Value) const {
2950 // Default implementation assumes that at least 0 works since it is likely
2951 // that a zero register exists or a zero immediate is allowed.
2952 return Value == 0;
2953 }
2954
2955 /// Given a shuffle vector SVI representing a vector splat, return a new
2956 /// scalar type of size equal to SVI's scalar type if the new type is more
2957 /// profitable. Returns nullptr otherwise. For example under MVE float splats
2958 /// are converted to integer to prevent the need to move from SPR to GPR
2959 /// registers.
2961 return nullptr;
2962 }
2963
2964 /// Given a set in interconnected phis of type 'From' that are loaded/stored
2965 /// or bitcast to type 'To', return true if the set should be converted to
2966 /// 'To'.
2967 virtual bool shouldConvertPhiType(Type *From, Type *To) const {
2968 return (From->isIntegerTy() || From->isFloatingPointTy()) &&
2969 (To->isIntegerTy() || To->isFloatingPointTy());
2970 }
2971
2972 /// Returns true if the opcode is a commutative binary operation.
2973 virtual bool isCommutativeBinOp(unsigned Opcode) const {
2974 // FIXME: This should get its info from the td file.
2975 switch (Opcode) {
2976 case ISD::ADD:
2977 case ISD::SMIN:
2978 case ISD::SMAX:
2979 case ISD::UMIN:
2980 case ISD::UMAX:
2981 case ISD::MUL:
2982 case ISD::MULHU:
2983 case ISD::MULHS:
2984 case ISD::SMUL_LOHI:
2985 case ISD::UMUL_LOHI:
2986 case ISD::FADD:
2987 case ISD::FMUL:
2988 case ISD::AND:
2989 case ISD::OR:
2990 case ISD::XOR:
2991 case ISD::SADDO:
2992 case ISD::UADDO:
2993 case ISD::ADDC:
2994 case ISD::ADDE:
2995 case ISD::SADDSAT:
2996 case ISD::UADDSAT:
2997 case ISD::FMINNUM:
2998 case ISD::FMAXNUM:
2999 case ISD::FMINNUM_IEEE:
3000 case ISD::FMAXNUM_IEEE:
3001 case ISD::FMINIMUM:
3002 case ISD::FMAXIMUM:
3003 case ISD::FMINIMUMNUM:
3004 case ISD::FMAXIMUMNUM:
3005 case ISD::AVGFLOORS:
3006 case ISD::AVGFLOORU:
3007 case ISD::AVGCEILS:
3008 case ISD::AVGCEILU:
3009 case ISD::ABDS:
3010 case ISD::ABDU:
3011 return true;
3012 default: return false;
3013 }
3014 }
3015
3016 /// Return true if the node is a math/logic binary operator.
3017 virtual bool isBinOp(unsigned Opcode) const {
3018 // A commutative binop must be a binop.
3019 if (isCommutativeBinOp(Opcode))
3020 return true;
3021 // These are non-commutative binops.
3022 switch (Opcode) {
3023 case ISD::SUB:
3024 case ISD::SHL:
3025 case ISD::SRL:
3026 case ISD::SRA:
3027 case ISD::ROTL:
3028 case ISD::ROTR:
3029 case ISD::SDIV:
3030 case ISD::UDIV:
3031 case ISD::SREM:
3032 case ISD::UREM:
3033 case ISD::SSUBSAT:
3034 case ISD::USUBSAT:
3035 case ISD::FSUB:
3036 case ISD::FDIV:
3037 case ISD::FREM:
3038 return true;
3039 default:
3040 return false;
3041 }
3042 }
3043
3044 /// Return true if it's free to truncate a value of type FromTy to type
3045 /// ToTy. e.g. On x86 it's free to truncate a i32 value in register EAX to i16
3046 /// by referencing its sub-register AX.
3047 /// Targets must return false when FromTy <= ToTy.
3048 virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const {
3049 return false;
3050 }
3051
3052 /// Return true if a truncation from FromTy to ToTy is permitted when deciding
3053 /// whether a call is in tail position. Typically this means that both results
3054 /// would be assigned to the same register or stack slot, but it could mean
3055 /// the target performs adequate checks of its own before proceeding with the
3056 /// tail call. Targets must return false when FromTy <= ToTy.
3057 virtual bool allowTruncateForTailCall(Type *FromTy, Type *ToTy) const {
3058 return false;
3059 }
3060
3061 virtual bool isTruncateFree(EVT FromVT, EVT ToVT) const { return false; }
3062 virtual bool isTruncateFree(LLT FromTy, LLT ToTy, LLVMContext &Ctx) const {
3063 return isTruncateFree(getApproximateEVTForLLT(FromTy, Ctx),
3064 getApproximateEVTForLLT(ToTy, Ctx));
3065 }
3066
3067 /// Return true if truncating the specific node Val to type VT2 is free.
3068 virtual bool isTruncateFree(SDValue Val, EVT VT2) const {
3069 // Fallback to type matching.
3070 return isTruncateFree(Val.getValueType(), VT2);
3071 }
3072
3073 virtual bool isProfitableToHoist(Instruction *I) const { return true; }
3074
3075 /// Return true if the extension represented by \p I is free.
3076 /// Unlikely the is[Z|FP]ExtFree family which is based on types,
3077 /// this method can use the context provided by \p I to decide
3078 /// whether or not \p I is free.
3079 /// This method extends the behavior of the is[Z|FP]ExtFree family.
3080 /// In other words, if is[Z|FP]Free returns true, then this method
3081 /// returns true as well. The converse is not true.
3082 /// The target can perform the adequate checks by overriding isExtFreeImpl.
3083 /// \pre \p I must be a sign, zero, or fp extension.
3084 bool isExtFree(const Instruction *I) const {
3085 switch (I->getOpcode()) {
3086 case Instruction::FPExt:
3087 if (isFPExtFree(EVT::getEVT(I->getType()),
3088 EVT::getEVT(I->getOperand(0)->getType())))
3089 return true;
3090 break;
3091 case Instruction::ZExt:
3092 if (isZExtFree(I->getOperand(0)->getType(), I->getType()))
3093 return true;
3094 break;
3095 case Instruction::SExt:
3096 break;
3097 default:
3098 llvm_unreachable("Instruction is not an extension");
3099 }
3100 return isExtFreeImpl(I);
3101 }
3102
3103 /// Return true if \p Load and \p Ext can form an ExtLoad.
3104 /// For example, in AArch64
3105 /// %L = load i8, i8* %ptr
3106 /// %E = zext i8 %L to i32
3107 /// can be lowered into one load instruction
3108 /// ldrb w0, [x0]
3109 bool isExtLoad(const LoadInst *Load, const Instruction *Ext,
3110 const DataLayout &DL) const {
3111 EVT VT = getValueType(DL, Ext->getType());
3112 EVT LoadVT = getValueType(DL, Load->getType());
3113
3114 // If the load has other users and the truncate is not free, the ext
3115 // probably isn't free.
3116 if (!Load->hasOneUse() && (isTypeLegal(LoadVT) || !isTypeLegal(VT)) &&
3117 !isTruncateFree(Ext->getType(), Load->getType()))
3118 return false;
3119
3120 // Check whether the target supports casts folded into loads.
3121 unsigned LType;
3122 if (isa<ZExtInst>(Ext))
3123 LType = ISD::ZEXTLOAD;
3124 else {
3125 assert(isa<SExtInst>(Ext) && "Unexpected ext type!");
3126 LType = ISD::SEXTLOAD;
3127 }
3128
3129 return isLoadExtLegal(LType, VT, LoadVT);
3130 }
3131
3132 /// Return true if any actual instruction that defines a value of type FromTy
3133 /// implicitly zero-extends the value to ToTy in the result register.
3134 ///
3135 /// The function should return true when it is likely that the truncate can
3136 /// be freely folded with an instruction defining a value of FromTy. If
3137 /// the defining instruction is unknown (because you're looking at a
3138 /// function argument, PHI, etc.) then the target may require an
3139 /// explicit truncate, which is not necessarily free, but this function
3140 /// does not deal with those cases.
3141 /// Targets must return false when FromTy >= ToTy.
3142 virtual bool isZExtFree(Type *FromTy, Type *ToTy) const {
3143 return false;
3144 }
3145
3146 virtual bool isZExtFree(EVT FromTy, EVT ToTy) const { return false; }
3147 virtual bool isZExtFree(LLT FromTy, LLT ToTy, LLVMContext &Ctx) const {
3148 return isZExtFree(getApproximateEVTForLLT(FromTy, Ctx),
3149 getApproximateEVTForLLT(ToTy, Ctx));
3150 }
3151
3152 /// Return true if zero-extending the specific node Val to type VT2 is free
3153 /// (either because it's implicitly zero-extended such as ARM ldrb / ldrh or
3154 /// because it's folded such as X86 zero-extending loads).
3155 virtual bool isZExtFree(SDValue Val, EVT VT2) const {
3156 return isZExtFree(Val.getValueType(), VT2);
3157 }
3158
3159 /// Return true if sign-extension from FromTy to ToTy is cheaper than
3160 /// zero-extension.
3161 virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const {
3162 return false;
3163 }
3164
3165 /// Return true if this constant should be sign extended when promoting to
3166 /// a larger type.
3167 virtual bool signExtendConstant(const ConstantInt *C) const { return false; }
3168
3169 /// Try to optimize extending or truncating conversion instructions (like
3170 /// zext, trunc, fptoui, uitofp) for the target.
3171 virtual bool
3173 const TargetTransformInfo &TTI) const {
3174 return false;
3175 }
3176
3177 /// Return true if the target supplies and combines to a paired load
3178 /// two loaded values of type LoadedType next to each other in memory.
3179 /// RequiredAlignment gives the minimal alignment constraints that must be met
3180 /// to be able to select this paired load.
3181 ///
3182 /// This information is *not* used to generate actual paired loads, but it is
3183 /// used to generate a sequence of loads that is easier to combine into a
3184 /// paired load.
3185 /// For instance, something like this:
3186 /// a = load i64* addr
3187 /// b = trunc i64 a to i32
3188 /// c = lshr i64 a, 32
3189 /// d = trunc i64 c to i32
3190 /// will be optimized into:
3191 /// b = load i32* addr1
3192 /// d = load i32* addr2
3193 /// Where addr1 = addr2 +/- sizeof(i32).
3194 ///
3195 /// In other words, unless the target performs a post-isel load combining,
3196 /// this information should not be provided because it will generate more
3197 /// loads.
3198 virtual bool hasPairedLoad(EVT /*LoadedType*/,
3199 Align & /*RequiredAlignment*/) const {
3200 return false;
3201 }
3202
3203 /// Return true if the target has a vector blend instruction.
3204 virtual bool hasVectorBlend() const { return false; }
3205
3206 /// Get the maximum supported factor for interleaved memory accesses.
3207 /// Default to be the minimum interleave factor: 2.
3208 virtual unsigned getMaxSupportedInterleaveFactor() const { return 2; }
3209
3210 /// Lower an interleaved load to target specific intrinsics. Return
3211 /// true on success.
3212 ///
3213 /// \p Load is the vector load instruction. Can be either a plain load
3214 /// instruction or a vp.load intrinsic.
3215 /// \p Mask is a per-segment (i.e. number of lanes equal to that of one
3216 /// component being interwoven) mask. Can be nullptr, in which case the
3217 /// result is uncondiitional.
3218 /// \p Shuffles is the shufflevector list to DE-interleave the loaded vector.
3219 /// \p Indices is the corresponding indices for each shufflevector.
3220 /// \p Factor is the interleave factor.
3221 /// \p GapMask is a mask with zeros for components / fields that may not be
3222 /// accessed.
3223 virtual bool lowerInterleavedLoad(Instruction *Load, Value *Mask,
3225 ArrayRef<unsigned> Indices, unsigned Factor,
3226 const APInt &GapMask) const {
3227 return false;
3228 }
3229
3230 /// Lower an interleaved store to target specific intrinsics. Return
3231 /// true on success.
3232 ///
3233 /// \p SI is the vector store instruction. Can be either a plain store
3234 /// or a vp.store.
3235 /// \p Mask is a per-segment (i.e. number of lanes equal to that of one
3236 /// component being interwoven) mask. Can be nullptr, in which case the
3237 /// result is unconditional.
3238 /// \p SVI is the shufflevector to RE-interleave the stored vector.
3239 /// \p Factor is the interleave factor.
3240 /// \p GapMask is a mask with zeros for components / fields that may not be
3241 /// accessed.
3242 virtual bool lowerInterleavedStore(Instruction *Store, Value *Mask,
3243 ShuffleVectorInst *SVI, unsigned Factor,
3244 const APInt &GapMask) const {
3245 return false;
3246 }
3247
3248 /// Lower a deinterleave intrinsic to a target specific load intrinsic.
3249 /// Return true on success. Currently only supports
3250 /// llvm.vector.deinterleave{2,3,5,7}
3251 ///
3252 /// \p Load is the accompanying load instruction. Can be either a plain load
3253 /// instruction or a vp.load intrinsic.
3254 /// \p DI represents the deinterleaveN intrinsic.
3256 IntrinsicInst *DI) const {
3257 return false;
3258 }
3259
3260 /// Lower an interleave intrinsic to a target specific store intrinsic.
3261 /// Return true on success. Currently only supports
3262 /// llvm.vector.interleave{2,3,5,7}
3263 ///
3264 /// \p Store is the accompanying store instruction. Can be either a plain
3265 /// store or a vp.store intrinsic.
3266 /// \p Mask is a per-segment (i.e. number of lanes equal to that of one
3267 /// component being interwoven) mask. Can be nullptr, in which case the
3268 /// result is uncondiitional.
3269 /// \p InterleaveValues contains the interleaved values.
3270 virtual bool
3272 ArrayRef<Value *> InterleaveValues) const {
3273 return false;
3274 }
3275
3276 /// Return true if an fpext operation is free (for instance, because
3277 /// single-precision floating-point numbers are implicitly extended to
3278 /// double-precision).
3279 virtual bool isFPExtFree(EVT DestVT, EVT SrcVT) const {
3280 assert(SrcVT.isFloatingPoint() && DestVT.isFloatingPoint() &&
3281 "invalid fpext types");
3282 return false;
3283 }
3284
3285 /// Return true if an fpext operation input to an \p Opcode operation is free
3286 /// (for instance, because half-precision floating-point numbers are
3287 /// implicitly extended to float-precision) for an FMA instruction.
3288 virtual bool isFPExtFoldable(const MachineInstr &MI, unsigned Opcode,
3289 LLT DestTy, LLT SrcTy) const {
3290 return false;
3291 }
3292
3293 /// Return true if an fpext operation input to an \p Opcode operation is free
3294 /// (for instance, because half-precision floating-point numbers are
3295 /// implicitly extended to float-precision) for an FMA instruction.
3296 virtual bool isFPExtFoldable(const SelectionDAG &DAG, unsigned Opcode,
3297 EVT DestVT, EVT SrcVT) const {
3298 assert(DestVT.isFloatingPoint() && SrcVT.isFloatingPoint() &&
3299 "invalid fpext types");
3300 return isFPExtFree(DestVT, SrcVT);
3301 }
3302
3303 /// Return true if folding a vector load into ExtVal (a sign, zero, or any
3304 /// extend node) is profitable.
3305 virtual bool isVectorLoadExtDesirable(SDValue ExtVal) const { return false; }
3306
3307 /// Return true if an fneg operation is free to the point where it is never
3308 /// worthwhile to replace it with a bitwise operation.
3309 virtual bool isFNegFree(EVT VT) const {
3310 assert(VT.isFloatingPoint());
3311 return false;
3312 }
3313
3314 /// Return true if an fabs operation is free to the point where it is never
3315 /// worthwhile to replace it with a bitwise operation.
3316 virtual bool isFAbsFree(EVT VT) const {
3317 assert(VT.isFloatingPoint());
3318 return false;
3319 }
3320
3321 /// Return true if an FMA operation is faster than a pair of fmul and fadd
3322 /// instructions. fmuladd intrinsics will be expanded to FMAs when this method
3323 /// returns true, otherwise fmuladd is expanded to fmul + fadd.
3324 ///
3325 /// NOTE: This may be called before legalization on types for which FMAs are
3326 /// not legal, but should return true if those types will eventually legalize
3327 /// to types that support FMAs. After legalization, it will only be called on
3328 /// types that support FMAs (via Legal or Custom actions)
3329 ///
3330 /// Targets that care about soft float support should return false when soft
3331 /// float code is being generated (i.e. use-soft-float).
3333 EVT) const {
3334 return false;
3335 }
3336
3337 /// Return true if an FMA operation is faster than a pair of fmul and fadd
3338 /// instructions. fmuladd intrinsics will be expanded to FMAs when this method
3339 /// returns true, otherwise fmuladd is expanded to fmul + fadd.
3340 ///
3341 /// NOTE: This may be called before legalization on types for which FMAs are
3342 /// not legal, but should return true if those types will eventually legalize
3343 /// to types that support FMAs. After legalization, it will only be called on
3344 /// types that support FMAs (via Legal or Custom actions)
3346 LLT) const {
3347 return false;
3348 }
3349
3350 /// IR version
3351 virtual bool isFMAFasterThanFMulAndFAdd(const Function &F, Type *) const {
3352 return false;
3353 }
3354
3355 /// Returns true if \p MI can be combined with another instruction to
3356 /// form TargetOpcode::G_FMAD. \p N may be an TargetOpcode::G_FADD,
3357 /// TargetOpcode::G_FSUB, or an TargetOpcode::G_FMUL which will be
3358 /// distributed into an fadd/fsub.
3359 virtual bool isFMADLegal(const MachineInstr &MI, LLT Ty) const {
3360 assert((MI.getOpcode() == TargetOpcode::G_FADD ||
3361 MI.getOpcode() == TargetOpcode::G_FSUB ||
3362 MI.getOpcode() == TargetOpcode::G_FMUL) &&
3363 "unexpected node in FMAD forming combine");
3364 switch (Ty.getScalarSizeInBits()) {
3365 case 16:
3366 return isOperationLegal(TargetOpcode::G_FMAD, MVT::f16);
3367 case 32:
3368 return isOperationLegal(TargetOpcode::G_FMAD, MVT::f32);
3369 case 64:
3370 return isOperationLegal(TargetOpcode::G_FMAD, MVT::f64);
3371 default:
3372 break;
3373 }
3374
3375 return false;
3376 }
3377
3378 /// Returns true if be combined with to form an ISD::FMAD. \p N may be an
3379 /// ISD::FADD, ISD::FSUB, or an ISD::FMUL which will be distributed into an
3380 /// fadd/fsub.
3381 virtual bool isFMADLegal(const SelectionDAG &DAG, const SDNode *N) const {
3382 assert((N->getOpcode() == ISD::FADD || N->getOpcode() == ISD::FSUB ||
3383 N->getOpcode() == ISD::FMUL) &&
3384 "unexpected node in FMAD forming combine");
3385 return isOperationLegal(ISD::FMAD, N->getValueType(0));
3386 }
3387
3388 // Return true when the decision to generate FMA's (or FMS, FMLA etc) rather
3389 // than FMUL and ADD is delegated to the machine combiner.
3391 CodeGenOptLevel OptLevel) const {
3392 return false;
3393 }
3394
3395 /// Return true if it's profitable to narrow operations of type SrcVT to
3396 /// DestVT. e.g. on x86, it's profitable to narrow from i32 to i8 but not from
3397 /// i32 to i16.
3398 virtual bool isNarrowingProfitable(SDNode *N, EVT SrcVT, EVT DestVT) const {
3399 return false;
3400 }
3401
3402 /// Return true if pulling a binary operation into a select with an identity
3403 /// constant is profitable. This is the inverse of an IR transform.
3404 /// Example: X + (Cond ? Y : 0) --> Cond ? (X + Y) : X
3405 virtual bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT,
3406 unsigned SelectOpcode,
3407 SDValue X,
3408 SDValue Y) const {
3409 return false;
3410 }
3411
3412 /// Return true if it is beneficial to convert a load of a constant to
3413 /// just the constant itself.
3414 /// On some targets it might be more efficient to use a combination of
3415 /// arithmetic instructions to materialize the constant instead of loading it
3416 /// from a constant pool.
3418 Type *Ty) const {
3419 return false;
3420 }
3421
3422 /// Return true if EXTRACT_SUBVECTOR is cheap for extracting this result type
3423 /// from this source type with this index. This is needed because
3424 /// EXTRACT_SUBVECTOR usually has custom lowering that depends on the index of
3425 /// the first element, and only the target knows which lowering is cheap.
3426 virtual bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
3427 unsigned Index) const {
3428 return false;
3429 }
3430
3431 /// Try to convert an extract element of a vector binary operation into an
3432 /// extract element followed by a scalar operation.
3433 virtual bool shouldScalarizeBinop(SDValue VecOp) const {
3434 return false;
3435 }
3436
3437 /// Return true if extraction of a scalar element from the given vector type
3438 /// at the given index is cheap. For example, if scalar operations occur on
3439 /// the same register file as vector operations, then an extract element may
3440 /// be a sub-register rename rather than an actual instruction.
3441 virtual bool isExtractVecEltCheap(EVT VT, unsigned Index) const {
3442 return false;
3443 }
3444
3445 /// Try to convert math with an overflow comparison into the corresponding DAG
3446 /// node operation. Targets may want to override this independently of whether
3447 /// the operation is legal/custom for the given type because it may obscure
3448 /// matching of other patterns.
3449 virtual bool shouldFormOverflowOp(unsigned Opcode, EVT VT,
3450 bool MathUsed) const {
3451 // Form it if it is legal.
3452 if (isOperationLegal(Opcode, VT))
3453 return true;
3454
3455 // TODO: The default logic is inherited from code in CodeGenPrepare.
3456 // The opcode should not make a difference by default?
3457 if (Opcode != ISD::UADDO)
3458 return false;
3459
3460 // Allow the transform as long as we have an integer type that is not
3461 // obviously illegal and unsupported and if the math result is used
3462 // besides the overflow check. On some targets (e.g. SPARC), it is
3463 // not profitable to form on overflow op if the math result has no
3464 // concrete users.
3465 if (VT.isVector())
3466 return false;
3467 return MathUsed && (VT.isSimple() || !isOperationExpand(Opcode, VT));
3468 }
3469
3470 // Return true if it is profitable to use a scalar input to a BUILD_VECTOR
3471 // even if the vector itself has multiple uses.
3472 virtual bool aggressivelyPreferBuildVectorSources(EVT VecVT) const {
3473 return false;
3474 }
3475
3476 // Return true if CodeGenPrepare should consider splitting large offset of a
3477 // GEP to make the GEP fit into the addressing mode and can be sunk into the
3478 // same blocks of its users.
3479 virtual bool shouldConsiderGEPOffsetSplit() const { return false; }
3480
3481 /// Return true if creating a shift of the type by the given
3482 /// amount is not profitable.
3483 virtual bool shouldAvoidTransformToShift(EVT VT, unsigned Amount) const {
3484 return false;
3485 }
3486
3487 // Should we fold (select_cc seteq (and x, y), 0, 0, A) -> (and (sra (shl x))
3488 // A) where y has a single bit set?
3490 const APInt &AndMask) const {
3491 unsigned ShCt = AndMask.getBitWidth() - 1;
3492 return !shouldAvoidTransformToShift(VT, ShCt);
3493 }
3494
3495 /// Does this target require the clearing of high-order bits in a register
3496 /// passed to the fp16 to fp conversion library function.
3497 virtual bool shouldKeepZExtForFP16Conv() const { return false; }
3498
3499 /// Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT
3500 /// from min(max(fptoi)) saturation patterns.
3501 virtual bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const {
3502 return isOperationLegalOrCustom(Op, VT);
3503 }
3504
3505 /// Should we prefer selects to doing arithmetic on boolean types
3507 return false;
3508 }
3509
3510 /// True if target has some particular form of dealing with pointer arithmetic
3511 /// semantics for pointers with the given value type. False if pointer
3512 /// arithmetic should not be preserved for passes such as instruction
3513 /// selection, and can fallback to regular arithmetic.
3514 /// This should be removed when PTRADD nodes are widely supported by backends.
3515 virtual bool shouldPreservePtrArith(const Function &F, EVT PtrVT) const {
3516 return false;
3517 }
3518
3519 /// True if the target allows transformations of in-bounds pointer
3520 /// arithmetic that cause out-of-bounds intermediate results.
3522 EVT PtrVT) const {
3523 return false;
3524 }
3525
3526 /// Does this target support complex deinterleaving
3527 virtual bool isComplexDeinterleavingSupported() const { return false; }
3528
3529 /// Does this target support complex deinterleaving with the given operation
3530 /// and type
3533 return false;
3534 }
3535
3536 // Get the preferred opcode for FP_TO_XINT nodes.
3537 // By default, this checks if the provded operation is an illegal FP_TO_UINT
3538 // and if so, checks if FP_TO_SINT is legal or custom for use as a
3539 // replacement. If both UINT and SINT conversions are Custom, we choose SINT
3540 // by default because that's the right thing on PPC.
3541 virtual unsigned getPreferredFPToIntOpcode(unsigned Op, EVT FromVT,
3542 EVT ToVT) const {
3543 if (isOperationLegal(Op, ToVT))
3544 return Op;
3545 switch (Op) {
3546 case ISD::FP_TO_UINT:
3548 return ISD::FP_TO_SINT;
3549 break;
3553 break;
3554 case ISD::VP_FP_TO_UINT:
3555 if (isOperationLegalOrCustom(ISD::VP_FP_TO_SINT, ToVT))
3556 return ISD::VP_FP_TO_SINT;
3557 break;
3558 default:
3559 break;
3560 }
3561 return Op;
3562 }
3563
3564 /// Create the IR node for the given complex deinterleaving operation.
3565 /// If one cannot be created using all the given inputs, nullptr should be
3566 /// returned.
3569 ComplexDeinterleavingRotation Rotation, Value *InputA, Value *InputB,
3570 Value *Accumulator = nullptr) const {
3571 return nullptr;
3572 }
3573
3574 void setLibcallImpl(RTLIB::Libcall Call, RTLIB::LibcallImpl Impl) {
3575 Libcalls.setLibcallImpl(Call, Impl);
3576 }
3577
3578 /// Get the libcall impl routine name for the specified libcall.
3579 RTLIB::LibcallImpl getLibcallImpl(RTLIB::Libcall Call) const {
3580 return Libcalls.getLibcallImpl(Call);
3581 }
3582
3583 /// Get the libcall routine name for the specified libcall.
3584 const char *getLibcallName(RTLIB::Libcall Call) const {
3585 // FIXME: Return StringRef
3586 return Libcalls.getLibcallName(Call).data();
3587 }
3588
3589 /// Get the libcall routine name for the specified libcall implementation
3593
3594 const char *getMemcpyName() const {
3595 // FIXME: Return StringRef
3596 return Libcalls.getMemcpyName().data();
3597 }
3598
3599 /// Check if this is valid libcall for the current module, otherwise
3600 /// RTLIB::Unsupported.
3601 RTLIB::LibcallImpl getSupportedLibcallImpl(StringRef FuncName) const {
3602 return Libcalls.getSupportedLibcallImpl(FuncName);
3603 }
3604
3605 /// Get the comparison predicate that's to be used to test the result of the
3606 /// comparison libcall against zero. This should only be used with
3607 /// floating-point compare libcalls.
3608 ISD::CondCode getSoftFloatCmpLibcallPredicate(RTLIB::LibcallImpl Call) const;
3609
3610 /// Set the CallingConv that should be used for the specified libcall.
3611 void setLibcallImplCallingConv(RTLIB::LibcallImpl Call, CallingConv::ID CC) {
3612 Libcalls.setLibcallImplCallingConv(Call, CC);
3613 }
3614
3615 /// Get the CallingConv that should be used for the specified libcall
3616 /// implementation.
3618 return Libcalls.getLibcallImplCallingConv(Call);
3619 }
3620
3621 /// Get the CallingConv that should be used for the specified libcall.
3622 // FIXME: Remove this wrapper and directly use the used LibcallImpl
3624 return Libcalls.getLibcallCallingConv(Call);
3625 }
3626
3627 /// Execute target specific actions to finalize target lowering.
3628 /// This is used to set extra flags in MachineFrameInformation and freezing
3629 /// the set of reserved registers.
3630 /// The default implementation just freezes the set of reserved registers.
3631 virtual void finalizeLowering(MachineFunction &MF) const;
3632
3633 /// Returns true if it's profitable to allow merging store of loads when there
3634 /// are functions calls between the load and the store.
3635 virtual bool shouldMergeStoreOfLoadsOverCall(EVT, EVT) const { return true; }
3636
3637 //===----------------------------------------------------------------------===//
3638 // GlobalISel Hooks
3639 //===----------------------------------------------------------------------===//
3640 /// Check whether or not \p MI needs to be moved close to its uses.
3641 virtual bool shouldLocalize(const MachineInstr &MI, const TargetTransformInfo *TTI) const;
3642
3643
3644private:
3645 const TargetMachine &TM;
3646
3647 /// Tells the code generator that the target has BitExtract instructions.
3648 /// The code generator will aggressively sink "shift"s into the blocks of
3649 /// their users if the users will generate "and" instructions which can be
3650 /// combined with "shift" to BitExtract instructions.
3651 bool HasExtractBitsInsn;
3652
3653 /// Tells the code generator to bypass slow divide or remainder
3654 /// instructions. For example, BypassSlowDivWidths[32,8] tells the code
3655 /// generator to bypass 32-bit integer div/rem with an 8-bit unsigned integer
3656 /// div/rem when the operands are positive and less than 256.
3657 DenseMap <unsigned int, unsigned int> BypassSlowDivWidths;
3658
3659 /// Tells the code generator that it shouldn't generate extra flow control
3660 /// instructions and should attempt to combine flow control instructions via
3661 /// predication.
3662 bool JumpIsExpensive;
3663
3664 /// Information about the contents of the high-bits in boolean values held in
3665 /// a type wider than i1. See getBooleanContents.
3666 BooleanContent BooleanContents;
3667
3668 /// Information about the contents of the high-bits in boolean values held in
3669 /// a type wider than i1. See getBooleanContents.
3670 BooleanContent BooleanFloatContents;
3671
3672 /// Information about the contents of the high-bits in boolean vector values
3673 /// when the element type is wider than i1. See getBooleanContents.
3674 BooleanContent BooleanVectorContents;
3675
3676 /// The target scheduling preference: shortest possible total cycles or lowest
3677 /// register usage.
3678 Sched::Preference SchedPreferenceInfo;
3679
3680 /// The minimum alignment that any argument on the stack needs to have.
3681 Align MinStackArgumentAlignment;
3682
3683 /// The minimum function alignment (used when optimizing for size, and to
3684 /// prevent explicitly provided alignment from leading to incorrect code).
3685 Align MinFunctionAlignment;
3686
3687 /// The preferred function alignment (used when alignment unspecified and
3688 /// optimizing for speed).
3689 Align PrefFunctionAlignment;
3690
3691 /// The preferred loop alignment (in log2 bot in bytes).
3692 Align PrefLoopAlignment;
3693 /// The maximum amount of bytes permitted to be emitted for alignment.
3694 unsigned MaxBytesForAlignment;
3695
3696 /// Size in bits of the maximum atomics size the backend supports.
3697 /// Accesses larger than this will be expanded by AtomicExpandPass.
3698 unsigned MaxAtomicSizeInBitsSupported;
3699
3700 /// Size in bits of the maximum div/rem size the backend supports.
3701 /// Larger operations will be expanded by ExpandLargeDivRem.
3702 unsigned MaxDivRemBitWidthSupported;
3703
3704 /// Size in bits of the maximum fp to/from int conversion size the
3705 /// backend supports. Larger operations will be expanded by
3706 /// ExpandFp.
3707 unsigned MaxLargeFPConvertBitWidthSupported;
3708
3709 /// Size in bits of the minimum cmpxchg or ll/sc operation the
3710 /// backend supports.
3711 unsigned MinCmpXchgSizeInBits;
3712
3713 /// This indicates if the target supports unaligned atomic operations.
3714 bool SupportsUnalignedAtomics;
3715
3716 /// If set to a physical register, this specifies the register that
3717 /// llvm.savestack/llvm.restorestack should save and restore.
3718 Register StackPointerRegisterToSaveRestore;
3719
3720 /// This indicates the default register class to use for each ValueType the
3721 /// target supports natively.
3722 const TargetRegisterClass *RegClassForVT[MVT::VALUETYPE_SIZE];
3723 uint16_t NumRegistersForVT[MVT::VALUETYPE_SIZE];
3724 MVT RegisterTypeForVT[MVT::VALUETYPE_SIZE];
3725
3726 /// This indicates the "representative" register class to use for each
3727 /// ValueType the target supports natively. This information is used by the
3728 /// scheduler to track register pressure. By default, the representative
3729 /// register class is the largest legal super-reg register class of the
3730 /// register class of the specified type. e.g. On x86, i8, i16, and i32's
3731 /// representative class would be GR32.
3732 const TargetRegisterClass *RepRegClassForVT[MVT::VALUETYPE_SIZE] = {0};
3733
3734 /// This indicates the "cost" of the "representative" register class for each
3735 /// ValueType. The cost is used by the scheduler to approximate register
3736 /// pressure.
3737 uint8_t RepRegClassCostForVT[MVT::VALUETYPE_SIZE];
3738
3739 /// For any value types we are promoting or expanding, this contains the value
3740 /// type that we are changing to. For Expanded types, this contains one step
3741 /// of the expand (e.g. i64 -> i32), even if there are multiple steps required
3742 /// (e.g. i64 -> i16). For types natively supported by the system, this holds
3743 /// the same type (e.g. i32 -> i32).
3744 MVT TransformToType[MVT::VALUETYPE_SIZE];
3745
3746 /// For each operation and each value type, keep a LegalizeAction that
3747 /// indicates how instruction selection should deal with the operation. Most
3748 /// operations are Legal (aka, supported natively by the target), but
3749 /// operations that are not should be described. Note that operations on
3750 /// non-legal value types are not described here.
3751 LegalizeAction OpActions[MVT::VALUETYPE_SIZE][ISD::BUILTIN_OP_END];
3752
3753 /// For each load extension type and each value type, keep a LegalizeAction
3754 /// that indicates how instruction selection should deal with a load of a
3755 /// specific value type and extension type. Uses 4-bits to store the action
3756 /// for each of the 4 load ext types.
3757 uint16_t LoadExtActions[MVT::VALUETYPE_SIZE][MVT::VALUETYPE_SIZE];
3758
3759 /// Similar to LoadExtActions, but for atomic loads. Only Legal or Expand
3760 /// (default) values are supported.
3761 uint16_t AtomicLoadExtActions[MVT::VALUETYPE_SIZE][MVT::VALUETYPE_SIZE];
3762
3763 /// For each value type pair keep a LegalizeAction that indicates whether a
3764 /// truncating store of a specific value type and truncating type is legal.
3765 LegalizeAction TruncStoreActions[MVT::VALUETYPE_SIZE][MVT::VALUETYPE_SIZE];
3766
3767 /// For each indexed mode and each value type, keep a quad of LegalizeAction
3768 /// that indicates how instruction selection should deal with the load /
3769 /// store / maskedload / maskedstore.
3770 ///
3771 /// The first dimension is the value_type for the reference. The second
3772 /// dimension represents the various modes for load store.
3773 uint16_t IndexedModeActions[MVT::VALUETYPE_SIZE][ISD::LAST_INDEXED_MODE];
3774
3775 /// For each condition code (ISD::CondCode) keep a LegalizeAction that
3776 /// indicates how instruction selection should deal with the condition code.
3777 ///
3778 /// Because each CC action takes up 4 bits, we need to have the array size be
3779 /// large enough to fit all of the value types. This can be done by rounding
3780 /// up the MVT::VALUETYPE_SIZE value to the next multiple of 8.
3781 uint32_t CondCodeActions[ISD::SETCC_INVALID][(MVT::VALUETYPE_SIZE + 7) / 8];
3782
3783 using PartialReduceActionTypes =
3784 std::tuple<unsigned, MVT::SimpleValueType, MVT::SimpleValueType>;
3785 /// For each partial reduce opcode, result type and input type combination,
3786 /// keep a LegalizeAction which indicates how instruction selection should
3787 /// deal with this operation.
3788 DenseMap<PartialReduceActionTypes, LegalizeAction> PartialReduceMLAActions;
3789
3790 ValueTypeActionImpl ValueTypeActions;
3791
3792private:
3793 /// Targets can specify ISD nodes that they would like PerformDAGCombine
3794 /// callbacks for by calling setTargetDAGCombine(), which sets a bit in this
3795 /// array.
3796 unsigned char
3797 TargetDAGCombineArray[(ISD::BUILTIN_OP_END+CHAR_BIT-1)/CHAR_BIT];
3798
3799 /// For operations that must be promoted to a specific type, this holds the
3800 /// destination type. This map should be sparse, so don't hold it as an
3801 /// array.
3802 ///
3803 /// Targets add entries to this map with AddPromotedToType(..), clients access
3804 /// this with getTypeToPromoteTo(..).
3805 std::map<std::pair<unsigned, MVT::SimpleValueType>, MVT::SimpleValueType>
3806 PromoteToType;
3807
3808 /// The list of libcalls that the target will use.
3809 RTLIB::RuntimeLibcallsInfo Libcalls;
3810
3811 /// The ISD::CondCode that should be used to test the result of each of the
3812 /// comparison libcall against zero.
3813 ISD::CondCode CmpLibcallCCs[RTLIB::UNKNOWN_LIBCALL];
3814
3815 /// The bits of IndexedModeActions used to store the legalisation actions
3816 /// We store the data as | ML | MS | L | S | each taking 4 bits.
3817 enum IndexedModeActionsBits {
3818 IMAB_Store = 0,
3819 IMAB_Load = 4,
3820 IMAB_MaskedStore = 8,
3821 IMAB_MaskedLoad = 12
3822 };
3823
3824 void setIndexedModeAction(unsigned IdxMode, MVT VT, unsigned Shift,
3825 LegalizeAction Action) {
3826 assert(VT.isValid() && IdxMode < ISD::LAST_INDEXED_MODE &&
3827 (unsigned)Action < 0xf && "Table isn't big enough!");
3828 unsigned Ty = (unsigned)VT.SimpleTy;
3829 IndexedModeActions[Ty][IdxMode] &= ~(0xf << Shift);
3830 IndexedModeActions[Ty][IdxMode] |= ((uint16_t)Action) << Shift;
3831 }
3832
3833 LegalizeAction getIndexedModeAction(unsigned IdxMode, MVT VT,
3834 unsigned Shift) const {
3835 assert(IdxMode < ISD::LAST_INDEXED_MODE && VT.isValid() &&
3836 "Table isn't big enough!");
3837 unsigned Ty = (unsigned)VT.SimpleTy;
3838 return (LegalizeAction)((IndexedModeActions[Ty][IdxMode] >> Shift) & 0xf);
3839 }
3840
3841protected:
3842 /// Return true if the extension represented by \p I is free.
3843 /// \pre \p I is a sign, zero, or fp extension and
3844 /// is[Z|FP]ExtFree of the related types is not true.
3845 virtual bool isExtFreeImpl(const Instruction *I) const { return false; }
3846
3847 /// Depth that GatherAllAliases should continue looking for chain
3848 /// dependencies when trying to find a more preferable chain. As an
3849 /// approximation, this should be more than the number of consecutive stores
3850 /// expected to be merged.
3852
3853 /// \brief Specify maximum number of store instructions per memset call.
3854 ///
3855 /// When lowering \@llvm.memset this field specifies the maximum number of
3856 /// store operations that may be substituted for the call to memset. Targets
3857 /// must set this value based on the cost threshold for that target. Targets
3858 /// should assume that the memset will be done using as many of the largest
3859 /// store operations first, followed by smaller ones, if necessary, per
3860 /// alignment restrictions. For example, storing 9 bytes on a 32-bit machine
3861 /// with 16-bit alignment would result in four 2-byte stores and one 1-byte
3862 /// store. This only applies to setting a constant array of a constant size.
3864 /// Likewise for functions with the OptSize attribute.
3866
3867 /// \brief Specify maximum number of store instructions per memcpy call.
3868 ///
3869 /// When lowering \@llvm.memcpy this field specifies the maximum number of
3870 /// store operations that may be substituted for a call to memcpy. Targets
3871 /// must set this value based on the cost threshold for that target. Targets
3872 /// should assume that the memcpy will be done using as many of the largest
3873 /// store operations first, followed by smaller ones, if necessary, per
3874 /// alignment restrictions. For example, storing 7 bytes on a 32-bit machine
3875 /// with 32-bit alignment would result in one 4-byte store, a one 2-byte store
3876 /// and one 1-byte store. This only applies to copying a constant array of
3877 /// constant size.
3879 /// Likewise for functions with the OptSize attribute.
3881 /// \brief Specify max number of store instructions to glue in inlined memcpy.
3882 ///
3883 /// When memcpy is inlined based on MaxStoresPerMemcpy, specify maximum number
3884 /// of store instructions to keep together. This helps in pairing and
3885 // vectorization later on.
3887
3888 /// \brief Specify maximum number of load instructions per memcmp call.
3889 ///
3890 /// When lowering \@llvm.memcmp this field specifies the maximum number of
3891 /// pairs of load operations that may be substituted for a call to memcmp.
3892 /// Targets must set this value based on the cost threshold for that target.
3893 /// Targets should assume that the memcmp will be done using as many of the
3894 /// largest load operations first, followed by smaller ones, if necessary, per
3895 /// alignment restrictions. For example, loading 7 bytes on a 32-bit machine
3896 /// with 32-bit alignment would result in one 4-byte load, a one 2-byte load
3897 /// and one 1-byte load. This only applies to copying a constant array of
3898 /// constant size.
3900 /// Likewise for functions with the OptSize attribute.
3902
3903 /// \brief Specify maximum number of store instructions per memmove call.
3904 ///
3905 /// When lowering \@llvm.memmove this field specifies the maximum number of
3906 /// store instructions that may be substituted for a call to memmove. Targets
3907 /// must set this value based on the cost threshold for that target. Targets
3908 /// should assume that the memmove will be done using as many of the largest
3909 /// store operations first, followed by smaller ones, if necessary, per
3910 /// alignment restrictions. For example, moving 9 bytes on a 32-bit machine
3911 /// with 8-bit alignment would result in nine 1-byte stores. This only
3912 /// applies to copying a constant array of constant size.
3914 /// Likewise for functions with the OptSize attribute.
3916
3917 /// Tells the code generator that select is more expensive than a branch if
3918 /// the branch is usually predicted right.
3920
3921 /// \see enableExtLdPromotion.
3923
3924 /// Return true if the value types that can be represented by the specified
3925 /// register class are all legal.
3926 bool isLegalRC(const TargetRegisterInfo &TRI,
3927 const TargetRegisterClass &RC) const;
3928
3929 /// Replace/modify any TargetFrameIndex operands with a targte-dependent
3930 /// sequence of memory operands that is recognized by PrologEpilogInserter.
3932 MachineBasicBlock *MBB) const;
3933
3935};
3936
3937/// This class defines information used to lower LLVM code to legal SelectionDAG
3938/// operators that the target instruction selector can accept natively.
3939///
3940/// This class also defines callbacks that targets must implement to lower
3941/// target-specific constructs to SelectionDAG operators.
3943public:
3944 struct DAGCombinerInfo;
3945 struct MakeLibCallOptions;
3946
3949
3950 explicit TargetLowering(const TargetMachine &TM);
3952
3953 bool isPositionIndependent() const;
3954
3957 UniformityInfo *UA) const {
3958 return false;
3959 }
3960
3961 // Lets target to control the following reassociation of operands: (op (op x,
3962 // c1), y) -> (op (op x, y), c1) where N0 is (op x, c1) and N1 is y. By
3963 // default consider profitable any case where N0 has single use. This
3964 // behavior reflects the condition replaced by this target hook call in the
3965 // DAGCombiner. Any particular target can implement its own heuristic to
3966 // restrict common combiner.
3968 SDValue N1) const {
3969 return N0.hasOneUse();
3970 }
3971
3972 // Lets target to control the following reassociation of operands: (op (op x,
3973 // c1), y) -> (op (op x, y), c1) where N0 is (op x, c1) and N1 is y. By
3974 // default consider profitable any case where N0 has single use. This
3975 // behavior reflects the condition replaced by this target hook call in the
3976 // combiner. Any particular target can implement its own heuristic to
3977 // restrict common combiner.
3979 Register N1) const {
3980 return MRI.hasOneNonDBGUse(N0);
3981 }
3982
3983 virtual bool isSDNodeAlwaysUniform(const SDNode * N) const {
3984 return false;
3985 }
3986
3987 /// Returns true by value, base pointer and offset pointer and addressing mode
3988 /// by reference if the node's address can be legally represented as
3989 /// pre-indexed load / store address.
3990 virtual bool getPreIndexedAddressParts(SDNode * /*N*/, SDValue &/*Base*/,
3991 SDValue &/*Offset*/,
3992 ISD::MemIndexedMode &/*AM*/,
3993 SelectionDAG &/*DAG*/) const {
3994 return false;
3995 }
3996
3997 /// Returns true by value, base pointer and offset pointer and addressing mode
3998 /// by reference if this node can be combined with a load / store to form a
3999 /// post-indexed load / store.
4000 virtual bool getPostIndexedAddressParts(SDNode * /*N*/, SDNode * /*Op*/,
4001 SDValue &/*Base*/,
4002 SDValue &/*Offset*/,
4003 ISD::MemIndexedMode &/*AM*/,
4004 SelectionDAG &/*DAG*/) const {
4005 return false;
4006 }
4007
4008 /// Returns true if the specified base+offset is a legal indexed addressing
4009 /// mode for this target. \p MI is the load or store instruction that is being
4010 /// considered for transformation.
4012 bool IsPre, MachineRegisterInfo &MRI) const {
4013 return false;
4014 }
4015
4016 /// Return the entry encoding for a jump table in the current function. The
4017 /// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.
4018 virtual unsigned getJumpTableEncoding() const;
4019
4020 virtual MVT getJumpTableRegTy(const DataLayout &DL) const {
4021 return getPointerTy(DL);
4022 }
4023
4024 virtual const MCExpr *
4026 const MachineBasicBlock * /*MBB*/, unsigned /*uid*/,
4027 MCContext &/*Ctx*/) const {
4028 llvm_unreachable("Need to implement this hook if target has custom JTIs");
4029 }
4030
4031 /// Returns relocation base for the given PIC jumptable.
4032 virtual SDValue getPICJumpTableRelocBase(SDValue Table,
4033 SelectionDAG &DAG) const;
4034
4035 /// This returns the relocation base for the given PIC jumptable, the same as
4036 /// getPICJumpTableRelocBase, but as an MCExpr.
4037 virtual const MCExpr *
4038 getPICJumpTableRelocBaseExpr(const MachineFunction *MF,
4039 unsigned JTI, MCContext &Ctx) const;
4040
4041 /// Return true if folding a constant offset with the given GlobalAddress is
4042 /// legal. It is frequently not legal in PIC relocation models.
4043 virtual bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const;
4044
4045 /// On x86, return true if the operand with index OpNo is a CALL or JUMP
4046 /// instruction, which can use either a memory constraint or an address
4047 /// constraint. -fasm-blocks "__asm call foo" lowers to
4048 /// call void asm sideeffect inteldialect "call ${0:P}", "*m..."
4049 ///
4050 /// This function is used by a hack to choose the address constraint,
4051 /// lowering to a direct call.
4052 virtual bool
4054 unsigned OpNo) const {
4055 return false;
4056 }
4057
4059 SDValue &Chain) const;
4060
4061 void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS,
4062 SDValue &NewRHS, ISD::CondCode &CCCode,
4063 const SDLoc &DL, const SDValue OldLHS,
4064 const SDValue OldRHS) const;
4065
4066 void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS,
4067 SDValue &NewRHS, ISD::CondCode &CCCode,
4068 const SDLoc &DL, const SDValue OldLHS,
4069 const SDValue OldRHS, SDValue &Chain,
4070 bool IsSignaling = false) const;
4071
4073 SDValue Chain, MachineMemOperand *MMO,
4074 SDValue &NewLoad, SDValue Ptr,
4075 SDValue PassThru, SDValue Mask) const {
4076 llvm_unreachable("Not Implemented");
4077 }
4078
4080 SDValue Chain, MachineMemOperand *MMO,
4081 SDValue Ptr, SDValue Val,
4082 SDValue Mask) const {
4083 llvm_unreachable("Not Implemented");
4084 }
4085
4086 /// Returns a pair of (return value, chain).
4087 /// It is an error to pass RTLIB::UNKNOWN_LIBCALL as \p LC.
4088 std::pair<SDValue, SDValue> makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC,
4089 EVT RetVT, ArrayRef<SDValue> Ops,
4090 MakeLibCallOptions CallOptions,
4091 const SDLoc &dl,
4092 SDValue Chain = SDValue()) const;
4093
4094 /// Check whether parameters to a call that are passed in callee saved
4095 /// registers are the same as from the calling function. This needs to be
4096 /// checked for tail call eligibility.
4097 bool parametersInCSRMatch(const MachineRegisterInfo &MRI,
4098 const uint32_t *CallerPreservedMask,
4099 const SmallVectorImpl<CCValAssign> &ArgLocs,
4100 const SmallVectorImpl<SDValue> &OutVals) const;
4101
4102 //===--------------------------------------------------------------------===//
4103 // TargetLowering Optimization Methods
4104 //
4105
4106 /// A convenience struct that encapsulates a DAG, and two SDValues for
4107 /// returning information from TargetLowering to its clients that want to
4108 /// combine.
4115
4117 bool LT, bool LO) :
4118 DAG(InDAG), LegalTys(LT), LegalOps(LO) {}
4119
4120 bool LegalTypes() const { return LegalTys; }
4121 bool LegalOperations() const { return LegalOps; }
4122
4124 Old = O;
4125 New = N;
4126 return true;
4127 }
4128 };
4129
4130 /// Determines the optimal series of memory ops to replace the memset / memcpy.
4131 /// Return true if the number of memory ops is below the threshold (Limit).
4132 /// Note that this is always the case when Limit is ~0.
4133 /// It returns the types of the sequence of memory ops to perform
4134 /// memset / memcpy by reference.
4135 virtual bool
4136 findOptimalMemOpLowering(LLVMContext &Context, std::vector<EVT> &MemOps,
4137 unsigned Limit, const MemOp &Op, unsigned DstAS,
4138 unsigned SrcAS,
4139 const AttributeList &FuncAttributes) const;
4140
4141 /// Check to see if the specified operand of the specified instruction is a
4142 /// constant integer. If so, check to see if there are any bits set in the
4143 /// constant that are not demanded. If so, shrink the constant and return
4144 /// true.
4146 const APInt &DemandedElts,
4147 TargetLoweringOpt &TLO) const;
4148
4149 /// Helper wrapper around ShrinkDemandedConstant, demanding all elements.
4151 TargetLoweringOpt &TLO) const;
4152
4153 // Target hook to do target-specific const optimization, which is called by
4154 // ShrinkDemandedConstant. This function should return true if the target
4155 // doesn't want ShrinkDemandedConstant to further optimize the constant.
4157 const APInt &DemandedBits,
4158 const APInt &DemandedElts,
4159 TargetLoweringOpt &TLO) const {
4160 return false;
4161 }
4162
4163 /// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
4164 /// This uses isTruncateFree/isZExtFree and ANY_EXTEND for the widening cast,
4165 /// but it could be generalized for targets with other types of implicit
4166 /// widening casts.
4167 bool ShrinkDemandedOp(SDValue Op, unsigned BitWidth,
4168 const APInt &DemandedBits,
4169 TargetLoweringOpt &TLO) const;
4170
4171 /// Look at Op. At this point, we know that only the DemandedBits bits of the
4172 /// result of Op are ever used downstream. If we can use this information to
4173 /// simplify Op, create a new simplified DAG node and return true, returning
4174 /// the original and new nodes in Old and New. Otherwise, analyze the
4175 /// expression and return a mask of KnownOne and KnownZero bits for the
4176 /// expression (used to simplify the caller). The KnownZero/One bits may only
4177 /// be accurate for those bits in the Demanded masks.
4178 /// \p AssumeSingleUse When this parameter is true, this function will
4179 /// attempt to simplify \p Op even if there are multiple uses.
4180 /// Callers are responsible for correctly updating the DAG based on the
4181 /// results of this function, because simply replacing TLO.Old
4182 /// with TLO.New will be incorrect when this parameter is true and TLO.Old
4183 /// has multiple uses.
4184 bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
4185 const APInt &DemandedElts, KnownBits &Known,
4186 TargetLoweringOpt &TLO, unsigned Depth = 0,
4187 bool AssumeSingleUse = false) const;
4188
4189 /// Helper wrapper around SimplifyDemandedBits, demanding all elements.
4190 /// Adds Op back to the worklist upon success.
4191 bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
4192 KnownBits &Known, TargetLoweringOpt &TLO,
4193 unsigned Depth = 0,
4194 bool AssumeSingleUse = false) const;
4195
4196 /// Helper wrapper around SimplifyDemandedBits.
4197 /// Adds Op back to the worklist upon success.
4198 bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
4199 DAGCombinerInfo &DCI) const;
4200
4201 /// Helper wrapper around SimplifyDemandedBits.
4202 /// Adds Op back to the worklist upon success.
4203 bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
4204 const APInt &DemandedElts,
4205 DAGCombinerInfo &DCI) const;
4206
4207 /// More limited version of SimplifyDemandedBits that can be used to "look
4208 /// through" ops that don't contribute to the DemandedBits/DemandedElts -
4209 /// bitwise ops etc.
4210 SDValue SimplifyMultipleUseDemandedBits(SDValue Op, const APInt &DemandedBits,
4211 const APInt &DemandedElts,
4212 SelectionDAG &DAG,
4213 unsigned Depth = 0) const;
4214
4215 /// Helper wrapper around SimplifyMultipleUseDemandedBits, demanding all
4216 /// elements.
4217 SDValue SimplifyMultipleUseDemandedBits(SDValue Op, const APInt &DemandedBits,
4218 SelectionDAG &DAG,
4219 unsigned Depth = 0) const;
4220
4221 /// Helper wrapper around SimplifyMultipleUseDemandedBits, demanding all
4222 /// bits from only some vector elements.
4223 SDValue SimplifyMultipleUseDemandedVectorElts(SDValue Op,
4224 const APInt &DemandedElts,
4225 SelectionDAG &DAG,
4226 unsigned Depth = 0) const;
4227
4228 /// Look at Vector Op. At this point, we know that only the DemandedElts
4229 /// elements of the result of Op are ever used downstream. If we can use
4230 /// this information to simplify Op, create a new simplified DAG node and
4231 /// return true, storing the original and new nodes in TLO.
4232 /// Otherwise, analyze the expression and return a mask of KnownUndef and
4233 /// KnownZero elements for the expression (used to simplify the caller).
4234 /// The KnownUndef/Zero elements may only be accurate for those bits
4235 /// in the DemandedMask.
4236 /// \p AssumeSingleUse When this parameter is true, this function will
4237 /// attempt to simplify \p Op even if there are multiple uses.
4238 /// Callers are responsible for correctly updating the DAG based on the
4239 /// results of this function, because simply replacing TLO.Old
4240 /// with TLO.New will be incorrect when this parameter is true and TLO.Old
4241 /// has multiple uses.
4242 bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedEltMask,
4243 APInt &KnownUndef, APInt &KnownZero,
4244 TargetLoweringOpt &TLO, unsigned Depth = 0,
4245 bool AssumeSingleUse = false) const;
4246
4247 /// Helper wrapper around SimplifyDemandedVectorElts.
4248 /// Adds Op back to the worklist upon success.
4249 bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedElts,
4250 DAGCombinerInfo &DCI) const;
4251
4252 /// Return true if the target supports simplifying demanded vector elements by
4253 /// converting them to undefs.
4254 virtual bool
4256 const TargetLoweringOpt &TLO) const {
4257 return true;
4258 }
4259
4260 /// Determine which of the bits specified in Mask are known to be either zero
4261 /// or one and return them in the KnownZero/KnownOne bitsets. The DemandedElts
4262 /// argument allows us to only collect the known bits that are shared by the
4263 /// requested vector elements.
4264 virtual void computeKnownBitsForTargetNode(const SDValue Op,
4265 KnownBits &Known,
4266 const APInt &DemandedElts,
4267 const SelectionDAG &DAG,
4268 unsigned Depth = 0) const;
4269
4270 /// Determine which of the bits specified in Mask are known to be either zero
4271 /// or one and return them in the KnownZero/KnownOne bitsets. The DemandedElts
4272 /// argument allows us to only collect the known bits that are shared by the
4273 /// requested vector elements. This is for GISel.
4274 virtual void computeKnownBitsForTargetInstr(GISelValueTracking &Analysis,
4275 Register R, KnownBits &Known,
4276 const APInt &DemandedElts,
4277 const MachineRegisterInfo &MRI,
4278 unsigned Depth = 0) const;
4279
4280 virtual void computeKnownFPClassForTargetInstr(GISelValueTracking &Analysis,
4281 Register R,
4282 KnownFPClass &Known,
4283 const APInt &DemandedElts,
4284 const MachineRegisterInfo &MRI,
4285 unsigned Depth = 0) const;
4286
4287 /// Determine the known alignment for the pointer value \p R. This is can
4288 /// typically be inferred from the number of low known 0 bits. However, for a
4289 /// pointer with a non-integral address space, the alignment value may be
4290 /// independent from the known low bits.
4291 virtual Align computeKnownAlignForTargetInstr(GISelValueTracking &Analysis,
4292 Register R,
4293 const MachineRegisterInfo &MRI,
4294 unsigned Depth = 0) const;
4295
4296 /// Determine which of the bits of FrameIndex \p FIOp are known to be 0.
4297 /// Default implementation computes low bits based on alignment
4298 /// information. This should preserve known bits passed into it.
4299 virtual void computeKnownBitsForFrameIndex(int FIOp,
4300 KnownBits &Known,
4301 const MachineFunction &MF) const;
4302
4303 /// This method can be implemented by targets that want to expose additional
4304 /// information about sign bits to the DAG Combiner. The DemandedElts
4305 /// argument allows us to only collect the minimum sign bits that are shared
4306 /// by the requested vector elements.
4307 virtual unsigned ComputeNumSignBitsForTargetNode(SDValue Op,
4308 const APInt &DemandedElts,
4309 const SelectionDAG &DAG,
4310 unsigned Depth = 0) const;
4311
4312 /// This method can be implemented by targets that want to expose additional
4313 /// information about sign bits to GlobalISel combiners. The DemandedElts
4314 /// argument allows us to only collect the minimum sign bits that are shared
4315 /// by the requested vector elements.
4316 virtual unsigned computeNumSignBitsForTargetInstr(
4317 GISelValueTracking &Analysis, Register R, const APInt &DemandedElts,
4318 const MachineRegisterInfo &MRI, unsigned Depth = 0) const;
4319
4320 /// Attempt to simplify any target nodes based on the demanded vector
4321 /// elements, returning true on success. Otherwise, analyze the expression and
4322 /// return a mask of KnownUndef and KnownZero elements for the expression
4323 /// (used to simplify the caller). The KnownUndef/Zero elements may only be
4324 /// accurate for those bits in the DemandedMask.
4325 virtual bool SimplifyDemandedVectorEltsForTargetNode(
4326 SDValue Op, const APInt &DemandedElts, APInt &KnownUndef,
4327 APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth = 0) const;
4328
4329 /// Attempt to simplify any target nodes based on the demanded bits/elts,
4330 /// returning true on success. Otherwise, analyze the
4331 /// expression and return a mask of KnownOne and KnownZero bits for the
4332 /// expression (used to simplify the caller). The KnownZero/One bits may only
4333 /// be accurate for those bits in the Demanded masks.
4334 virtual bool SimplifyDemandedBitsForTargetNode(SDValue Op,
4335 const APInt &DemandedBits,
4336 const APInt &DemandedElts,
4337 KnownBits &Known,
4338 TargetLoweringOpt &TLO,
4339 unsigned Depth = 0) const;
4340
4341 /// More limited version of SimplifyDemandedBits that can be used to "look
4342 /// through" ops that don't contribute to the DemandedBits/DemandedElts -
4343 /// bitwise ops etc.
4344 virtual SDValue SimplifyMultipleUseDemandedBitsForTargetNode(
4345 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
4346 SelectionDAG &DAG, unsigned Depth) const;
4347
4348 /// Return true if this function can prove that \p Op is never poison
4349 /// and, if \p PoisonOnly is false, does not have undef bits. The DemandedElts
4350 /// argument limits the check to the requested vector elements.
4351 virtual bool isGuaranteedNotToBeUndefOrPoisonForTargetNode(
4352 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
4353 bool PoisonOnly, unsigned Depth) const;
4354
4355 /// Return true if Op can create undef or poison from non-undef & non-poison
4356 /// operands. The DemandedElts argument limits the check to the requested
4357 /// vector elements.
4358 virtual bool
4359 canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts,
4360 const SelectionDAG &DAG, bool PoisonOnly,
4361 bool ConsiderFlags, unsigned Depth) const;
4362
4363 /// Tries to build a legal vector shuffle using the provided parameters
4364 /// or equivalent variations. The Mask argument maybe be modified as the
4365 /// function tries different variations.
4366 /// Returns an empty SDValue if the operation fails.
4367 SDValue buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0,
4369 SelectionDAG &DAG) const;
4370
4371 /// This method returns the constant pool value that will be loaded by LD.
4372 /// NOTE: You must check for implicit extensions of the constant by LD.
4373 virtual const Constant *getTargetConstantFromLoad(LoadSDNode *LD) const;
4374
4375 /// If \p SNaN is false, \returns true if \p Op is known to never be any
4376 /// NaN. If \p sNaN is true, returns if \p Op is known to never be a signaling
4377 /// NaN.
4378 virtual bool isKnownNeverNaNForTargetNode(SDValue Op,
4379 const APInt &DemandedElts,
4380 const SelectionDAG &DAG,
4381 bool SNaN = false,
4382 unsigned Depth = 0) const;
4383
4384 /// Return true if vector \p Op has the same value across all \p DemandedElts,
4385 /// indicating any elements which may be undef in the output \p UndefElts.
4386 virtual bool isSplatValueForTargetNode(SDValue Op, const APInt &DemandedElts,
4387 APInt &UndefElts,
4388 const SelectionDAG &DAG,
4389 unsigned Depth = 0) const;
4390
4391 /// Returns true if the given Opc is considered a canonical constant for the
4392 /// target, which should not be transformed back into a BUILD_VECTOR.
4394 return Op.getOpcode() == ISD::SPLAT_VECTOR ||
4395 Op.getOpcode() == ISD::SPLAT_VECTOR_PARTS;
4396 }
4397
4398 /// Return true if the given select/vselect should be considered canonical and
4399 /// not be transformed. Currently only used for "vselect (not Cond), N1, N2 ->
4400 /// vselect Cond, N2, N1".
4401 virtual bool isTargetCanonicalSelect(SDNode *N) const { return false; }
4402
4404 void *DC; // The DAG Combiner object.
4407
4408 public:
4410
4411 DAGCombinerInfo(SelectionDAG &dag, CombineLevel level, bool cl, void *dc)
4412 : DC(dc), Level(level), CalledByLegalizer(cl), DAG(dag) {}
4413
4414 bool isBeforeLegalize() const { return Level == BeforeLegalizeTypes; }
4416 bool isAfterLegalizeDAG() const { return Level >= AfterLegalizeDAG; }
4419
4420 LLVM_ABI void AddToWorklist(SDNode *N);
4421 LLVM_ABI SDValue CombineTo(SDNode *N, ArrayRef<SDValue> To,
4422 bool AddTo = true);
4423 LLVM_ABI SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true);
4424 LLVM_ABI SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1,
4425 bool AddTo = true);
4426
4427 LLVM_ABI bool recursivelyDeleteUnusedNodes(SDNode *N);
4428
4429 LLVM_ABI void CommitTargetLoweringOpt(const TargetLoweringOpt &TLO);
4430 };
4431
4432 /// Return if the N is a constant or constant vector equal to the true value
4433 /// from getBooleanContents().
4434 bool isConstTrueVal(SDValue N) const;
4435
4436 /// Return if the N is a constant or constant vector equal to the false value
4437 /// from getBooleanContents().
4438 bool isConstFalseVal(SDValue N) const;
4439
4440 /// Return if \p N is a True value when extended to \p VT.
4441 bool isExtendedTrueVal(const ConstantSDNode *N, EVT VT, bool SExt) const;
4442
4443 /// Try to simplify a setcc built with the specified operands and cc. If it is
4444 /// unable to simplify it, return a null SDValue.
4445 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
4446 bool foldBooleans, DAGCombinerInfo &DCI,
4447 const SDLoc &dl) const;
4448
4449 // For targets which wrap address, unwrap for analysis.
4450 virtual SDValue unwrapAddress(SDValue N) const { return N; }
4451
4452 /// Returns true (and the GlobalValue and the offset) if the node is a
4453 /// GlobalAddress + offset.
4454 virtual bool
4455 isGAPlusOffset(SDNode *N, const GlobalValue* &GA, int64_t &Offset) const;
4456
4457 /// This method will be invoked for all target nodes and for any
4458 /// target-independent nodes that the target has registered with invoke it
4459 /// for.
4460 ///
4461 /// The semantics are as follows:
4462 /// Return Value:
4463 /// SDValue.Val == 0 - No change was made
4464 /// SDValue.Val == N - N was replaced, is dead, and is already handled.
4465 /// otherwise - N should be replaced by the returned Operand.
4466 ///
4467 /// In addition, methods provided by DAGCombinerInfo may be used to perform
4468 /// more complex transformations.
4469 ///
4470 virtual SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const;
4471
4472 /// Return true if it is profitable to move this shift by a constant amount
4473 /// through its operand, adjusting any immediate operands as necessary to
4474 /// preserve semantics. This transformation may not be desirable if it
4475 /// disrupts a particularly auspicious target-specific tree (e.g. bitfield
4476 /// extraction in AArch64). By default, it returns true.
4477 ///
4478 /// @param N the shift node
4479 /// @param Level the current DAGCombine legalization level.
4481 CombineLevel Level) const {
4482 SDValue ShiftLHS = N->getOperand(0);
4483 if (!ShiftLHS->hasOneUse())
4484 return false;
4485 if (ShiftLHS.getOpcode() == ISD::SIGN_EXTEND &&
4486 !ShiftLHS.getOperand(0)->hasOneUse())
4487 return false;
4488 return true;
4489 }
4490
4491 /// GlobalISel - return true if it is profitable to move this shift by a
4492 /// constant amount through its operand, adjusting any immediate operands as
4493 /// necessary to preserve semantics. This transformation may not be desirable
4494 /// if it disrupts a particularly auspicious target-specific tree (e.g.
4495 /// bitfield extraction in AArch64). By default, it returns true.
4496 ///
4497 /// @param MI the shift instruction
4498 /// @param IsAfterLegal true if running after legalization.
4500 bool IsAfterLegal) const {
4501 return true;
4502 }
4503
4504 /// GlobalISel - return true if it's profitable to perform the combine:
4505 /// shl ([sza]ext x), y => zext (shl x, y)
4506 virtual bool isDesirableToPullExtFromShl(const MachineInstr &MI) const {
4507 return true;
4508 }
4509
4510 // Return AndOrSETCCFoldKind::{AddAnd, ABS} if its desirable to try and
4511 // optimize LogicOp(SETCC0, SETCC1). An example (what is implemented as of
4512 // writing this) is:
4513 // With C as a power of 2 and C != 0 and C != INT_MIN:
4514 // AddAnd:
4515 // (icmp eq A, C) | (icmp eq A, -C)
4516 // -> (icmp eq and(add(A, C), ~(C + C)), 0)
4517 // (icmp ne A, C) & (icmp ne A, -C)w
4518 // -> (icmp ne and(add(A, C), ~(C + C)), 0)
4519 // ABS:
4520 // (icmp eq A, C) | (icmp eq A, -C)
4521 // -> (icmp eq Abs(A), C)
4522 // (icmp ne A, C) & (icmp ne A, -C)w
4523 // -> (icmp ne Abs(A), C)
4524 //
4525 // @param LogicOp the logic op
4526 // @param SETCC0 the first of the SETCC nodes
4527 // @param SETCC0 the second of the SETCC nodes
4529 const SDNode *LogicOp, const SDNode *SETCC0, const SDNode *SETCC1) const {
4531 }
4532
4533 /// Return true if it is profitable to combine an XOR of a logical shift
4534 /// to create a logical shift of NOT. This transformation may not be desirable
4535 /// if it disrupts a particularly auspicious target-specific tree (e.g.
4536 /// BIC on ARM/AArch64). By default, it returns true.
4537 virtual bool isDesirableToCommuteXorWithShift(const SDNode *N) const {
4538 return true;
4539 }
4540
4541 /// Return true if the target has native support for the specified value type
4542 /// and it is 'desirable' to use the type for the given node type. e.g. On x86
4543 /// i16 is legal, but undesirable since i16 instruction encodings are longer
4544 /// and some i16 instructions are slow.
4545 virtual bool isTypeDesirableForOp(unsigned /*Opc*/, EVT VT) const {
4546 // By default, assume all legal types are desirable.
4547 return isTypeLegal(VT);
4548 }
4549
4550 /// Return true if it is profitable for dag combiner to transform a floating
4551 /// point op of specified opcode to a equivalent op of an integer
4552 /// type. e.g. f32 load -> i32 load can be profitable on ARM.
4553 virtual bool isDesirableToTransformToIntegerOp(unsigned /*Opc*/,
4554 EVT /*VT*/) const {
4555 return false;
4556 }
4557
4558 /// This method query the target whether it is beneficial for dag combiner to
4559 /// promote the specified node. If true, it should return the desired
4560 /// promotion type by reference.
4561 virtual bool IsDesirableToPromoteOp(SDValue /*Op*/, EVT &/*PVT*/) const {
4562 return false;
4563 }
4564
4565 /// Return true if the target supports swifterror attribute. It optimizes
4566 /// loads and stores to reading and writing a specific register.
4567 virtual bool supportSwiftError() const {
4568 return false;
4569 }
4570
4571 /// Return true if the target supports that a subset of CSRs for the given
4572 /// machine function is handled explicitly via copies.
4573 virtual bool supportSplitCSR(MachineFunction *MF) const {
4574 return false;
4575 }
4576
4577 /// Return true if the target supports kcfi operand bundles.
4578 virtual bool supportKCFIBundles() const { return false; }
4579
4580 /// Return true if the target supports ptrauth operand bundles.
4581 virtual bool supportPtrAuthBundles() const { return false; }
4582
4583 /// Perform necessary initialization to handle a subset of CSRs explicitly
4584 /// via copies. This function is called at the beginning of instruction
4585 /// selection.
4586 virtual void initializeSplitCSR(MachineBasicBlock *Entry) const {
4587 llvm_unreachable("Not Implemented");
4588 }
4589
4590 /// Insert explicit copies in entry and exit blocks. We copy a subset of
4591 /// CSRs to virtual registers in the entry block, and copy them back to
4592 /// physical registers in the exit blocks. This function is called at the end
4593 /// of instruction selection.
4595 MachineBasicBlock *Entry,
4596 const SmallVectorImpl<MachineBasicBlock *> &Exits) const {
4597 llvm_unreachable("Not Implemented");
4598 }
4599
4600 /// Return the newly negated expression if the cost is not expensive and
4601 /// set the cost in \p Cost to indicate that if it is cheaper or neutral to
4602 /// do the negation.
4603 virtual SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG,
4604 bool LegalOps, bool OptForSize,
4605 NegatibleCost &Cost,
4606 unsigned Depth = 0) const;
4607
4609 SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize,
4611 unsigned Depth = 0) const {
4613 SDValue Neg =
4614 getNegatedExpression(Op, DAG, LegalOps, OptForSize, Cost, Depth);
4615 if (!Neg)
4616 return SDValue();
4617
4618 if (Cost <= CostThreshold)
4619 return Neg;
4620
4621 // Remove the new created node to avoid the side effect to the DAG.
4622 if (Neg->use_empty())
4623 DAG.RemoveDeadNode(Neg.getNode());
4624 return SDValue();
4625 }
4626
4627 /// This is the helper function to return the newly negated expression only
4628 /// when the cost is cheaper.
4630 bool LegalOps, bool OptForSize,
4631 unsigned Depth = 0) const {
4632 return getCheaperOrNeutralNegatedExpression(Op, DAG, LegalOps, OptForSize,
4634 }
4635
4636 /// This is the helper function to return the newly negated expression if
4637 /// the cost is not expensive.
4639 bool OptForSize, unsigned Depth = 0) const {
4641 return getNegatedExpression(Op, DAG, LegalOps, OptForSize, Cost, Depth);
4642 }
4643
4644 //===--------------------------------------------------------------------===//
4645 // Lowering methods - These methods must be implemented by targets so that
4646 // the SelectionDAGBuilder code knows how to lower these.
4647 //
4648
4649 /// Target-specific splitting of values into parts that fit a register
4650 /// storing a legal type
4652 SelectionDAG & DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
4653 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
4654 return false;
4655 }
4656
4657 /// Allows the target to handle physreg-carried dependency
4658 /// in target-specific way. Used from the ScheduleDAGSDNodes to decide whether
4659 /// to add the edge to the dependency graph.
4660 /// Def - input: Selection DAG node defininfg physical register
4661 /// User - input: Selection DAG node using physical register
4662 /// Op - input: Number of User operand
4663 /// PhysReg - inout: set to the physical register if the edge is
4664 /// necessary, unchanged otherwise
4665 /// Cost - inout: physical register copy cost.
4666 /// Returns 'true' is the edge is necessary, 'false' otherwise
4667 virtual bool checkForPhysRegDependency(SDNode *Def, SDNode *User, unsigned Op,
4668 const TargetRegisterInfo *TRI,
4669 const TargetInstrInfo *TII,
4670 MCRegister &PhysReg, int &Cost) const {
4671 return false;
4672 }
4673
4674 /// Target-specific combining of register parts into its original value
4675 virtual SDValue
4677 const SDValue *Parts, unsigned NumParts,
4678 MVT PartVT, EVT ValueVT,
4679 std::optional<CallingConv::ID> CC) const {
4680 return SDValue();
4681 }
4682
4683 /// This hook must be implemented to lower the incoming (formal) arguments,
4684 /// described by the Ins array, into the specified DAG. The implementation
4685 /// should fill in the InVals array with legal-type argument values, and
4686 /// return the resulting token chain value.
4688 SDValue /*Chain*/, CallingConv::ID /*CallConv*/, bool /*isVarArg*/,
4689 const SmallVectorImpl<ISD::InputArg> & /*Ins*/, const SDLoc & /*dl*/,
4690 SelectionDAG & /*DAG*/, SmallVectorImpl<SDValue> & /*InVals*/) const {
4691 llvm_unreachable("Not Implemented");
4692 }
4693
4694 /// Optional target hook to add target-specific actions when entering EH pad
4695 /// blocks. The implementation should return the resulting token chain value.
4696 virtual SDValue lowerEHPadEntry(SDValue Chain, const SDLoc &DL,
4697 SelectionDAG &DAG) const {
4698 return SDValue();
4699 }
4700
4701 virtual void markLibCallAttributes(MachineFunction *MF, unsigned CC,
4702 ArgListTy &Args) const {}
4703
4704 /// This structure contains the information necessary for lowering
4705 /// pointer-authenticating indirect calls. It is equivalent to the "ptrauth"
4706 /// operand bundle found on the call instruction, if any.
4711
4712 /// This structure contains all information that is necessary for lowering
4713 /// calls. It is passed to TLI::LowerCallTo when the SelectionDAG builder
4714 /// needs to lower a call, and targets will see this struct in their LowerCall
4715 /// implementation.
4718 /// Original unlegalized return type.
4719 Type *OrigRetTy = nullptr;
4720 /// Same as OrigRetTy, or partially legalized for soft float libcalls.
4721 Type *RetTy = nullptr;
4722 bool RetSExt : 1;
4723 bool RetZExt : 1;
4724 bool IsVarArg : 1;
4725 bool IsInReg : 1;
4731 bool NoMerge : 1;
4732
4733 // IsTailCall should be modified by implementations of
4734 // TargetLowering::LowerCall that perform tail call conversions.
4735 bool IsTailCall = false;
4736
4737 // Is Call lowering done post SelectionDAG type legalization.
4739
4740 unsigned NumFixedArgs = -1;
4746 const CallBase *CB = nullptr;
4751 const ConstantInt *CFIType = nullptr;
4753
4754 std::optional<PtrAuthInfo> PAI;
4755
4761
4763 DL = dl;
4764 return *this;
4765 }
4766
4768 Chain = InChain;
4769 return *this;
4770 }
4771
4772 // setCallee with target/module-specific attributes
4774 SDValue Target, ArgListTy &&ArgsList) {
4775 return setLibCallee(CC, ResultType, ResultType, Target,
4776 std::move(ArgsList));
4777 }
4778
4780 Type *OrigResultType, SDValue Target,
4781 ArgListTy &&ArgsList) {
4782 OrigRetTy = OrigResultType;
4783 RetTy = ResultType;
4784 Callee = Target;
4785 CallConv = CC;
4786 NumFixedArgs = ArgsList.size();
4787 Args = std::move(ArgsList);
4788
4789 DAG.getTargetLoweringInfo().markLibCallAttributes(
4790 &(DAG.getMachineFunction()), CC, Args);
4791 return *this;
4792 }
4793
4795 SDValue Target, ArgListTy &&ArgsList,
4796 AttributeSet ResultAttrs = {}) {
4797 RetTy = OrigRetTy = ResultType;
4798 IsInReg = ResultAttrs.hasAttribute(Attribute::InReg);
4799 RetSExt = ResultAttrs.hasAttribute(Attribute::SExt);
4800 RetZExt = ResultAttrs.hasAttribute(Attribute::ZExt);
4801 NoMerge = ResultAttrs.hasAttribute(Attribute::NoMerge);
4802
4803 Callee = Target;
4804 CallConv = CC;
4805 NumFixedArgs = ArgsList.size();
4806 Args = std::move(ArgsList);
4807 return *this;
4808 }
4809
4811 SDValue Target, ArgListTy &&ArgsList,
4812 const CallBase &Call) {
4813 RetTy = OrigRetTy = ResultType;
4814
4815 IsInReg = Call.hasRetAttr(Attribute::InReg);
4817 Call.doesNotReturn() ||
4818 (!isa<InvokeInst>(Call) && isa<UnreachableInst>(Call.getNextNode()));
4819 IsVarArg = FTy->isVarArg();
4820 IsReturnValueUsed = !Call.use_empty();
4821 RetSExt = Call.hasRetAttr(Attribute::SExt);
4822 RetZExt = Call.hasRetAttr(Attribute::ZExt);
4823 NoMerge = Call.hasFnAttr(Attribute::NoMerge);
4824
4825 Callee = Target;
4826
4827 CallConv = Call.getCallingConv();
4828 NumFixedArgs = FTy->getNumParams();
4829 Args = std::move(ArgsList);
4830
4831 CB = &Call;
4832
4833 return *this;
4834 }
4835
4837 IsInReg = Value;
4838 return *this;
4839 }
4840
4843 return *this;
4844 }
4845
4847 IsVarArg = Value;
4848 return *this;
4849 }
4850
4852 IsTailCall = Value;
4853 return *this;
4854 }
4855
4858 return *this;
4859 }
4860
4863 return *this;
4864 }
4865
4867 RetSExt = Value;
4868 return *this;
4869 }
4870
4872 RetZExt = Value;
4873 return *this;
4874 }
4875
4878 return *this;
4879 }
4880
4883 return *this;
4884 }
4885
4887 PAI = Value;
4888 return *this;
4889 }
4890
4893 return *this;
4894 }
4895
4897 CFIType = Type;
4898 return *this;
4899 }
4900
4903 return *this;
4904 }
4905
4907 return Args;
4908 }
4909 };
4910
4911 /// This structure is used to pass arguments to makeLibCall function.
4913 // By passing type list before soften to makeLibCall, the target hook
4914 // shouldExtendTypeInLibCall can get the original type before soften.
4918
4919 bool IsSigned : 1;
4923 bool IsSoften : 1;
4924
4928
4930 IsSigned = Value;
4931 return *this;
4932 }
4933
4936 return *this;
4937 }
4938
4941 return *this;
4942 }
4943
4946 return *this;
4947 }
4948
4950 OpsVTBeforeSoften = OpsVT;
4951 RetVTBeforeSoften = RetVT;
4952 IsSoften = true;
4953 return *this;
4954 }
4955
4956 /// Override the argument type for an operand. Leave the type as null to use
4957 /// the type from the operand's node.
4959 OpsTypeOverrides = OpsTypes;
4960 return *this;
4961 }
4962 };
4963
4964 /// This function lowers an abstract call to a function into an actual call.
4965 /// This returns a pair of operands. The first element is the return value
4966 /// for the function (if RetTy is not VoidTy). The second element is the
4967 /// outgoing token chain. It calls LowerCall to do the actual lowering.
4968 std::pair<SDValue, SDValue> LowerCallTo(CallLoweringInfo &CLI) const;
4969
4970 /// This hook must be implemented to lower calls into the specified
4971 /// DAG. The outgoing arguments to the call are described by the Outs array,
4972 /// and the values to be returned by the call are described by the Ins
4973 /// array. The implementation should fill in the InVals array with legal-type
4974 /// return values from the call, and return the resulting token chain value.
4975 virtual SDValue
4977 SmallVectorImpl<SDValue> &/*InVals*/) const {
4978 llvm_unreachable("Not Implemented");
4979 }
4980
4981 /// Target-specific cleanup for formal ByVal parameters.
4982 virtual void HandleByVal(CCState *, unsigned &, Align) const {}
4983
4984 /// This hook should be implemented to check whether the return values
4985 /// described by the Outs array can fit into the return registers. If false
4986 /// is returned, an sret-demotion is performed.
4987 virtual bool CanLowerReturn(CallingConv::ID /*CallConv*/,
4988 MachineFunction &/*MF*/, bool /*isVarArg*/,
4989 const SmallVectorImpl<ISD::OutputArg> &/*Outs*/,
4990 LLVMContext &/*Context*/, const Type *RetTy) const
4991 {
4992 // Return true by default to get preexisting behavior.
4993 return true;
4994 }
4995
4996 /// This hook must be implemented to lower outgoing return values, described
4997 /// by the Outs array, into the specified DAG. The implementation should
4998 /// return the resulting token chain value.
4999 virtual SDValue LowerReturn(SDValue /*Chain*/, CallingConv::ID /*CallConv*/,
5000 bool /*isVarArg*/,
5001 const SmallVectorImpl<ISD::OutputArg> & /*Outs*/,
5002 const SmallVectorImpl<SDValue> & /*OutVals*/,
5003 const SDLoc & /*dl*/,
5004 SelectionDAG & /*DAG*/) const {
5005 llvm_unreachable("Not Implemented");
5006 }
5007
5008 /// Return true if result of the specified node is used by a return node
5009 /// only. It also compute and return the input chain for the tail call.
5010 ///
5011 /// This is used to determine whether it is possible to codegen a libcall as
5012 /// tail call at legalization time.
5013 virtual bool isUsedByReturnOnly(SDNode *, SDValue &/*Chain*/) const {
5014 return false;
5015 }
5016
5017 /// Return true if the target may be able emit the call instruction as a tail
5018 /// call. This is used by optimization passes to determine if it's profitable
5019 /// to duplicate return instructions to enable tailcall optimization.
5020 virtual bool mayBeEmittedAsTailCall(const CallInst *) const {
5021 return false;
5022 }
5023
5024 /// Return the register ID of the name passed in. Used by named register
5025 /// global variables extension. There is no target-independent behaviour
5026 /// so the default action is to bail.
5027 virtual Register getRegisterByName(const char* RegName, LLT Ty,
5028 const MachineFunction &MF) const {
5029 report_fatal_error("Named registers not implemented for this target");
5030 }
5031
5032 /// Return the type that should be used to zero or sign extend a
5033 /// zeroext/signext integer return value. FIXME: Some C calling conventions
5034 /// require the return type to be promoted, but this is not true all the time,
5035 /// e.g. i1/i8/i16 on x86/x86_64. It is also not necessary for non-C calling
5036 /// conventions. The frontend should handle this and include all of the
5037 /// necessary information.
5039 ISD::NodeType /*ExtendKind*/) const {
5040 EVT MinVT = getRegisterType(MVT::i32);
5041 return VT.bitsLT(MinVT) ? MinVT : VT;
5042 }
5043
5044 /// For some targets, an LLVM struct type must be broken down into multiple
5045 /// simple types, but the calling convention specifies that the entire struct
5046 /// must be passed in a block of consecutive registers.
5047 virtual bool
5049 bool isVarArg,
5050 const DataLayout &DL) const {
5051 return false;
5052 }
5053
5054 /// For most targets, an LLVM type must be broken down into multiple
5055 /// smaller types. Usually the halves are ordered according to the endianness
5056 /// but for some platform that would break. So this method will default to
5057 /// matching the endianness but can be overridden.
5058 virtual bool
5060 return DL.isLittleEndian();
5061 }
5062
5063 /// Returns a 0 terminated array of registers that can be safely used as
5064 /// scratch registers.
5066 return nullptr;
5067 }
5068
5069 /// Returns a 0 terminated array of rounding control registers that can be
5070 /// attached into strict FP call.
5074
5075 /// This callback is used to prepare for a volatile or atomic load.
5076 /// It takes a chain node as input and returns the chain for the load itself.
5077 ///
5078 /// Having a callback like this is necessary for targets like SystemZ,
5079 /// which allows a CPU to reuse the result of a previous load indefinitely,
5080 /// even if a cache-coherent store is performed by another CPU. The default
5081 /// implementation does nothing.
5083 SelectionDAG &DAG) const {
5084 return Chain;
5085 }
5086
5087 /// This callback is invoked by the type legalizer to legalize nodes with an
5088 /// illegal operand type but legal result types. It replaces the
5089 /// LowerOperation callback in the type Legalizer. The reason we can not do
5090 /// away with LowerOperation entirely is that LegalizeDAG isn't yet ready to
5091 /// use this callback.
5092 ///
5093 /// TODO: Consider merging with ReplaceNodeResults.
5094 ///
5095 /// The target places new result values for the node in Results (their number
5096 /// and types must exactly match those of the original return values of
5097 /// the node), or leaves Results empty, which indicates that the node is not
5098 /// to be custom lowered after all.
5099 /// The default implementation calls LowerOperation.
5100 virtual void LowerOperationWrapper(SDNode *N,
5102 SelectionDAG &DAG) const;
5103
5104 /// This callback is invoked for operations that are unsupported by the
5105 /// target, which are registered to use 'custom' lowering, and whose defined
5106 /// values are all legal. If the target has no operations that require custom
5107 /// lowering, it need not implement this. The default implementation of this
5108 /// aborts.
5109 virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const;
5110
5111 /// This callback is invoked when a node result type is illegal for the
5112 /// target, and the operation was registered to use 'custom' lowering for that
5113 /// result type. The target places new result values for the node in Results
5114 /// (their number and types must exactly match those of the original return
5115 /// values of the node), or leaves Results empty, which indicates that the
5116 /// node is not to be custom lowered after all.
5117 ///
5118 /// If the target has no operations that require custom lowering, it need not
5119 /// implement this. The default implementation aborts.
5120 virtual void ReplaceNodeResults(SDNode * /*N*/,
5121 SmallVectorImpl<SDValue> &/*Results*/,
5122 SelectionDAG &/*DAG*/) const {
5123 llvm_unreachable("ReplaceNodeResults not implemented for this target!");
5124 }
5125
5126 /// This method returns the name of a target specific DAG node.
5127 virtual const char *getTargetNodeName(unsigned Opcode) const;
5128
5129 /// This method returns a target specific FastISel object, or null if the
5130 /// target does not support "fast" ISel.
5132 const TargetLibraryInfo *) const {
5133 return nullptr;
5134 }
5135
5136 //===--------------------------------------------------------------------===//
5137 // Inline Asm Support hooks
5138 //
5139
5141 C_Register, // Constraint represents specific register(s).
5142 C_RegisterClass, // Constraint represents any of register(s) in class.
5143 C_Memory, // Memory constraint.
5144 C_Address, // Address constraint.
5145 C_Immediate, // Requires an immediate.
5146 C_Other, // Something else.
5147 C_Unknown // Unsupported constraint.
5148 };
5149
5151 // Generic weights.
5152 CW_Invalid = -1, // No match.
5153 CW_Okay = 0, // Acceptable.
5154 CW_Good = 1, // Good weight.
5155 CW_Better = 2, // Better weight.
5156 CW_Best = 3, // Best weight.
5157
5158 // Well-known weights.
5159 CW_SpecificReg = CW_Okay, // Specific register operands.
5160 CW_Register = CW_Good, // Register operands.
5161 CW_Memory = CW_Better, // Memory operands.
5162 CW_Constant = CW_Best, // Constant operand.
5163 CW_Default = CW_Okay // Default or don't know type.
5164 };
5165
5166 /// This contains information for each constraint that we are lowering.
5168 /// This contains the actual string for the code, like "m". TargetLowering
5169 /// picks the 'best' code from ConstraintInfo::Codes that most closely
5170 /// matches the operand.
5171 std::string ConstraintCode;
5172
5173 /// Information about the constraint code, e.g. Register, RegisterClass,
5174 /// Memory, Other, Unknown.
5176
5177 /// If this is the result output operand or a clobber, this is null,
5178 /// otherwise it is the incoming operand to the CallInst. This gets
5179 /// modified as the asm is processed.
5181
5182 /// The ValueType for the operand value.
5183 MVT ConstraintVT = MVT::Other;
5184
5185 /// Copy constructor for copying from a ConstraintInfo.
5188
5189 /// Return true of this is an input operand that is a matching constraint
5190 /// like "4".
5191 LLVM_ABI bool isMatchingInputConstraint() const;
5192
5193 /// If this is an input matching constraint, this method returns the output
5194 /// operand it matches.
5195 LLVM_ABI unsigned getMatchedOperand() const;
5196 };
5197
5198 using AsmOperandInfoVector = std::vector<AsmOperandInfo>;
5199
5200 /// Split up the constraint string from the inline assembly value into the
5201 /// specific constraints and their prefixes, and also tie in the associated
5202 /// operand values. If this returns an empty vector, and if the constraint
5203 /// string itself isn't empty, there was an error parsing.
5205 const TargetRegisterInfo *TRI,
5206 const CallBase &Call) const;
5207
5208 /// Examine constraint type and operand type and determine a weight value.
5209 /// The operand object must already have been set up with the operand type.
5211 AsmOperandInfo &info, int maIndex) const;
5212
5213 /// Examine constraint string and operand type and determine a weight value.
5214 /// The operand object must already have been set up with the operand type.
5216 AsmOperandInfo &info, const char *constraint) const;
5217
5218 /// Determines the constraint code and constraint type to use for the specific
5219 /// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.
5220 /// If the actual operand being passed in is available, it can be passed in as
5221 /// Op, otherwise an empty SDValue can be passed.
5222 virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo,
5223 SDValue Op,
5224 SelectionDAG *DAG = nullptr) const;
5225
5226 /// Given a constraint, return the type of constraint it is for this target.
5227 virtual ConstraintType getConstraintType(StringRef Constraint) const;
5228
5229 using ConstraintPair = std::pair<StringRef, TargetLowering::ConstraintType>;
5231 /// Given an OpInfo with list of constraints codes as strings, return a
5232 /// sorted Vector of pairs of constraint codes and their types in priority of
5233 /// what we'd prefer to lower them as. This may contain immediates that
5234 /// cannot be lowered, but it is meant to be a machine agnostic order of
5235 /// preferences.
5237
5238 /// Given a physical register constraint (e.g. {edx}), return the register
5239 /// number and the register class for the register.
5240 ///
5241 /// Given a register class constraint, like 'r', if this corresponds directly
5242 /// to an LLVM register class, return a register of 0 and the register class
5243 /// pointer.
5244 ///
5245 /// This should only be used for C_Register constraints. On error, this
5246 /// returns a register number of 0 and a null register class pointer.
5247 virtual std::pair<unsigned, const TargetRegisterClass *>
5249 StringRef Constraint, MVT VT) const;
5250
5252 getInlineAsmMemConstraint(StringRef ConstraintCode) const {
5253 if (ConstraintCode == "m")
5255 if (ConstraintCode == "o")
5257 if (ConstraintCode == "X")
5259 if (ConstraintCode == "p")
5262 }
5263
5264 /// Try to replace an X constraint, which matches anything, with another that
5265 /// has more specific requirements based on the type of the corresponding
5266 /// operand. This returns null if there is no replacement to make.
5267 virtual const char *LowerXConstraint(EVT ConstraintVT) const;
5268
5269 /// Lower the specified operand into the Ops vector. If it is invalid, don't
5270 /// add anything to Ops.
5271 virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint,
5272 std::vector<SDValue> &Ops,
5273 SelectionDAG &DAG) const;
5274
5275 // Lower custom output constraints. If invalid, return SDValue().
5276 virtual SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Glue,
5277 const SDLoc &DL,
5278 const AsmOperandInfo &OpInfo,
5279 SelectionDAG &DAG) const;
5280
5281 // Targets may override this function to collect operands from the CallInst
5282 // and for example, lower them into the SelectionDAG operands.
5283 virtual void CollectTargetIntrinsicOperands(const CallInst &I,
5285 SelectionDAG &DAG) const;
5286
5287 //===--------------------------------------------------------------------===//
5288 // Div utility functions
5289 //
5290
5291 SDValue BuildSDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization,
5292 bool IsAfterLegalTypes,
5293 SmallVectorImpl<SDNode *> &Created) const;
5294 SDValue BuildUDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization,
5295 bool IsAfterLegalTypes,
5296 SmallVectorImpl<SDNode *> &Created) const;
5297 // Build sdiv by power-of-2 with conditional move instructions
5298 SDValue buildSDIVPow2WithCMov(SDNode *N, const APInt &Divisor,
5299 SelectionDAG &DAG,
5300 SmallVectorImpl<SDNode *> &Created) const;
5301
5302 /// Targets may override this function to provide custom SDIV lowering for
5303 /// power-of-2 denominators. If the target returns an empty SDValue, LLVM
5304 /// assumes SDIV is expensive and replaces it with a series of other integer
5305 /// operations.
5306 virtual SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor,
5307 SelectionDAG &DAG,
5308 SmallVectorImpl<SDNode *> &Created) const;
5309
5310 /// Targets may override this function to provide custom SREM lowering for
5311 /// power-of-2 denominators. If the target returns an empty SDValue, LLVM
5312 /// assumes SREM is expensive and replaces it with a series of other integer
5313 /// operations.
5314 virtual SDValue BuildSREMPow2(SDNode *N, const APInt &Divisor,
5315 SelectionDAG &DAG,
5316 SmallVectorImpl<SDNode *> &Created) const;
5317
5318 /// Indicate whether this target prefers to combine FDIVs with the same
5319 /// divisor. If the transform should never be done, return zero. If the
5320 /// transform should be done, return the minimum number of divisor uses
5321 /// that must exist.
5322 virtual unsigned combineRepeatedFPDivisors() const {
5323 return 0;
5324 }
5325
5326 /// Hooks for building estimates in place of slower divisions and square
5327 /// roots.
5328
5329 /// Return either a square root or its reciprocal estimate value for the input
5330 /// operand.
5331 /// \p Enabled is a ReciprocalEstimate enum with value either 'Unspecified' or
5332 /// 'Enabled' as set by a potential default override attribute.
5333 /// If \p RefinementSteps is 'Unspecified', the number of Newton-Raphson
5334 /// refinement iterations required to generate a sufficient (though not
5335 /// necessarily IEEE-754 compliant) estimate is returned in that parameter.
5336 /// The boolean UseOneConstNR output is used to select a Newton-Raphson
5337 /// algorithm implementation that uses either one or two constants.
5338 /// The boolean Reciprocal is used to select whether the estimate is for the
5339 /// square root of the input operand or the reciprocal of its square root.
5340 /// A target may choose to implement its own refinement within this function.
5341 /// If that's true, then return '0' as the number of RefinementSteps to avoid
5342 /// any further refinement of the estimate.
5343 /// An empty SDValue return means no estimate sequence can be created.
5345 int Enabled, int &RefinementSteps,
5346 bool &UseOneConstNR, bool Reciprocal) const {
5347 return SDValue();
5348 }
5349
5350 /// Try to convert the fminnum/fmaxnum to a compare/select sequence. This is
5351 /// required for correctness since InstCombine might have canonicalized a
5352 /// fcmp+select sequence to a FMINNUM/FMAXNUM intrinsic. If we were to fall
5353 /// through to the default expansion/soften to libcall, we might introduce a
5354 /// link-time dependency on libm into a file that originally did not have one.
5355 SDValue createSelectForFMINNUM_FMAXNUM(SDNode *Node, SelectionDAG &DAG) const;
5356
5357 /// Return a reciprocal estimate value for the input operand.
5358 /// \p Enabled is a ReciprocalEstimate enum with value either 'Unspecified' or
5359 /// 'Enabled' as set by a potential default override attribute.
5360 /// If \p RefinementSteps is 'Unspecified', the number of Newton-Raphson
5361 /// refinement iterations required to generate a sufficient (though not
5362 /// necessarily IEEE-754 compliant) estimate is returned in that parameter.
5363 /// A target may choose to implement its own refinement within this function.
5364 /// If that's true, then return '0' as the number of RefinementSteps to avoid
5365 /// any further refinement of the estimate.
5366 /// An empty SDValue return means no estimate sequence can be created.
5368 int Enabled, int &RefinementSteps) const {
5369 return SDValue();
5370 }
5371
5372 /// Return a target-dependent comparison result if the input operand is
5373 /// suitable for use with a square root estimate calculation. For example, the
5374 /// comparison may check if the operand is NAN, INF, zero, normal, etc. The
5375 /// result should be used as the condition operand for a select or branch.
5376 virtual SDValue getSqrtInputTest(SDValue Operand, SelectionDAG &DAG,
5377 const DenormalMode &Mode) const;
5378
5379 /// Return a target-dependent result if the input operand is not suitable for
5380 /// use with a square root estimate calculation.
5382 SelectionDAG &DAG) const {
5383 return DAG.getConstantFP(0.0, SDLoc(Operand), Operand.getValueType());
5384 }
5385
5386 //===--------------------------------------------------------------------===//
5387 // Legalization utility functions
5388 //
5389
5390 /// Expand a MUL or [US]MUL_LOHI of n-bit values into two or four nodes,
5391 /// respectively, each computing an n/2-bit part of the result.
5392 /// \param Result A vector that will be filled with the parts of the result
5393 /// in little-endian order.
5394 /// \param LL Low bits of the LHS of the MUL. You can use this parameter
5395 /// if you want to control how low bits are extracted from the LHS.
5396 /// \param LH High bits of the LHS of the MUL. See LL for meaning.
5397 /// \param RL Low bits of the RHS of the MUL. See LL for meaning
5398 /// \param RH High bits of the RHS of the MUL. See LL for meaning.
5399 /// \returns true if the node has been expanded, false if it has not
5400 bool expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl, SDValue LHS,
5401 SDValue RHS, SmallVectorImpl<SDValue> &Result, EVT HiLoVT,
5402 SelectionDAG &DAG, MulExpansionKind Kind,
5403 SDValue LL = SDValue(), SDValue LH = SDValue(),
5404 SDValue RL = SDValue(), SDValue RH = SDValue()) const;
5405
5406 /// Expand a MUL into two nodes. One that computes the high bits of
5407 /// the result and one that computes the low bits.
5408 /// \param HiLoVT The value type to use for the Lo and Hi nodes.
5409 /// \param LL Low bits of the LHS of the MUL. You can use this parameter
5410 /// if you want to control how low bits are extracted from the LHS.
5411 /// \param LH High bits of the LHS of the MUL. See LL for meaning.
5412 /// \param RL Low bits of the RHS of the MUL. See LL for meaning
5413 /// \param RH High bits of the RHS of the MUL. See LL for meaning.
5414 /// \returns true if the node has been expanded. false if it has not
5415 bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT,
5416 SelectionDAG &DAG, MulExpansionKind Kind,
5417 SDValue LL = SDValue(), SDValue LH = SDValue(),
5418 SDValue RL = SDValue(), SDValue RH = SDValue()) const;
5419
5420 /// Attempt to expand an n-bit div/rem/divrem by constant using a n/2-bit
5421 /// urem by constant and other arithmetic ops. The n/2-bit urem by constant
5422 /// will be expanded by DAGCombiner. This is not possible for all constant
5423 /// divisors.
5424 /// \param N Node to expand
5425 /// \param Result A vector that will be filled with the lo and high parts of
5426 /// the results. For *DIVREM, this will be the quotient parts followed
5427 /// by the remainder parts.
5428 /// \param HiLoVT The value type to use for the Lo and Hi parts. Should be
5429 /// half of VT.
5430 /// \param LL Low bits of the LHS of the operation. You can use this
5431 /// parameter if you want to control how low bits are extracted from
5432 /// the LHS.
5433 /// \param LH High bits of the LHS of the operation. See LL for meaning.
5434 /// \returns true if the node has been expanded, false if it has not.
5435 bool expandDIVREMByConstant(SDNode *N, SmallVectorImpl<SDValue> &Result,
5436 EVT HiLoVT, SelectionDAG &DAG,
5437 SDValue LL = SDValue(),
5438 SDValue LH = SDValue()) const;
5439
5440 /// Expand funnel shift.
5441 /// \param N Node to expand
5442 /// \returns The expansion if successful, SDValue() otherwise
5443 SDValue expandFunnelShift(SDNode *N, SelectionDAG &DAG) const;
5444
5445 /// Expand rotations.
5446 /// \param N Node to expand
5447 /// \param AllowVectorOps expand vector rotate, this should only be performed
5448 /// if the legalization is happening outside of LegalizeVectorOps
5449 /// \returns The expansion if successful, SDValue() otherwise
5450 SDValue expandROT(SDNode *N, bool AllowVectorOps, SelectionDAG &DAG) const;
5451
5452 /// Expand shift-by-parts.
5453 /// \param N Node to expand
5454 /// \param Lo lower-output-part after conversion
5455 /// \param Hi upper-output-part after conversion
5456 void expandShiftParts(SDNode *N, SDValue &Lo, SDValue &Hi,
5457 SelectionDAG &DAG) const;
5458
5459 /// Expand float(f32) to SINT(i64) conversion
5460 /// \param N Node to expand
5461 /// \param Result output after conversion
5462 /// \returns True, if the expansion was successful, false otherwise
5463 bool expandFP_TO_SINT(SDNode *N, SDValue &Result, SelectionDAG &DAG) const;
5464
5465 /// Expand float to UINT conversion
5466 /// \param N Node to expand
5467 /// \param Result output after conversion
5468 /// \param Chain output chain after conversion
5469 /// \returns True, if the expansion was successful, false otherwise
5470 bool expandFP_TO_UINT(SDNode *N, SDValue &Result, SDValue &Chain,
5471 SelectionDAG &DAG) const;
5472
5473 /// Expand UINT(i64) to double(f64) conversion
5474 /// \param N Node to expand
5475 /// \param Result output after conversion
5476 /// \param Chain output chain after conversion
5477 /// \returns True, if the expansion was successful, false otherwise
5478 bool expandUINT_TO_FP(SDNode *N, SDValue &Result, SDValue &Chain,
5479 SelectionDAG &DAG) const;
5480
5481 /// Expand fminnum/fmaxnum into fminnum_ieee/fmaxnum_ieee with quieted inputs.
5482 SDValue expandFMINNUM_FMAXNUM(SDNode *N, SelectionDAG &DAG) const;
5483
5484 /// Expand fminimum/fmaximum into multiple comparison with selects.
5485 SDValue expandFMINIMUM_FMAXIMUM(SDNode *N, SelectionDAG &DAG) const;
5486
5487 /// Expand fminimumnum/fmaximumnum into multiple comparison with selects.
5488 SDValue expandFMINIMUMNUM_FMAXIMUMNUM(SDNode *N, SelectionDAG &DAG) const;
5489
5490 /// Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
5491 /// \param N Node to expand
5492 /// \returns The expansion result
5493 SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const;
5494
5495 /// Truncate Op to ResultVT. If the result is exact, leave it alone. If it is
5496 /// not exact, force the result to be odd.
5497 /// \param ResultVT The type of result.
5498 /// \param Op The value to round.
5499 /// \returns The expansion result
5500 SDValue expandRoundInexactToOdd(EVT ResultVT, SDValue Op, const SDLoc &DL,
5501 SelectionDAG &DAG) const;
5502
5503 /// Expand round(fp) to fp conversion
5504 /// \param N Node to expand
5505 /// \returns The expansion result
5506 SDValue expandFP_ROUND(SDNode *Node, SelectionDAG &DAG) const;
5507
5508 /// Expand check for floating point class.
5509 /// \param ResultVT The type of intrinsic call result.
5510 /// \param Op The tested value.
5511 /// \param Test The test to perform.
5512 /// \param Flags The optimization flags.
5513 /// \returns The expansion result or SDValue() if it fails.
5514 SDValue expandIS_FPCLASS(EVT ResultVT, SDValue Op, FPClassTest Test,
5515 SDNodeFlags Flags, const SDLoc &DL,
5516 SelectionDAG &DAG) const;
5517
5518 /// Expand CTPOP nodes. Expands vector/scalar CTPOP nodes,
5519 /// vector nodes can only succeed if all operations are legal/custom.
5520 /// \param N Node to expand
5521 /// \returns The expansion result or SDValue() if it fails.
5522 SDValue expandCTPOP(SDNode *N, SelectionDAG &DAG) const;
5523
5524 /// Expand VP_CTPOP nodes.
5525 /// \returns The expansion result or SDValue() if it fails.
5526 SDValue expandVPCTPOP(SDNode *N, SelectionDAG &DAG) const;
5527
5528 /// Expand CTLZ/CTLZ_ZERO_UNDEF nodes. Expands vector/scalar CTLZ nodes,
5529 /// vector nodes can only succeed if all operations are legal/custom.
5530 /// \param N Node to expand
5531 /// \returns The expansion result or SDValue() if it fails.
5532 SDValue expandCTLZ(SDNode *N, SelectionDAG &DAG) const;
5533
5534 /// Expand VP_CTLZ/VP_CTLZ_ZERO_UNDEF nodes.
5535 /// \param N Node to expand
5536 /// \returns The expansion result or SDValue() if it fails.
5537 SDValue expandVPCTLZ(SDNode *N, SelectionDAG &DAG) const;
5538
5539 /// Expand CTTZ via Table Lookup.
5540 /// \param N Node to expand
5541 /// \returns The expansion result or SDValue() if it fails.
5542 SDValue CTTZTableLookup(SDNode *N, SelectionDAG &DAG, const SDLoc &DL, EVT VT,
5543 SDValue Op, unsigned NumBitsPerElt) const;
5544
5545 /// Expand CTTZ/CTTZ_ZERO_UNDEF nodes. Expands vector/scalar CTTZ nodes,
5546 /// vector nodes can only succeed if all operations are legal/custom.
5547 /// \param N Node to expand
5548 /// \returns The expansion result or SDValue() if it fails.
5549 SDValue expandCTTZ(SDNode *N, SelectionDAG &DAG) const;
5550
5551 /// Expand VP_CTTZ/VP_CTTZ_ZERO_UNDEF nodes.
5552 /// \param N Node to expand
5553 /// \returns The expansion result or SDValue() if it fails.
5554 SDValue expandVPCTTZ(SDNode *N, SelectionDAG &DAG) const;
5555
5556 /// Expand VP_CTTZ_ELTS/VP_CTTZ_ELTS_ZERO_UNDEF nodes.
5557 /// \param N Node to expand
5558 /// \returns The expansion result or SDValue() if it fails.
5559 SDValue expandVPCTTZElements(SDNode *N, SelectionDAG &DAG) const;
5560
5561 /// Expand VECTOR_FIND_LAST_ACTIVE nodes
5562 /// \param N Node to expand
5563 /// \returns The expansion result or SDValue() if it fails.
5564 SDValue expandVectorFindLastActive(SDNode *N, SelectionDAG &DAG) const;
5565
5566 /// Expand ABS nodes. Expands vector/scalar ABS nodes,
5567 /// vector nodes can only succeed if all operations are legal/custom.
5568 /// (ABS x) -> (XOR (ADD x, (SRA x, type_size)), (SRA x, type_size))
5569 /// \param N Node to expand
5570 /// \param IsNegative indicate negated abs
5571 /// \returns The expansion result or SDValue() if it fails.
5572 SDValue expandABS(SDNode *N, SelectionDAG &DAG,
5573 bool IsNegative = false) const;
5574
5575 /// Expand ABDS/ABDU nodes. Expands vector/scalar ABDS/ABDU nodes.
5576 /// \param N Node to expand
5577 /// \returns The expansion result or SDValue() if it fails.
5578 SDValue expandABD(SDNode *N, SelectionDAG &DAG) const;
5579
5580 /// Expand vector/scalar AVGCEILS/AVGCEILU/AVGFLOORS/AVGFLOORU nodes.
5581 /// \param N Node to expand
5582 /// \returns The expansion result or SDValue() if it fails.
5583 SDValue expandAVG(SDNode *N, SelectionDAG &DAG) const;
5584
5585 /// Expand BSWAP nodes. Expands scalar/vector BSWAP nodes with i16/i32/i64
5586 /// scalar types. Returns SDValue() if expand fails.
5587 /// \param N Node to expand
5588 /// \returns The expansion result or SDValue() if it fails.
5589 SDValue expandBSWAP(SDNode *N, SelectionDAG &DAG) const;
5590
5591 /// Expand VP_BSWAP nodes. Expands VP_BSWAP nodes with
5592 /// i16/i32/i64 scalar types. Returns SDValue() if expand fails. \param N Node
5593 /// to expand \returns The expansion result or SDValue() if it fails.
5594 SDValue expandVPBSWAP(SDNode *N, SelectionDAG &DAG) const;
5595
5596 /// Expand BITREVERSE nodes. Expands scalar/vector BITREVERSE nodes.
5597 /// Returns SDValue() if expand fails.
5598 /// \param N Node to expand
5599 /// \returns The expansion result or SDValue() if it fails.
5600 SDValue expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const;
5601
5602 /// Expand VP_BITREVERSE nodes. Expands VP_BITREVERSE nodes with
5603 /// i8/i16/i32/i64 scalar types. \param N Node to expand \returns The
5604 /// expansion result or SDValue() if it fails.
5605 SDValue expandVPBITREVERSE(SDNode *N, SelectionDAG &DAG) const;
5606
5607 /// Turn load of vector type into a load of the individual elements.
5608 /// \param LD load to expand
5609 /// \returns BUILD_VECTOR and TokenFactor nodes.
5610 std::pair<SDValue, SDValue> scalarizeVectorLoad(LoadSDNode *LD,
5611 SelectionDAG &DAG) const;
5612
5613 // Turn a store of a vector type into stores of the individual elements.
5614 /// \param ST Store with a vector value type
5615 /// \returns TokenFactor of the individual store chains.
5617
5618 /// Expands an unaligned load to 2 half-size loads for an integer, and
5619 /// possibly more for vectors.
5620 std::pair<SDValue, SDValue> expandUnalignedLoad(LoadSDNode *LD,
5621 SelectionDAG &DAG) const;
5622
5623 /// Expands an unaligned store to 2 half-size stores for integer values, and
5624 /// possibly more for vectors.
5625 SDValue expandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG) const;
5626
5627 /// Increments memory address \p Addr according to the type of the value
5628 /// \p DataVT that should be stored. If the data is stored in compressed
5629 /// form, the memory address should be incremented according to the number of
5630 /// the stored elements. This number is equal to the number of '1's bits
5631 /// in the \p Mask.
5632 /// \p DataVT is a vector type. \p Mask is a vector value.
5633 /// \p DataVT and \p Mask have the same number of vector elements.
5634 SDValue IncrementMemoryAddress(SDValue Addr, SDValue Mask, const SDLoc &DL,
5635 EVT DataVT, SelectionDAG &DAG,
5636 bool IsCompressedMemory) const;
5637
5638 /// Get a pointer to vector element \p Idx located in memory for a vector of
5639 /// type \p VecVT starting at a base address of \p VecPtr. If \p Idx is out of
5640 /// bounds the returned pointer is unspecified, but will be within the vector
5641 /// bounds.
5642 SDValue getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT,
5643 SDValue Index) const;
5644
5645 /// Get a pointer to a sub-vector of type \p SubVecVT at index \p Idx located
5646 /// in memory for a vector of type \p VecVT starting at a base address of
5647 /// \p VecPtr. If \p Idx plus the size of \p SubVecVT is out of bounds the
5648 /// returned pointer is unspecified, but the value returned will be such that
5649 /// the entire subvector would be within the vector bounds.
5650 SDValue getVectorSubVecPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT,
5651 EVT SubVecVT, SDValue Index) const;
5652
5653 /// Method for building the DAG expansion of ISD::[US][MIN|MAX]. This
5654 /// method accepts integers as its arguments.
5655 SDValue expandIntMINMAX(SDNode *Node, SelectionDAG &DAG) const;
5656
5657 /// Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT. This
5658 /// method accepts integers as its arguments.
5659 SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const;
5660
5661 /// Method for building the DAG expansion of ISD::[US]CMP. This
5662 /// method accepts integers as its arguments
5663 SDValue expandCMP(SDNode *Node, SelectionDAG &DAG) const;
5664
5665 /// Method for building the DAG expansion of ISD::[US]SHLSAT. This
5666 /// method accepts integers as its arguments.
5667 SDValue expandShlSat(SDNode *Node, SelectionDAG &DAG) const;
5668
5669 /// Method for building the DAG expansion of ISD::[U|S]MULFIX[SAT]. This
5670 /// method accepts integers as its arguments.
5671 SDValue expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const;
5672
5673 /// Method for building the DAG expansion of ISD::[US]DIVFIX[SAT]. This
5674 /// method accepts integers as its arguments.
5675 /// Note: This method may fail if the division could not be performed
5676 /// within the type. Clients must retry with a wider type if this happens.
5677 SDValue expandFixedPointDiv(unsigned Opcode, const SDLoc &dl,
5679 unsigned Scale, SelectionDAG &DAG) const;
5680
5681 /// Method for building the DAG expansion of ISD::U(ADD|SUB)O. Expansion
5682 /// always suceeds and populates the Result and Overflow arguments.
5683 void expandUADDSUBO(SDNode *Node, SDValue &Result, SDValue &Overflow,
5684 SelectionDAG &DAG) const;
5685
5686 /// Method for building the DAG expansion of ISD::S(ADD|SUB)O. Expansion
5687 /// always suceeds and populates the Result and Overflow arguments.
5688 void expandSADDSUBO(SDNode *Node, SDValue &Result, SDValue &Overflow,
5689 SelectionDAG &DAG) const;
5690
5691 /// Method for building the DAG expansion of ISD::[US]MULO. Returns whether
5692 /// expansion was successful and populates the Result and Overflow arguments.
5693 bool expandMULO(SDNode *Node, SDValue &Result, SDValue &Overflow,
5694 SelectionDAG &DAG) const;
5695
5696 /// Calculate the product twice the width of LHS and RHS. If HiLHS/HiRHS are
5697 /// non-null they will be included in the multiplication. The expansion works
5698 /// by splitting the 2 inputs into 4 pieces that we can multiply and add
5699 /// together without neding MULH or MUL_LOHI.
5700 void forceExpandMultiply(SelectionDAG &DAG, const SDLoc &dl, bool Signed,
5702 SDValue HiLHS = SDValue(),
5703 SDValue HiRHS = SDValue()) const;
5704
5705 /// Calculate full product of LHS and RHS either via a libcall or through
5706 /// brute force expansion of the multiplication. The expansion works by
5707 /// splitting the 2 inputs into 4 pieces that we can multiply and add together
5708 /// without needing MULH or MUL_LOHI.
5709 void forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl, bool Signed,
5710 const SDValue LHS, const SDValue RHS, SDValue &Lo,
5711 SDValue &Hi) const;
5712
5713 /// Expand a VECREDUCE_* into an explicit calculation. If Count is specified,
5714 /// only the first Count elements of the vector are used.
5715 SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const;
5716
5717 /// Expand a VECREDUCE_SEQ_* into an explicit ordered calculation.
5718 SDValue expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const;
5719
5720 /// Expand an SREM or UREM using SDIV/UDIV or SDIVREM/UDIVREM, if legal.
5721 /// Returns true if the expansion was successful.
5722 bool expandREM(SDNode *Node, SDValue &Result, SelectionDAG &DAG) const;
5723
5724 /// Method for building the DAG expansion of ISD::VECTOR_SPLICE. This
5725 /// method accepts vectors as its arguments.
5726 SDValue expandVectorSplice(SDNode *Node, SelectionDAG &DAG) const;
5727
5728 /// Expand a vector VECTOR_COMPRESS into a sequence of extract element, store
5729 /// temporarily, advance store position, before re-loading the final vector.
5730 SDValue expandVECTOR_COMPRESS(SDNode *Node, SelectionDAG &DAG) const;
5731
5732 /// Expands PARTIAL_REDUCE_S/UMLA nodes to a series of simpler operations,
5733 /// consisting of zext/sext, extract_subvector, mul and add operations.
5734 SDValue expandPartialReduceMLA(SDNode *Node, SelectionDAG &DAG) const;
5735
5736 /// Legalize a SETCC or VP_SETCC with given LHS and RHS and condition code CC
5737 /// on the current target. A VP_SETCC will additionally be given a Mask
5738 /// and/or EVL not equal to SDValue().
5739 ///
5740 /// If the SETCC has been legalized using AND / OR, then the legalized node
5741 /// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert
5742 /// will be set to false. This will also hold if the VP_SETCC has been
5743 /// legalized using VP_AND / VP_OR.
5744 ///
5745 /// If the SETCC / VP_SETCC has been legalized by using
5746 /// getSetCCSwappedOperands(), then the values of LHS and RHS will be
5747 /// swapped, CC will be set to the new condition, and NeedInvert will be set
5748 /// to false.
5749 ///
5750 /// If the SETCC / VP_SETCC has been legalized using the inverse condcode,
5751 /// then LHS and RHS will be unchanged, CC will set to the inverted condcode,
5752 /// and NeedInvert will be set to true. The caller must invert the result of
5753 /// the SETCC with SelectionDAG::getLogicalNOT() or take equivalent action to
5754 /// swap the effect of a true/false result.
5755 ///
5756 /// \returns true if the SETCC / VP_SETCC has been legalized, false if it
5757 /// hasn't.
5758 bool LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT, SDValue &LHS,
5759 SDValue &RHS, SDValue &CC, SDValue Mask,
5760 SDValue EVL, bool &NeedInvert, const SDLoc &dl,
5761 SDValue &Chain, bool IsSignaling = false) const;
5762
5763 //===--------------------------------------------------------------------===//
5764 // Instruction Emitting Hooks
5765 //
5766
5767 /// This method should be implemented by targets that mark instructions with
5768 /// the 'usesCustomInserter' flag. These instructions are special in various
5769 /// ways, which require special support to insert. The specified MachineInstr
5770 /// is created but not inserted into any basic blocks, and this method is
5771 /// called to expand it into a sequence of instructions, potentially also
5772 /// creating new basic blocks and control flow.
5773 /// As long as the returned basic block is different (i.e., we created a new
5774 /// one), the custom inserter is free to modify the rest of \p MBB.
5775 virtual MachineBasicBlock *
5776 EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const;
5777
5778 /// This method should be implemented by targets that mark instructions with
5779 /// the 'hasPostISelHook' flag. These instructions must be adjusted after
5780 /// instruction selection by target hooks. e.g. To fill in optional defs for
5781 /// ARM 's' setting instructions.
5782 virtual void AdjustInstrPostInstrSelection(MachineInstr &MI,
5783 SDNode *Node) const;
5784
5785 /// If this function returns true, SelectionDAGBuilder emits a
5786 /// LOAD_STACK_GUARD node when it is lowering Intrinsic::stackprotector.
5787 virtual bool useLoadStackGuardNode(const Module &M) const { return false; }
5788
5790 const SDLoc &DL) const {
5791 llvm_unreachable("not implemented for this target");
5792 }
5793
5794 /// Lower TLS global address SDNode for target independent emulated TLS model.
5795 virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA,
5796 SelectionDAG &DAG) const;
5797
5798 /// Expands target specific indirect branch for the case of JumpTable
5799 /// expansion.
5800 virtual SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value,
5801 SDValue Addr, int JTI,
5802 SelectionDAG &DAG) const;
5803
5804 // seteq(x, 0) -> truncate(srl(ctlz(zext(x)), log2(#bits)))
5805 // If we're comparing for equality to zero and isCtlzFast is true, expose the
5806 // fact that this can be implemented as a ctlz/srl pair, so that the dag
5807 // combiner can fold the new nodes.
5808 SDValue lowerCmpEqZeroToCtlzSrl(SDValue Op, SelectionDAG &DAG) const;
5809
5810 // Return true if `X & Y eq/ne 0` is preferable to `X & Y ne/eq Y`
5812 return true;
5813 }
5814
5815 // Expand vector operation by dividing it into smaller length operations and
5816 // joining their results. SDValue() is returned when expansion did not happen.
5817 SDValue expandVectorNaryOpBySplitting(SDNode *Node, SelectionDAG &DAG) const;
5818
5819 /// Replace an extraction of a load with a narrowed load.
5820 ///
5821 /// \param ResultVT type of the result extraction.
5822 /// \param InVecVT type of the input vector to with bitcasts resolved.
5823 /// \param EltNo index of the vector element to load.
5824 /// \param OriginalLoad vector load that to be replaced.
5825 /// \returns \p ResultVT Load on success SDValue() on failure.
5826 SDValue scalarizeExtractedVectorLoad(EVT ResultVT, const SDLoc &DL,
5827 EVT InVecVT, SDValue EltNo,
5828 LoadSDNode *OriginalLoad,
5829 SelectionDAG &DAG) const;
5830
5831private:
5832 SDValue foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
5833 const SDLoc &DL, DAGCombinerInfo &DCI) const;
5834 SDValue foldSetCCWithOr(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
5835 const SDLoc &DL, DAGCombinerInfo &DCI) const;
5836 SDValue foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
5837 const SDLoc &DL, DAGCombinerInfo &DCI) const;
5838
5839 SDValue optimizeSetCCOfSignedTruncationCheck(EVT SCCVT, SDValue N0,
5841 DAGCombinerInfo &DCI,
5842 const SDLoc &DL) const;
5843
5844 // (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
5845 SDValue optimizeSetCCByHoistingAndByConstFromLogicalShift(
5846 EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
5847 DAGCombinerInfo &DCI, const SDLoc &DL) const;
5848
5849 SDValue prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,
5850 SDValue CompTargetNode, ISD::CondCode Cond,
5851 DAGCombinerInfo &DCI, const SDLoc &DL,
5852 SmallVectorImpl<SDNode *> &Created) const;
5853 SDValue buildUREMEqFold(EVT SETCCVT, SDValue REMNode, SDValue CompTargetNode,
5854 ISD::CondCode Cond, DAGCombinerInfo &DCI,
5855 const SDLoc &DL) const;
5856
5857 SDValue prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,
5858 SDValue CompTargetNode, ISD::CondCode Cond,
5859 DAGCombinerInfo &DCI, const SDLoc &DL,
5860 SmallVectorImpl<SDNode *> &Created) const;
5861 SDValue buildSREMEqFold(EVT SETCCVT, SDValue REMNode, SDValue CompTargetNode,
5862 ISD::CondCode Cond, DAGCombinerInfo &DCI,
5863 const SDLoc &DL) const;
5864};
5865
5866/// Given an LLVM IR type and return type attributes, compute the return value
5867/// EVTs and flags, and optionally also the offsets, if the return value is
5868/// being lowered to memory.
5869LLVM_ABI void GetReturnInfo(CallingConv::ID CC, Type *ReturnType,
5870 AttributeList attr,
5871 SmallVectorImpl<ISD::OutputArg> &Outs,
5872 const TargetLowering &TLI, const DataLayout &DL);
5873
5874} // end namespace llvm
5875
5876#endif // LLVM_CODEGEN_TARGETLOWERING_H
unsigned const MachineRegisterInfo * MRI
return SDValue()
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
Function Alias Analysis Results
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
block Block Frequency Analysis
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Analysis containing CSE Info
Definition CSEInfo.cpp:27
#define LLVM_ABI
Definition Compiler.h:213
#define LLVM_READONLY
Definition Compiler.h:322
This file defines the DenseMap class.
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
static bool ShrinkDemandedConstant(Instruction *I, unsigned OpNo, const APInt &Demanded)
Check to see if the specified operand of the specified instruction is a constant integer.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
#define RegName(no)
lazy value info
Implement a low-level type suitable for MachineInstr level instruction selection.
#define F(x, y, z)
Definition MD5.cpp:55
#define I(x, y, z)
Definition MD5.cpp:58
Register const TargetRegisterInfo * TRI
Promote Memory to Register
Definition Mem2Reg.cpp:110
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t High
PowerPC Reduce CR logical Operation
const SmallVectorImpl< MachineOperand > & Cond
static cl::opt< RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development, "development", "for training")))
static Type * getValueType(Value *V)
Returns the type of the given value/instruction V.
This file defines the SmallVector class.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
static SDValue scalarizeVectorStore(StoreSDNode *Store, MVT StoreVT, SelectionDAG &DAG)
Scalarize a vector store, bitcasting to TargetVT to determine the scalar type.
Value * RHS
Value * LHS
Class for arbitrary precision integers.
Definition APInt.h:78
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition APInt.h:1488
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:41
A cache of @llvm.assume calls within a function.
An instruction that atomically checks whether a specified value is in a memory location,...
an instruction that atomically reads a memory location, combines it with another value,...
bool isFloatingPointOperation() const
BinOp getOperation() const
This class holds the attributes for a particular argument, parameter, function, or return value.
Definition Attributes.h:361
LLVM_ABI bool getValueAsBool() const
Return the attribute's value as a boolean.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
CCState - This class holds information needed while lowering arguments and return values.
CCValAssign - Represent assignment of one arg/retval to a location.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
This class represents a function call, abstracting a target machine's calling convention.
This is the shared class of boolean and integer constants.
Definition Constants.h:87
This class represents a range of values.
This is an important base class in LLVM.
Definition Constant.h:43
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:63
constexpr bool isScalar() const
Exactly one element.
Definition TypeSize.h:321
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Definition FastISel.h:66
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Class to represent function types.
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
bool isVarArg() const
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition Function.cpp:762
Common base class shared among various IRBuilders.
Definition IRBuilder.h:114
A wrapper class for inspecting calls to intrinsic functions.
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:68
An instruction for reading from memory.
This class is used to represent ISD::LOAD nodes.
Represents a single loop in the control flow graph.
Definition LoopInfo.h:40
Context object for machine code objects.
Definition MCContext.h:83
Base class for the full range of assembler expressions which are needed for parsing.
Definition MCExpr.h:34
Wrapper class representing physical registers. Should be passed by value.
Definition MCRegister.h:33
Machine Value Type.
@ INVALID_SIMPLE_VALUE_TYPE
SimpleValueType SimpleTy
uint64_t getScalarSizeInBits() const
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
ElementCount getVectorElementCount() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
bool isValid() const
Return true if this is a valid simple valuetype.
static MVT getIntegerVT(unsigned BitWidth)
Instructions::iterator instr_iterator
Representation of each machine instruction.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
This is an abstract virtual class for memory operations.
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:67
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Definition ArrayRef.h:303
A discriminated union of two or more pointer types, with the discriminator in the low bit of the poin...
Analysis providing profile information.
Wrapper class representing virtual and physical registers.
Definition Register.h:19
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
bool hasOneUse() const
Return true if there is exactly one use of this node.
bool use_empty() const
Return true if there are no uses of this node.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
LLVM_ABI SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
const DataLayout & getDataLayout() const
LLVM_ABI void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
LLVMContext * getContext() const
This instruction constructs a fixed permutation of two input vectors.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
This class is used to represent ISD::STORE nodes.
StringRef - Represent a constant reference to a string, i.e.
Definition StringRef.h:55
Multiway switch.
TargetInstrInfo - Interface to description of machine instruction set.
Provides information about what library functions are available for the current target.
ArgListEntry(Value *Val, SDValue Node=SDValue())
ArgListEntry(Value *Val, SDValue Node, Type *Ty)
Type * Ty
Same as OrigTy, or partially legalized for soft float libcalls.
Type * OrigTy
Original unlegalized argument type.
LegalizeTypeAction getTypeAction(MVT VT) const
void setTypeAction(MVT VT, LegalizeTypeAction Action)
This base class for TargetLowering contains the SelectionDAG-independent parts that can be used from ...
virtual Value * emitStoreConditional(IRBuilderBase &Builder, Value *Val, Value *Addr, AtomicOrdering Ord) const
Perform a store-conditional operation to Addr.
virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
bool isOperationExpand(unsigned Op, EVT VT) const
Return true if the specified operation is illegal on this target or unlikely to be made legal with cu...
EVT getMemValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
virtual bool enableAggressiveFMAFusion(LLT Ty) const
Return true if target always benefits from combining into FMA for a given value type.
virtual void emitBitTestAtomicRMWIntrinsic(AtomicRMWInst *AI) const
Perform a bit test atomicrmw using a target-specific intrinsic.
void setOperationAction(ArrayRef< unsigned > Ops, ArrayRef< MVT > VTs, LegalizeAction Action)
virtual bool requiresUniformRegister(MachineFunction &MF, const Value *) const
Allows target to decide about the register class of the specific value that is live outside the defin...
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
virtual unsigned getVaListSizeInBits(const DataLayout &DL) const
Returns the size of the platform's va_list object.
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
virtual bool preferSextInRegOfTruncate(EVT TruncVT, EVT VT, EVT ExtVT) const
virtual bool decomposeMulByConstant(LLVMContext &Context, EVT VT, SDValue C) const
Return true if it is profitable to transform an integer multiplication-by-constant into simpler opera...
void setMaxDivRemBitWidthSupported(unsigned SizeInBits)
Set the size in bits of the maximum div/rem the backend supports.
virtual bool hasAndNot(SDValue X) const
Return true if the target has a bitwise and-not operation: X = ~A & B This can be used to simplify se...
ReciprocalEstimate
Reciprocal estimate status values used by the functions below.
bool PredictableSelectIsExpensive
Tells the code generator that select is more expensive than a branch if the branch is usually predict...
virtual bool isShuffleMaskLegal(ArrayRef< int >, EVT) const
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations,...
virtual bool enableAggressiveFMAFusion(EVT VT) const
Return true if target always benefits from combining into FMA for a given value type.
virtual bool isComplexDeinterleavingOperationSupported(ComplexDeinterleavingOperation Operation, Type *Ty) const
Does this target support complex deinterleaving with the given operation and type.
virtual bool shouldRemoveRedundantExtend(SDValue Op) const
Return true (the default) if it is profitable to remove a sext_inreg(x) where the sext is redundant,...
bool isIndexedStoreLegal(unsigned IdxMode, EVT VT) const
Return true if the specified indexed load is legal on this target.
SDValue promoteTargetBoolean(SelectionDAG &DAG, SDValue Bool, EVT ValVT) const
Promote the given target boolean to a target boolean of the given type.
virtual bool isFMADLegal(const SelectionDAG &DAG, const SDNode *N) const
Returns true if be combined with to form an ISD::FMAD.
virtual bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT, std::optional< unsigned > ByteOffset=std::nullopt) const
Return true if it is profitable to reduce a load to a smaller type.
virtual bool hasStandaloneRem(EVT VT) const
Return true if the target can handle a standalone remainder operation.
virtual bool isExtFreeImpl(const Instruction *I) const
Return true if the extension represented by I is free.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual bool shouldExpandBuildVectorWithShuffles(EVT, unsigned DefinedValues) const
LegalizeAction getIndexedMaskedStoreAction(unsigned IdxMode, MVT VT) const
Return how the indexed store should be treated: either it is legal, needs to be promoted to a larger ...
virtual bool canCombineTruncStore(EVT ValVT, EVT MemVT, bool LegalOnly) const
virtual bool isSelectSupported(SelectSupportKind) const
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
MachineBasicBlock * emitPatchPoint(MachineInstr &MI, MachineBasicBlock *MBB) const
Replace/modify any TargetFrameIndex operands with a targte-dependent sequence of memory operands that...
virtual bool isEqualityCmpFoldedWithSignedCmp() const
Return true if instruction generated for equality comparison is folded with instruction generated for...
virtual bool preferSelectsOverBooleanArithmetic(EVT VT) const
Should we prefer selects to doing arithmetic on boolean types.
virtual bool useStackGuardXorFP() const
If this function returns true, stack protection checks should XOR the frame pointer (or whichever poi...
virtual bool isLegalICmpImmediate(int64_t) const
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
virtual bool convertSetCCLogicToBitwiseLogic(EVT VT) const
Use bitwise logic to make pairs of compares more efficient.
void setAtomicLoadExtAction(ArrayRef< unsigned > ExtTypes, MVT ValVT, ArrayRef< MVT > MemVTs, LegalizeAction Action)
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual bool shouldFormOverflowOp(unsigned Opcode, EVT VT, bool MathUsed) const
Try to convert math with an overflow comparison into the corresponding DAG node operation.
ShiftLegalizationStrategy
Return the preferred strategy to legalize tihs SHIFT instruction, with ExpansionFactor being the recu...
virtual bool isVectorLoadExtDesirable(SDValue ExtVal) const
Return true if folding a vector load into ExtVal (a sign, zero, or any extend node) is profitable.
virtual bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const
Return if the target supports combining a chain like:
virtual Value * createComplexDeinterleavingIR(IRBuilderBase &B, ComplexDeinterleavingOperation OperationType, ComplexDeinterleavingRotation Rotation, Value *InputA, Value *InputB, Value *Accumulator=nullptr) const
Create the IR node for the given complex deinterleaving operation.
virtual bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const
Return true if it is beneficial to convert a load of a constant to just the constant itself.
virtual bool isSupportedFixedPointOperation(unsigned Op, EVT VT, unsigned Scale) const
Custom method defined by each target to indicate if an operation which may require a scale is support...
void setLoadExtAction(ArrayRef< unsigned > ExtTypes, MVT ValVT, MVT MemVT, LegalizeAction Action)
virtual Sched::Preference getSchedulingPreference(SDNode *) const
Some scheduler, e.g.
void setLibcallImplCallingConv(RTLIB::LibcallImpl Call, CallingConv::ID CC)
Set the CallingConv that should be used for the specified libcall.
virtual MachineInstr * EmitKCFICheck(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator &MBBI, const TargetInstrInfo *TII) const
void setMinStackArgumentAlignment(Align Alignment)
Set the minimum stack alignment of an argument.
bool isExtLoad(const LoadInst *Load, const Instruction *Ext, const DataLayout &DL) const
Return true if Load and Ext can form an ExtLoad.
LegalizeTypeAction getTypeAction(MVT VT) const
virtual bool isLegalScaleForGatherScatter(uint64_t Scale, uint64_t ElemSize) const
EVT getTypeToExpandTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
virtual bool shouldInsertFencesForAtomic(const Instruction *I) const
Whether AtomicExpandPass should automatically insert fences and reduce ordering for this atomic.
virtual AtomicOrdering atomicOperationOrderAfterFenceSplit(const Instruction *I) const
MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
virtual bool allowsMisalignedMemoryAccesses(LLT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
LLT handling variant.
virtual bool isSafeMemOpType(MVT) const
Returns true if it's safe to use load / store of the specified type to expand memcpy / memset inline.
virtual void emitExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) const
Perform a cmpxchg expansion using a target-specific method.
virtual CondMergingParams getJumpConditionMergingParams(Instruction::BinaryOps, const Value *, const Value *) const
const TargetMachine & getTargetMachine() const
unsigned MaxLoadsPerMemcmp
Specify maximum number of load instructions per memcmp call.
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
bool rangeFitsInWord(const APInt &Low, const APInt &High, const DataLayout &DL) const
Check whether the range [Low,High] fits in a machine word.
virtual bool isCtpopFast(EVT VT) const
Return true if ctpop instruction is fast.
virtual MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
unsigned MaxGluedStoresPerMemcpy
Specify max number of store instructions to glue in inlined memcpy.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
bool isPaddedAtMostSignificantBitsWhenStored(EVT VT) const
Indicates if any padding is guaranteed to go at the most significant bits when storing the type to me...
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
void setOperationPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
Convenience method to set an operation to Promote and specify the type in a single call.
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
unsigned getMinCmpXchgSizeInBits() const
Returns the size of the smallest cmpxchg or ll/sc instruction the backend supports.
virtual Value * emitMaskedAtomicRMWIntrinsic(IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr, Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const
Perform a masked atomicrmw using a target-specific intrinsic.
virtual bool areJTsAllowed(const Function *Fn) const
Return true if lowering to a jump table is allowed.
bool enableExtLdPromotion() const
Return true if the target wants to use the optimization that turns ext(promotableInst1(....
virtual bool isFPExtFoldable(const MachineInstr &MI, unsigned Opcode, LLT DestTy, LLT SrcTy) const
Return true if an fpext operation input to an Opcode operation is free (for instance,...
void setIndexedMaskedLoadAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked load does or does not work with the specified type and ind...
void setMaxBytesForAlignment(unsigned MaxBytes)
bool isOperationLegalOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal using promotion.
void setHasExtractBitsInsn(bool hasExtractInsn=true)
Tells the code generator that the target has BitExtract instructions.
void addBypassSlowDiv(unsigned int SlowBitWidth, unsigned int FastBitWidth)
Tells the code generator which bitwidths to bypass.
virtual bool hasBitTest(SDValue X, SDValue Y) const
Return true if the target has a bit-test instruction: (X & (1 << Y)) ==/!= 0 This knowledge can be us...
MVT getRegisterType(LLVMContext &Context, EVT VT) const
Return the type of registers that this ValueType will eventually require.
virtual bool needsFixedCatchObjects() const
virtual Value * emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy, Value *Addr, AtomicOrdering Ord) const
Perform a load-linked operation on Addr, returning a "Value *" with the corresponding pointee type.
void setMaxLargeFPConvertBitWidthSupported(unsigned SizeInBits)
Set the size in bits of the maximum fp to/from int conversion the backend supports.
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const
Return the number of registers that this ValueType will eventually require.
virtual bool isCheapToSpeculateCttz(Type *Ty) const
Return true if it is cheap to speculate a call to intrinsic cttz.
bool isJumpExpensive() const
Return true if Flow Control is an expensive operation that should be avoided.
virtual bool useFPRegsForHalfType() const
LegalizeAction getCondCodeAction(ISD::CondCode CC, MVT VT) const
Return how the condition code should be treated: either it is legal, needs to be expanded to some oth...
bool hasExtractBitsInsn() const
Return true if the target has BitExtract instructions.
bool isTruncStoreLegal(EVT ValVT, EVT MemVT) const
Return true if the specified store with truncation is legal on this target.
virtual bool isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT, const SelectionDAG &DAG, const MachineMemOperand &MMO) const
Return true if the following transform is beneficial: fold (conv (load x)) -> (load (conv*)x) On arch...
LegalizeAction getIndexedStoreAction(unsigned IdxMode, MVT VT) const
Return how the indexed store should be treated: either it is legal, needs to be promoted to a larger ...
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
CallingConv::ID getLibcallImplCallingConv(RTLIB::LibcallImpl Call) const
Get the CallingConv that should be used for the specified libcall implementation.
unsigned getMaxStoresPerMemcpy(bool OptSize) const
Get maximum # of store operations permitted for llvm.memcpy.
void setPrefLoopAlignment(Align Alignment)
Set the target's preferred loop alignment.
virtual bool softPromoteHalfType() const
virtual bool areTwoSDNodeTargetMMOFlagsMergeable(const MemSDNode &NodeX, const MemSDNode &NodeY) const
Return true if it is valid to merge the TargetMMOFlags in two SDNodes.
virtual bool isCommutativeBinOp(unsigned Opcode) const
Returns true if the opcode is a commutative binary operation.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
virtual bool isFPImmLegal(const APFloat &, EVT, bool ForCodeSize=false) const
Returns true if the target can instruction select the specified FP immediate natively.
virtual unsigned getPreferredFPToIntOpcode(unsigned Op, EVT FromVT, EVT ToVT) const
virtual bool isExtractVecEltCheap(EVT VT, unsigned Index) const
Return true if extraction of a scalar element from the given vector type at the given index is cheap.
void setOperationAction(ArrayRef< unsigned > Ops, MVT VT, LegalizeAction Action)
virtual bool optimizeFMulOrFDivAsShiftAddBitcast(SDNode *N, SDValue FPConst, SDValue IntPow2) const
SelectSupportKind
Enum that describes what type of support for selects the target has.
LegalizeAction getIndexedLoadAction(unsigned IdxMode, MVT VT) const
Return how the indexed load should be treated: either it is legal, needs to be promoted to a larger s...
virtual bool shouldTransformSignedTruncationCheck(EVT XVT, unsigned KeptBits) const
Should we tranform the IR-optimal check for whether given truncation down into KeptBits would be trun...
virtual bool isFPExtFoldable(const SelectionDAG &DAG, unsigned Opcode, EVT DestVT, EVT SrcVT) const
Return true if an fpext operation input to an Opcode operation is free (for instance,...
bool isLegalRC(const TargetRegisterInfo &TRI, const TargetRegisterClass &RC) const
Return true if the value types that can be represented by the specified register class are all legal.
virtual TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const
Return the preferred vector type legalization action.
virtual bool allowTruncateForTailCall(Type *FromTy, Type *ToTy) const
Return true if a truncation from FromTy to ToTy is permitted when deciding whether a call is in tail ...
void setAtomicLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Let target indicate that an extending atomic load of the specified type is legal.
virtual bool shouldExtendGSIndex(EVT VT, EVT &EltTy) const
Returns true if the index type for a masked gather/scatter requires extending.
virtual unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
virtual bool shouldNormalizeToSelectSequence(LLVMContext &Context, EVT VT) const
Returns true if we should normalize select(N0&N1, X, Y) => select(N0, select(N1, X,...
Register getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
virtual StringRef getStackProbeSymbolName(const MachineFunction &MF) const
LegalizeAction getFixedPointOperationAction(unsigned Op, EVT VT, unsigned Scale) const
Some fixed point operations may be natively supported by the target but only for specific scales.
virtual bool preferScalarizeSplat(SDNode *N) const
bool isIndexedMaskedLoadLegal(unsigned IdxMode, EVT VT) const
Return true if the specified indexed load is legal on this target.
virtual ISD::NodeType getExtendForAtomicOps() const
Returns how the platform's atomic operations are extended (ZERO_EXTEND, SIGN_EXTEND,...
Sched::Preference getSchedulingPreference() const
Return target scheduling preference.
virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &, MachineFunction &, unsigned) const
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
Determine if the target supports unaligned memory accesses.
virtual LLT getOptimalMemOpLLT(const MemOp &Op, const AttributeList &) const
LLT returning variant.
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
virtual void emitExpandAtomicRMW(AtomicRMWInst *AI) const
Perform a atomicrmw expansion using a target-specific way.
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
virtual bool reduceSelectOfFPConstantLoads(EVT CmpOpVT) const
Return true if it is profitable to convert a select of FP constants into a constant pool load whose a...
bool hasBigEndianPartOrdering(EVT VT, const DataLayout &DL) const
When splitting a value of the specified type into parts, does the Lo or Hi part come first?
const char * getMemcpyName() const
virtual bool hasStackProbeSymbol(const MachineFunction &MF) const
Returns the name of the symbol used to emit stack probes or the empty string if not applicable.
bool isSlowDivBypassed() const
Returns true if target has indicated at least one type should be bypassed.
virtual Align getABIAlignmentForCallingConv(Type *ArgTy, const DataLayout &DL) const
Certain targets have context sensitive alignment requirements, where one type has the alignment requi...
virtual bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const
Return true if EXTRACT_SUBVECTOR is cheap for extracting this result type from this source type with ...
virtual bool isMulAddWithConstProfitable(SDValue AddNode, SDValue ConstNode) const
Return true if it may be profitable to transform (mul (add x, c1), c2) -> (add (mul x,...
virtual bool shouldExtendTypeInLibCall(EVT Type) const
Returns true if arguments should be extended in lib calls.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
bool isPartialReduceMLALegalOrCustom(unsigned Opc, EVT AccVT, EVT InputVT) const
Return true if a PARTIAL_REDUCE_U/SMLA node with the specified types is legal or custom for this targ...
virtual bool isFsqrtCheap(SDValue X, SelectionDAG &DAG) const
Return true if SQRT(X) shouldn't be replaced with X*RSQRT(X).
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
virtual bool shouldFoldConstantShiftPairToMask(const SDNode *N, CombineLevel Level) const
Return true if it is profitable to fold a pair of shifts into a mask.
virtual bool shouldExpandGetActiveLaneMask(EVT VT, EVT OpVT) const
Return true if the @llvm.get.active.lane.mask intrinsic should be expanded using generic code in Sele...
virtual bool shallExtractConstSplatVectorElementToStore(Type *VectorTy, unsigned ElemSizeInBits, unsigned &Index) const
Return true if the target shall perform extract vector element and store given that the vector is kno...
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it's free to truncate a value of type FromTy to type ToTy.
virtual bool hasMultipleConditionRegisters(EVT VT) const
Does the target have multiple (allocatable) condition registers that can be used to store the results...
unsigned getMaxExpandSizeMemcmp(bool OptSize) const
Get maximum # of load operations permitted for memcmp.
bool isStrictFPEnabled() const
Return true if the target support strict float operation.
virtual bool shouldAvoidTransformToShift(EVT VT, unsigned Amount) const
Return true if creating a shift of the type by the given amount is not profitable.
virtual bool shouldPreservePtrArith(const Function &F, EVT PtrVT) const
True if target has some particular form of dealing with pointer arithmetic semantics for pointers wit...
virtual bool isFPExtFree(EVT DestVT, EVT SrcVT) const
Return true if an fpext operation is free (for instance, because single-precision floating-point numb...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
virtual bool lowerInterleavedStore(Instruction *Store, Value *Mask, ShuffleVectorInst *SVI, unsigned Factor, const APInt &GapMask) const
Lower an interleaved store to target specific intrinsics.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
virtual bool shouldFoldSelectWithSingleBitTest(EVT VT, const APInt &AndMask) const
MVT getSimpleValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the MVT corresponding to this LLVM type. See getValueType.
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
virtual bool shouldReassociateReduction(unsigned RedOpc, EVT VT) const
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isCondCodeLegal(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal for a comparison of the specified types on this ...
virtual bool canCombineStoreAndExtract(Type *VectorTy, Value *Idx, unsigned &Cost) const
Return true if the target can combine store(extractelement VectorTy,Idx).
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
LegalizeAction getAtomicLoadExtAction(unsigned ExtType, EVT ValVT, EVT MemVT) const
Same as getLoadExtAction, but for atomic loads.
MVT getProgramPointerTy(const DataLayout &DL) const
Return the type for code pointers, which is determined by the program address space specified through...
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
virtual void emitAtomicCmpXchgNoStoreLLBalance(IRBuilderBase &Builder) const
void setSupportsUnalignedAtomics(bool UnalignedSupported)
Sets whether unaligned atomic operations are supported.
void setLoadExtAction(ArrayRef< unsigned > ExtTypes, MVT ValVT, ArrayRef< MVT > MemVTs, LegalizeAction Action)
virtual void emitExpandAtomicStore(StoreInst *SI) const
Perform a atomic store using a target-specific way.
bool isSuitableForBitTests(unsigned NumDests, unsigned NumCmps, const APInt &Low, const APInt &High, const DataLayout &DL) const
Return true if lowering to a bit test is suitable for a set of case clusters which contains NumDests ...
virtual bool preferIncOfAddToSubOfNot(EVT VT) const
These two forms are equivalent: sub y, (xor x, -1) add (add x, 1), y The variant with two add's is IR...
virtual bool ShouldShrinkFPConstant(EVT) const
If true, then instruction selection should seek to shrink the FP constant of the specified type to a ...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
unsigned getMaxDivRemBitWidthSupported() const
Returns the size in bits of the maximum div/rem the backend supports.
virtual bool isLegalAddImmediate(int64_t) const
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
virtual unsigned getMaxSupportedInterleaveFactor() const
Get the maximum supported factor for interleaved memory accesses.
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
LegalizeAction getTruncStoreAction(EVT ValVT, EVT MemVT) const
Return how this store with truncation should be treated: either it is legal, needs to be promoted to ...
virtual bool shouldKeepZExtForFP16Conv() const
Does this target require the clearing of high-order bits in a register passed to the fp16 to fp conve...
virtual AtomicExpansionKind shouldCastAtomicRMWIInIR(AtomicRMWInst *RMWI) const
Returns how the given atomic atomicrmw should be cast by the IR-level AtomicExpand pass.
void setIndexedMaskedStoreAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked store does or does not work with the specified type and in...
virtual bool canTransformPtrArithOutOfBounds(const Function &F, EVT PtrVT) const
True if the target allows transformations of in-bounds pointer arithmetic that cause out-of-bounds in...
virtual bool shouldConsiderGEPOffsetSplit() const
const ValueTypeActionImpl & getValueTypeActions() const
virtual AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
virtual bool isTruncateFree(SDValue Val, EVT VT2) const
Return true if truncating the specific node Val to type VT2 is free.
virtual bool shouldExpandVectorMatch(EVT VT, unsigned SearchSize) const
Return true if the @llvm.experimental.vector.match intrinsic should be expanded for vector type ‘VT’ ...
virtual bool isProfitableToCombineMinNumMaxNum(EVT VT) const
virtual unsigned getCustomCtpopCost(EVT VT, ISD::CondCode Cond) const
Return the maximum number of "x & (x - 1)" operations that can be done instead of deferring to a cust...
virtual bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y, unsigned OldShiftOpcode, unsigned NewShiftOpcode, SelectionDAG &DAG) const
Given the pattern (X & (C l>>/<< Y)) ==/!= 0 return true if it should be transformed into: ((X <</l>>...
virtual bool isFNegFree(EVT VT) const
Return true if an fneg operation is free to the point where it is never worthwhile to replace it with...
void setPartialReduceMLAAction(unsigned Opc, MVT AccVT, MVT InputVT, LegalizeAction Action)
Indicate how a PARTIAL_REDUCE_U/SMLA node with Acc type AccVT and Input type InputVT should be treate...
LegalizeAction getLoadExtAction(unsigned ExtType, EVT ValVT, EVT MemVT) const
Return how this load with extension should be treated: either it is legal, needs to be promoted to a ...
virtual bool shouldInsertTrailingFenceForAtomicStore(const Instruction *I) const
Whether AtomicExpandPass should automatically insert a trailing fence without reducing the ordering f...
virtual AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const
Returns how the given (atomic) load should be expanded by the IR-level AtomicExpand pass.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
bool isExtFree(const Instruction *I) const
Return true if the extension represented by I is free.
virtual MVT getFenceOperandTy(const DataLayout &DL) const
Return the type for operands of fence.
virtual Value * emitMaskedAtomicCmpXchgIntrinsic(IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr, Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const
Perform a masked cmpxchg using a target-specific intrinsic.
virtual bool isZExtFree(EVT FromTy, EVT ToTy) const
virtual ISD::NodeType getExtendForAtomicCmpSwapArg() const
Returns how the platform's atomic compare and swap expects its comparison value to be extended (ZERO_...
virtual bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT, unsigned SelectOpcode, SDValue X, SDValue Y) const
Return true if pulling a binary operation into a select with an identity constant is profitable.
BooleanContent
Enum that describes how the target represents true/false values.
virtual bool shouldExpandGetVectorLength(EVT CountVT, unsigned VF, bool IsScalable) const
virtual bool isIntDivCheap(EVT VT, AttributeList Attr) const
Return true if integer divide is usually cheaper than a sequence of several shifts,...
virtual ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const
virtual uint8_t getRepRegClassCostFor(MVT VT) const
Return the cost of the 'representative' register class for the specified value type.
virtual bool isZExtFree(LLT FromTy, LLT ToTy, LLVMContext &Ctx) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
LegalizeAction getPartialReduceMLAAction(unsigned Opc, EVT AccVT, EVT InputVT) const
Return how a PARTIAL_REDUCE_U/SMLA node with Acc type AccVT and Input type InputVT should be treated.
bool isPredictableSelectExpensive() const
Return true if selects are only cheaper than branches if the branch is unlikely to be predicted right...
virtual bool mergeStoresAfterLegalization(EVT MemVT) const
Allow store merging for the specified type after legalization in addition to before legalization.
virtual bool shouldMergeStoreOfLoadsOverCall(EVT, EVT) const
Returns true if it's profitable to allow merging store of loads when there are functions calls betwee...
RTLIB::LibcallImpl getSupportedLibcallImpl(StringRef FuncName) const
Check if this is valid libcall for the current module, otherwise RTLIB::Unsupported.
unsigned getMaxStoresPerMemmove(bool OptSize) const
Get maximum # of store operations permitted for llvm.memmove.
virtual bool isProfitableToHoist(Instruction *I) const
unsigned getGatherAllAliasesMaxDepth() const
virtual LegalizeAction getCustomOperationAction(SDNode &Op) const
How to legalize this custom operation?
virtual bool isFMAFasterThanFMulAndFAdd(const Function &F, Type *) const
IR version.
virtual bool hasAndNotCompare(SDValue Y) const
Return true if the target should transform: (X & Y) == Y ---> (~X & Y) == 0 (X & Y) !...
virtual bool storeOfVectorConstantIsCheap(bool IsZero, EVT MemVT, unsigned NumElem, unsigned AddrSpace) const
Return true if it is expected to be cheaper to do a store of vector constant with the given size and ...
unsigned MaxLoadsPerMemcmpOptSize
Likewise for functions with the OptSize attribute.
virtual MVT hasFastEqualityCompare(unsigned NumBits) const
Return the preferred operand type if the target has a quick way to compare integer values of the give...
virtual const TargetRegisterClass * getRepRegClassFor(MVT VT) const
Return the 'representative' register class for the specified value type.
virtual bool isNarrowingProfitable(SDNode *N, EVT SrcVT, EVT DestVT) const
Return true if it's profitable to narrow operations of type SrcVT to DestVT.
virtual bool isMultiStoresCheaperThanBitsMerge(EVT LTy, EVT HTy) const
Return true if it is cheaper to split the store of a merged int val from a pair of smaller values int...
bool isLoadExtLegalOrCustom(unsigned ExtType, EVT ValVT, EVT MemVT) const
Return true if the specified load with extension is legal or custom on this target.
TargetLoweringBase(const TargetLoweringBase &)=delete
virtual unsigned getMaxGluedStoresPerMemcpy() const
Get maximum # of store operations to be glued together.
bool isAtomicLoadExtLegal(unsigned ExtType, EVT ValVT, EVT MemVT) const
Return true if the specified atomic load with extension is legal on this target.
virtual bool isBinOp(unsigned Opcode) const
Return true if the node is a math/logic binary operator.
virtual bool shouldFoldMaskToVariableShiftPair(SDValue X) const
There are two ways to clear extreme bits (either low or high): Mask: x & (-1 << y) (the instcombine c...
virtual bool alignLoopsWithOptSize() const
Should loops be aligned even when the function is marked OptSize (but not MinSize).
unsigned getMaxAtomicSizeInBitsSupported() const
Returns the maximum atomic operation size (in bits) supported by the backend.
bool isIndexedLoadLegal(unsigned IdxMode, EVT VT) const
Return true if the specified indexed load is legal on this target.
void setMinCmpXchgSizeInBits(unsigned SizeInBits)
Sets the minimum cmpxchg or ll/sc size supported by the backend.
virtual bool canMergeStoresTo(unsigned AS, EVT MemVT, const MachineFunction &MF) const
Returns if it's reasonable to merge stores to MemVT size.
void setPartialReduceMLAAction(ArrayRef< unsigned > Opcodes, MVT AccVT, MVT InputVT, LegalizeAction Action)
LegalizeAction getStrictFPOperationAction(unsigned Op, EVT VT) const
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
virtual bool preferABDSToABSWithNSW(EVT VT) const
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
virtual bool getAddrModeArguments(const IntrinsicInst *, SmallVectorImpl< Value * > &, Type *&) const
CodeGenPrepare sinks address calculations into the same BB as Load/Store instructions reading the add...
bool isLoadExtLegal(unsigned ExtType, EVT ValVT, EVT MemVT) const
Return true if the specified load with extension is legal on this target.
virtual bool hasInlineStackProbe(const MachineFunction &MF) const
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setCondCodeAction(ArrayRef< ISD::CondCode > CCs, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn't supported on the target and indicate what to d...
void setBooleanContents(BooleanContent IntTy, BooleanContent FloatTy)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
const DenseMap< unsigned int, unsigned int > & getBypassSlowDivWidths() const
Returns map of slow types for division or remainder with corresponding fast types.
void setOperationPromotedToType(ArrayRef< unsigned > Ops, MVT OrigVT, MVT DestVT)
unsigned getMaxLargeFPConvertBitWidthSupported() const
Returns the size in bits of the maximum fp to/from int conversion the backend supports.
virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, LLT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
virtual bool isTruncateFree(EVT FromVT, EVT ToVT) const
virtual bool isCheapToSpeculateCtlz(Type *Ty) const
Return true if it is cheap to speculate a call to intrinsic ctlz.
virtual bool shouldExpandCttzElements(EVT VT) const
Return true if the @llvm.experimental.cttz.elts intrinsic should be expanded using generic code in Se...
virtual bool signExtendConstant(const ConstantInt *C) const
Return true if this constant should be sign extended when promoting to a larger type.
virtual bool lowerInterleaveIntrinsicToStore(Instruction *Store, Value *Mask, ArrayRef< Value * > InterleaveValues) const
Lower an interleave intrinsic to a target specific store intrinsic.
virtual bool isTruncateFree(LLT FromTy, LLT ToTy, LLVMContext &Ctx) const
AndOrSETCCFoldKind
Enum of different potentially desirable ways to fold (and/or (setcc ...), (setcc ....
virtual bool shouldScalarizeBinop(SDValue VecOp) const
Try to convert an extract element of a vector binary operation into an extract element followed by a ...
Align getPrefFunctionAlignment() const
Return the preferred function alignment.
RTLIB::LibcallImpl getLibcallImpl(RTLIB::Libcall Call) const
Get the libcall impl routine name for the specified libcall.
virtual void emitExpandAtomicLoad(LoadInst *LI) const
Perform a atomic load using a target-specific way.
TargetLoweringBase(const TargetMachine &TM)
NOTE: The TargetMachine owns TLOF.
Align getMinFunctionAlignment() const
Return the minimum function alignment.
virtual AtomicExpansionKind shouldExpandAtomicStoreInIR(StoreInst *SI) const
Returns how the given (atomic) store should be expanded by the IR-level AtomicExpand pass into.
static StringRef getLibcallImplName(RTLIB::LibcallImpl Call)
Get the libcall routine name for the specified libcall implementation.
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
virtual bool isCtlzFast() const
Return true if ctlz instruction is fast.
virtual bool useSoftFloat() const
virtual bool isStoreBitCastBeneficial(EVT StoreVT, EVT BitcastVT, const SelectionDAG &DAG, const MachineMemOperand &MMO) const
Return true if the following transform is beneficial: (store (y (conv x)), y*)) -> (store x,...
BooleanContent getBooleanContents(EVT Type) const
virtual AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
bool isIndexedMaskedStoreLegal(unsigned IdxMode, EVT VT) const
Return true if the specified indexed load is legal on this target.
virtual int64_t getPreferredLargeGEPBaseOffset(int64_t MinOffset, int64_t MaxOffset) const
Return the prefered common base offset.
virtual bool isVectorClearMaskLegal(ArrayRef< int >, EVT) const
Similar to isShuffleMaskLegal.
LegalizeKind getTypeConversion(LLVMContext &Context, EVT VT) const
Return pair that represents the legalization kind (first) that needs to happen to EVT (second) in ord...
Align getMinStackArgumentAlignment() const
Return the minimum stack alignment of an argument.
virtual bool shouldUseStrictFP_TO_INT(EVT FpVT, EVT IntVT, bool IsSigned) const
Return true if it is more correct/profitable to use strict FP_TO_INT conversion operations - canonica...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
bool hasTargetDAGCombine(ISD::NodeType NT) const
If true, the target has custom DAG combine transformations that it can perform for the specified node...
void setLibcallImpl(RTLIB::Libcall Call, RTLIB::LibcallImpl Impl)
virtual bool fallBackToDAGISel(const Instruction &Inst) const
unsigned GatherAllAliasesMaxDepth
Depth that GatherAllAliases should continue looking for chain dependencies when trying to find a more...
virtual bool shouldSplatInsEltVarIndex(EVT) const
Return true if inserting a scalar into a variable element of an undef vector is more efficiently hand...
LegalizeAction getIndexedMaskedLoadAction(unsigned IdxMode, MVT VT) const
Return how the indexed load should be treated: either it is legal, needs to be promoted to a larger s...
NegatibleCost
Enum that specifies when a float negation is beneficial.
bool isTruncStoreLegalOrCustom(EVT ValVT, EVT MemVT) const
Return true if the specified store with truncation has solution on this target.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
virtual unsigned preferedOpcodeForCmpEqPiecesOfOperand(EVT VT, unsigned ShiftOpc, bool MayTransformRotate, const APInt &ShiftOrRotateAmt, const std::optional< APInt > &AndMask) const
virtual void emitCmpArithAtomicRMWIntrinsic(AtomicRMWInst *AI) const
Perform a atomicrmw which the result is only used by comparison, using a target-specific intrinsic.
virtual bool shouldSignExtendTypeInLibCall(Type *Ty, bool IsSigned) const
Returns true if arguments should be sign-extended in lib calls.
virtual Register getExceptionPointerRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
virtual bool isFMADLegal(const MachineInstr &MI, LLT Ty) const
Returns true if MI can be combined with another instruction to form TargetOpcode::G_FMAD.
void setCondCodeAction(ArrayRef< ISD::CondCode > CCs, ArrayRef< MVT > VTs, LegalizeAction Action)
bool supportsUnalignedAtomics() const
Whether the target supports unaligned atomic operations.
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
virtual bool isLegalAddScalableImmediate(int64_t) const
Return true if adding the specified scalable immediate is legal, that is the target has add instructi...
std::vector< ArgListEntry > ArgListTy
virtual bool shouldAlignPointerArgs(CallInst *, unsigned &, Align &) const
Return true if the pointer arguments to CI should be aligned by aligning the object whose address is ...
virtual bool hasVectorBlend() const
Return true if the target has a vector blend instruction.
virtual AtomicExpansionKind shouldCastAtomicStoreInIR(StoreInst *SI) const
Returns how the given (atomic) store should be cast by the IR-level AtomicExpand pass into.
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, ArrayRef< MVT > VTs, LegalizeAction Action)
virtual bool isVScaleKnownToBeAPowerOfTwo() const
Return true only if vscale must be a power of two.
virtual bool aggressivelyPreferBuildVectorSources(EVT VecVT) const
virtual MachineMemOperand::Flags getTargetMMOFlags(const MemSDNode &Node) const
This callback is used to inspect load/store SDNode.
virtual EVT getOptimalMemOpType(LLVMContext &Context, const MemOp &Op, const AttributeList &) const
Returns the target specific optimal type for load and store operations as a result of memset,...
virtual Type * shouldConvertSplatType(ShuffleVectorInst *SVI) const
Given a shuffle vector SVI representing a vector splat, return a new scalar type of size equal to SVI...
virtual bool isZExtFree(SDValue Val, EVT VT2) const
Return true if zero-extending the specific node Val to type VT2 is free (either because it's implicit...
void setAtomicLoadExtAction(ArrayRef< unsigned > ExtTypes, MVT ValVT, MVT MemVT, LegalizeAction Action)
virtual bool shouldRemoveExtendFromGSIndex(SDValue Extend, EVT DataVT) const
unsigned getMaxStoresPerMemset(bool OptSize) const
Get maximum # of store operations permitted for llvm.memset.
virtual LLVM_READONLY LLT getPreferredShiftAmountTy(LLT ShiftValueTy) const
Return the preferred type to use for a shift opcode, given the shifted amount type is ShiftValueTy.
bool isBeneficialToExpandPowI(int64_t Exponent, bool OptForSize) const
Return true if it is beneficial to expand an @llvm.powi.
LLT getVectorIdxLLT(const DataLayout &DL) const
Returns the type to be used for the index operand of: G_INSERT_VECTOR_ELT, G_EXTRACT_VECTOR_ELT,...
virtual EVT getAsmOperandValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, ArrayRef< MVT > VTs, LegalizeAction Action)
virtual AtomicExpansionKind shouldCastAtomicLoadInIR(LoadInst *LI) const
Returns how the given (atomic) load should be cast by the IR-level AtomicExpand pass.
bool isCondCodeLegalOrCustom(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal or custom for a comparison of the specified type...
virtual bool isComplexDeinterleavingSupported() const
Does this target support complex deinterleaving.
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
MVT getFrameIndexTy(const DataLayout &DL) const
Return the type for frame index, which is determined by the alloca address space specified through th...
virtual Register getExceptionSelectorRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception typeid on entry to a la...
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
virtual bool addressingModeSupportsTLS(const GlobalValue &) const
Returns true if the targets addressing mode can target thread local storage (TLS).
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
virtual bool shouldConvertPhiType(Type *From, Type *To) const
Given a set in interconnected phis of type 'From' that are loaded/stored or bitcast to type 'To',...
virtual bool isFAbsFree(EVT VT) const
Return true if an fabs operation is free to the point where it is never worthwhile to replace it with...
virtual bool isLegalStoreImmediate(int64_t Value) const
Return true if the specified immediate is legal for the value input of a store instruction.
virtual bool preferZeroCompareBranch() const
Return true if the heuristic to prefer icmp eq zero should be used in code gen prepare.
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
virtual bool lowerInterleavedLoad(Instruction *Load, Value *Mask, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor, const APInt &GapMask) const
Lower an interleaved load to target specific intrinsics.
virtual unsigned getVectorIdxWidth(const DataLayout &DL) const
Returns the type to be used for the index operand vector operations.
MVT getTypeToPromoteTo(unsigned Op, MVT VT) const
If the action for this operation is to promote, this method returns the ValueType to promote to.
virtual bool generateFMAsInMachineCombiner(EVT VT, CodeGenOptLevel OptLevel) const
virtual LoadInst * lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *RMWI) const
On some platforms, an AtomicRMW that never actually modifies the value (such as fetch_add of 0) can b...
virtual bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AddrSpace, Instruction *I=nullptr) const
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
virtual bool hasPairedLoad(EVT, Align &) const
Return true if the target supplies and combines to a paired load two loaded values of type LoadedType...
virtual bool convertSelectOfConstantsToMath(EVT VT) const
Return true if a select of constants (select Cond, C1, C2) should be transformed into simple math ops...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
unsigned getVectorTypeBreakdown(LLVMContext &Context, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Vector types are broken down into some number of legal first class types.
virtual bool optimizeExtendOrTruncateConversion(Instruction *I, Loop *L, const TargetTransformInfo &TTI) const
Try to optimize extending or truncating conversion instructions (like zext, trunc,...
virtual MVT getVPExplicitVectorLengthTy() const
Returns the type to be used for the EVL/AVL operand of VP nodes: ISD::VP_ADD, ISD::VP_SUB,...
std::pair< LegalizeTypeAction, EVT > LegalizeKind
LegalizeKind holds the legalization kind that needs to happen to EVT in order to type-legalize it.
TargetLoweringBase & operator=(const TargetLoweringBase &)=delete
MulExpansionKind
Enum that specifies when a multiplication should be expanded.
static ISD::NodeType getExtendForContent(BooleanContent Content)
virtual bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT from min(max(fptoi)) satur...
virtual bool lowerDeinterleaveIntrinsicToLoad(Instruction *Load, Value *Mask, IntrinsicInst *DI) const
Lower a deinterleave intrinsic to a target specific load intrinsic.
virtual bool supportKCFIBundles() const
Return true if the target supports kcfi operand bundles.
virtual ConstraintWeight getMultipleConstraintMatchWeight(AsmOperandInfo &info, int maIndex) const
Examine constraint type and operand type and determine a weight value.
SmallVector< ConstraintPair > ConstraintGroup
virtual SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps, bool &UseOneConstNR, bool Reciprocal) const
Hooks for building estimates in place of slower divisions and square roots.
virtual bool isDesirableToCommuteWithShift(const MachineInstr &MI, bool IsAfterLegal) const
GlobalISel - return true if it is profitable to move this shift by a constant amount through its oper...
virtual bool supportPtrAuthBundles() const
Return true if the target supports ptrauth operand bundles.
virtual void ReplaceNodeResults(SDNode *, SmallVectorImpl< SDValue > &, SelectionDAG &) const
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
virtual bool isUsedByReturnOnly(SDNode *, SDValue &) const
Return true if result of the specified node is used by a return node only.
virtual bool supportSwiftError() const
Return true if the target supports swifterror attribute.
virtual SDValue visitMaskedLoad(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, MachineMemOperand *MMO, SDValue &NewLoad, SDValue Ptr, SDValue PassThru, SDValue Mask) const
virtual SDValue emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val, const SDLoc &DL) const
SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, unsigned Depth=0) const
This is the helper function to return the newly negated expression if the cost is not expensive.
virtual bool isReassocProfitable(SelectionDAG &DAG, SDValue N0, SDValue N1) const
virtual EVT getTypeForExtReturn(LLVMContext &Context, EVT VT, ISD::NodeType) const
Return the type that should be used to zero or sign extend a zeroext/signext integer return value.
SDValue getCheaperOrNeutralNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, const NegatibleCost CostThreshold=NegatibleCost::Neutral, unsigned Depth=0) const
virtual Register getRegisterByName(const char *RegName, LLT Ty, const MachineFunction &MF) const
Return the register ID of the name passed in.
virtual InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const
virtual bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const
std::vector< AsmOperandInfo > AsmOperandInfoVector
virtual bool isTargetCanonicalConstantNode(SDValue Op) const
Returns true if the given Opc is considered a canonical constant for the target, which should not be ...
virtual bool isTargetCanonicalSelect(SDNode *N) const
Return true if the given select/vselect should be considered canonical and not be transformed.
SDValue getCheaperNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, unsigned Depth=0) const
This is the helper function to return the newly negated expression only when the cost is cheaper.
virtual SDValue prepareVolatileOrAtomicLoad(SDValue Chain, const SDLoc &DL, SelectionDAG &DAG) const
This callback is used to prepare for a volatile or atomic load.
virtual SDValue lowerEHPadEntry(SDValue Chain, const SDLoc &DL, SelectionDAG &DAG) const
Optional target hook to add target-specific actions when entering EH pad blocks.
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
virtual SDValue unwrapAddress(SDValue N) const
virtual bool splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, std::optional< CallingConv::ID > CC) const
Target-specific splitting of values into parts that fit a register storing a legal type.
virtual bool IsDesirableToPromoteOp(SDValue, EVT &) const
This method query the target whether it is beneficial for dag combiner to promote the specified node.
virtual SDValue joinRegisterPartsIntoValue(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, std::optional< CallingConv::ID > CC) const
Target-specific combining of register parts into its original value.
virtual void insertCopiesSplitCSR(MachineBasicBlock *Entry, const SmallVectorImpl< MachineBasicBlock * > &Exits) const
Insert explicit copies in entry and exit blocks.
virtual SDValue LowerCall(CallLoweringInfo &, SmallVectorImpl< SDValue > &) const
This hook must be implemented to lower calls into the specified DAG.
virtual bool isTypeDesirableForOp(unsigned, EVT VT) const
Return true if the target has native support for the specified value type and it is 'desirable' to us...
~TargetLowering() override
TargetLowering & operator=(const TargetLowering &)=delete
virtual bool isDesirableToPullExtFromShl(const MachineInstr &MI) const
GlobalISel - return true if it's profitable to perform the combine: shl ([sza]ext x),...
bool isPositionIndependent() const
std::pair< StringRef, TargetLowering::ConstraintType > ConstraintPair
virtual SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, NegatibleCost &Cost, unsigned Depth=0) const
Return the newly negated expression if the cost is not expensive and set the cost in Cost to indicate...
virtual ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const
Examine constraint string and operand type and determine a weight value.
virtual bool isIndexingLegal(MachineInstr &MI, Register Base, Register Offset, bool IsPre, MachineRegisterInfo &MRI) const
Returns true if the specified base+offset is a legal indexed addressing mode for this target.
ConstraintGroup getConstraintPreferences(AsmOperandInfo &OpInfo) const
Given an OpInfo with list of constraints codes as strings, return a sorted Vector of pairs of constra...
virtual void initializeSplitCSR(MachineBasicBlock *Entry) const
Perform necessary initialization to handle a subset of CSRs explicitly via copies.
virtual bool isSDNodeSourceOfDivergence(const SDNode *N, FunctionLoweringInfo *FLI, UniformityInfo *UA) const
virtual SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps) const
Return a reciprocal estimate value for the input operand.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
virtual bool isSDNodeAlwaysUniform(const SDNode *N) const
virtual bool isDesirableToCommuteXorWithShift(const SDNode *N) const
Return true if it is profitable to combine an XOR of a logical shift to create a logical shift of NOT...
TargetLowering(const TargetLowering &)=delete
virtual bool shouldSimplifyDemandedVectorElts(SDValue Op, const TargetLoweringOpt &TLO) const
Return true if the target supports simplifying demanded vector elements by converting them to undefs.
virtual SDValue LowerFormalArguments(SDValue, CallingConv::ID, bool, const SmallVectorImpl< ISD::InputArg > &, const SDLoc &, SelectionDAG &, SmallVectorImpl< SDValue > &) const
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, const CallBase &Call) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
virtual SDValue getSqrtResultForDenormInput(SDValue Operand, SelectionDAG &DAG) const
Return a target-dependent result if the input operand is not suitable for use with a square root esti...
virtual bool getPostIndexedAddressParts(SDNode *, SDNode *, SDValue &, SDValue &, ISD::MemIndexedMode &, SelectionDAG &) const
Returns true by value, base pointer and offset pointer and addressing mode by reference if this node ...
virtual bool shouldSplitFunctionArgumentsAsLittleEndian(const DataLayout &DL) const
For most targets, an LLVM type must be broken down into multiple smaller types.
virtual ArrayRef< MCPhysReg > getRoundingControlRegisters() const
Returns a 0 terminated array of rounding control registers that can be attached into strict FP call.
virtual SDValue LowerReturn(SDValue, CallingConv::ID, bool, const SmallVectorImpl< ISD::OutputArg > &, const SmallVectorImpl< SDValue > &, const SDLoc &, SelectionDAG &) const
This hook must be implemented to lower outgoing return values, described by the Outs array,...
virtual bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg, const DataLayout &DL) const
For some targets, an LLVM struct type must be broken down into multiple simple types,...
virtual bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const
Return true if it is profitable to move this shift by a constant amount through its operand,...
virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG=nullptr) const
Determines the constraint code and constraint type to use for the specific AsmOperandInfo,...
virtual SDValue visitMaskedStore(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, MachineMemOperand *MMO, SDValue Ptr, SDValue Val, SDValue Mask) const
virtual const MCExpr * LowerCustomJumpTableEntry(const MachineJumpTableInfo *, const MachineBasicBlock *, unsigned, MCContext &) const
virtual bool useLoadStackGuardNode(const Module &M) const
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
virtual unsigned combineRepeatedFPDivisors() const
Indicate whether this target prefers to combine FDIVs with the same divisor.
virtual AndOrSETCCFoldKind isDesirableToCombineLogicOpOfSETCC(const SDNode *LogicOp, const SDNode *SETCC0, const SDNode *SETCC1) const
virtual void HandleByVal(CCState *, unsigned &, Align) const
Target-specific cleanup for formal ByVal parameters.
virtual const MCPhysReg * getScratchRegisters(CallingConv::ID CC) const
Returns a 0 terminated array of registers that can be safely used as scratch registers.
virtual bool getPreIndexedAddressParts(SDNode *, SDValue &, SDValue &, ISD::MemIndexedMode &, SelectionDAG &) const
Returns true by value, base pointer and offset pointer and addressing mode by reference if the node's...
virtual FastISel * createFastISel(FunctionLoweringInfo &, const TargetLibraryInfo *) const
This method returns a target specific FastISel object, or null if the target does not support "fast" ...
virtual bool supportSplitCSR(MachineFunction *MF) const
Return true if the target supports that a subset of CSRs for the given machine function is handled ex...
virtual bool isReassocProfitable(MachineRegisterInfo &MRI, Register N0, Register N1) const
virtual bool mayBeEmittedAsTailCall(const CallInst *) const
Return true if the target may be able emit the call instruction as a tail call.
virtual bool isInlineAsmTargetBranch(const SmallVectorImpl< StringRef > &AsmStrs, unsigned OpNo) const
On x86, return true if the operand with index OpNo is a CALL or JUMP instruction, which can use eithe...
virtual MVT getJumpTableRegTy(const DataLayout &DL) const
virtual void markLibCallAttributes(MachineFunction *MF, unsigned CC, ArgListTy &Args) const
virtual bool checkForPhysRegDependency(SDNode *Def, SDNode *User, unsigned Op, const TargetRegisterInfo *TRI, const TargetInstrInfo *TII, MCRegister &PhysReg, int &Cost) const
Allows the target to handle physreg-carried dependency in target-specific way.
virtual bool CanLowerReturn(CallingConv::ID, MachineFunction &, bool, const SmallVectorImpl< ISD::OutputArg > &, LLVMContext &, const Type *RetTy) const
This hook should be implemented to check whether the return values described by the Outs array can fi...
virtual bool isXAndYEqZeroPreferableToXAndYEqY(ISD::CondCode, EVT) const
virtual bool isDesirableToTransformToIntegerOp(unsigned, EVT) const
Return true if it is profitable for dag combiner to transform a floating point op of specified opcode...
Primary interface to the complete machine description for the target machine.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
Target - Wrapper for Target specific information.
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:45
bool isPointerTy() const
True if this is an instance of PointerType.
Definition Type.h:267
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
Definition Type.h:184
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:240
This is the common base class for vector predication intrinsics.
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:256
CallInst * Call
#define UINT64_MAX
Definition DataTypes.h:77
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition CallingConv.h:24
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition CallingConv.h:41
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition ISDOpcodes.h:41
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition ISDOpcodes.h:270
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition ISDOpcodes.h:387
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
Definition ISDOpcodes.h:289
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition ISDOpcodes.h:515
@ ADD
Simple integer binary arithmetic operators.
Definition ISDOpcodes.h:259
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition ISDOpcodes.h:393
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition ISDOpcodes.h:835
@ FADD
Simple binary floating point operators.
Definition ISDOpcodes.h:410
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
Definition ISDOpcodes.h:400
@ SIGN_EXTEND
Conversion operators.
Definition ISDOpcodes.h:826
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
Definition ISDOpcodes.h:706
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition ISDOpcodes.h:369
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition ISDOpcodes.h:663
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition ISDOpcodes.h:343
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition ISDOpcodes.h:695
@ SHL
Shift and rotation operations.
Definition ISDOpcodes.h:756
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition ISDOpcodes.h:832
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition ISDOpcodes.h:718
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition ISDOpcodes.h:406
@ STRICT_FP_TO_UINT
Definition ISDOpcodes.h:471
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition ISDOpcodes.h:470
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition ISDOpcodes.h:908
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition ISDOpcodes.h:730
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
Definition ISDOpcodes.h:701
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
Definition ISDOpcodes.h:299
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
Definition ISDOpcodes.h:672
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition ISDOpcodes.h:360
@ ABDS
ABDS/ABDU - Absolute difference - Return the absolute difference between two numbers interpreted as s...
Definition ISDOpcodes.h:713
static const int LAST_LOADEXT_TYPE
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
@ System
Synchronized with respect to all concurrently executing threads.
Definition LLVMContext.h:58
This namespace contains all of the command line option processing machinery.
Definition CommandLine.h:53
This is an optimization pass for GlobalISel generic memory operations.
GenericUniformityInfo< SSAContext > UniformityInfo
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
Definition Threading.h:262
@ Offset
Definition DWP.cpp:477
void fill(R &&Range, T &&Value)
Provide wrappers to std::fill which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1725
LLVM_ABI void GetReturnInfo(CallingConv::ID CC, Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags,...
InstructionCost Cost
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:644
bool isAligned(Align Lhs, uint64_t SizeInBytes)
Checks that SizeInBytes is a multiple of the alignment.
Definition Alignment.h:134
void * PointerTy
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:348
LLVM_ABI bool isConstTrueVal(const TargetLowering &TLI, int64_t Val, bool IsVector, bool IsFP)
Returns true if given the TargetLowering's boolean contents information, the value Val contains a tru...
Definition Utils.cpp:1654
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:167
CodeGenOptLevel
Code generation optimization level.
Definition CodeGen.h:82
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:548
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
AtomicOrdering
Atomic ordering for LLVM's memory model.
LLVM_ABI EVT getApproximateEVTForLLT(LLT Ty, LLVMContext &Ctx)
TargetTransformInfo TTI
CombineLevel
Definition DAGCombine.h:15
@ AfterLegalizeDAG
Definition DAGCombine.h:19
@ AfterLegalizeVectorOps
Definition DAGCombine.h:18
@ BeforeLegalizeTypes
Definition DAGCombine.h:16
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition MCRegister.h:21
bool isInTailCallPosition(const CallBase &Call, const TargetMachine &TM, bool ReturnsFirstArg=false)
Test if the given instruction is in a position to be optimized with a tail-call.
Definition Analysis.cpp:543
DWARFExpression::Operation Op
LLVM_ABI bool isConstFalseVal(const TargetLowering &TLI, int64_t Val, bool IsVector, bool IsFP)
Definition Utils.cpp:1667
constexpr unsigned BitWidth
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1847
static cl::opt< int > CostThreshold("sbvec-cost-threshold", cl::init(0), cl::Hidden, cl::desc("Vectorization cost threshold."))
int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition bit.h:154
Implement std::hash so that hash_code can be used in STL containers.
Definition BitVector.h:870
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
Definition Alignment.h:77
Represent subnormal handling kind for floating point instruction inputs and outputs.
Extended Value Type.
Definition ValueTypes.h:35
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition ValueTypes.h:137
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition ValueTypes.h:74
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition ValueTypes.h:300
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition ValueTypes.h:147
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition ValueTypes.h:373
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition ValueTypes.h:243
static LLVM_ABI EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition ValueTypes.h:316
bool isVector() const
Return true if this is a vector value type.
Definition ValueTypes.h:168
bool isExtended() const
Test if the given EVT is extended (as opposed to being simple).
Definition ValueTypes.h:142
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition ValueTypes.h:157
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition ValueTypes.h:152
ConstraintInfo()=default
Default constructor.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition Alignment.h:106
bool isDstAligned(Align AlignCheck) const
bool allowOverlap() const
bool isFixedDstAlign() const
uint64_t size() const
static MemOp Set(uint64_t Size, bool DstAlignCanChange, Align DstAlign, bool IsZeroMemset, bool IsVolatile)
Align getDstAlign() const
bool isMemcpyStrSrc() const
bool isAligned(Align AlignCheck) const
static MemOp Copy(uint64_t Size, bool DstAlignCanChange, Align DstAlign, Align SrcAlign, bool IsVolatile, bool MemcpyStrSrc=false)
bool isSrcAligned(Align AlignCheck) const
bool isMemset() const
bool isMemcpy() const
bool isMemcpyWithFixedDstAlign() const
bool isZeroMemset() const
Align getSrcAlign() const
static StringRef getLibcallImplName(RTLIB::LibcallImpl CallImpl)
Get the libcall routine name for the specified libcall implementation.
These are IR-level optimization flags that may be propagated to SDNodes.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
std::optional< unsigned > fallbackAddressSpace
PointerUnion< const Value *, const PseudoSourceValue * > ptrVal
This contains information for each constraint that we are lowering.
AsmOperandInfo(InlineAsm::ConstraintInfo Info)
Copy constructor for copying from a ConstraintInfo.
MVT ConstraintVT
The ValueType for the operand value.
TargetLowering::ConstraintType ConstraintType
Information about the constraint code, e.g.
std::string ConstraintCode
This contains the actual string for the code, like "m".
Value * CallOperandVal
If this is the result output operand or a clobber, this is null, otherwise it is the incoming operand...
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setConvergent(bool Value=true)
CallLoweringInfo & setIsPostTypeLegalization(bool Value=true)
CallLoweringInfo & setCallee(Type *ResultType, FunctionType *FTy, SDValue Target, ArgListTy &&ArgsList, const CallBase &Call)
CallLoweringInfo & setCFIType(const ConstantInt *Type)
CallLoweringInfo & setInRegister(bool Value=true)
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
SmallVector< ISD::InputArg, 32 > Ins
CallLoweringInfo & setVarArg(bool Value=true)
Type * OrigRetTy
Original unlegalized return type.
std::optional< PtrAuthInfo > PAI
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setZExtResult(bool Value=true)
CallLoweringInfo & setIsPatchPoint(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, Type *OrigResultType, SDValue Target, ArgListTy &&ArgsList)
CallLoweringInfo & setTailCall(bool Value=true)
CallLoweringInfo & setIsPreallocated(bool Value=true)
CallLoweringInfo & setSExtResult(bool Value=true)
CallLoweringInfo & setNoReturn(bool Value=true)
CallLoweringInfo & setConvergenceControlToken(SDValue Token)
SmallVector< ISD::OutputArg, 32 > Outs
Type * RetTy
Same as OrigRetTy, or partially legalized for soft float libcalls.
CallLoweringInfo & setChain(SDValue InChain)
CallLoweringInfo & setPtrAuth(PtrAuthInfo Value)
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList, AttributeSet ResultAttrs={})
DAGCombinerInfo(SelectionDAG &dag, CombineLevel level, bool cl, void *dc)
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setIsPostTypeLegalization(bool Value=true)
MakeLibCallOptions & setDiscardResult(bool Value=true)
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT)
MakeLibCallOptions & setIsSigned(bool Value=true)
MakeLibCallOptions & setNoReturn(bool Value=true)
MakeLibCallOptions & setOpsTypeOverrides(ArrayRef< Type * > OpsTypes)
Override the argument type for an operand.
This structure contains the information necessary for lowering pointer-authenticating indirect calls.
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
TargetLoweringOpt(SelectionDAG &InDAG, bool LT, bool LO)