44constexpr unsigned MinBWOps = 3;
47constexpr unsigned MinVPOps = 5;
51bool isTargetMD(
const MDNode *ProfData,
const char *Name,
unsigned MinOps) {
54 if (!ProfData || !Name || MinOps < 2)
65 return ProfDataName->getString() ==
Name;
69 typename =
typename std::enable_if<std::is_arithmetic_v<T>>>
70static void extractFromBranchWeightMD(
const MDNode *ProfileData,
76 assert(WeightsIdx < NOps &&
"Weights Index must be less than NOps.");
77 Weights.
resize(NOps - WeightsIdx);
79 for (
unsigned Idx = WeightsIdx,
E = NOps; Idx !=
E; ++Idx) {
82 assert(Weight &&
"Malformed branch_weight in MD_prof node");
84 "Too many bits for MD_prof branch_weight");
108#
if defined(LLVM_ENABLE_PROFCHECK)
119 "synthetic_function_entry_count";
124 return I.hasMetadata(LLVMContext::MD_prof);
136 auto *ProfileData =
I.getMetadata(LLVMContext::MD_prof);
141 auto *ProfileData =
I.getMetadata(LLVMContext::MD_prof);
155 auto *ProfileData =
I.getMetadata(LLVMContext::MD_prof);
166 assert(ProfDataName ==
nullptr ||
168 return ProfDataName !=
nullptr;
180 auto *ProfileData =
I.getMetadata(LLVMContext::MD_prof);
195 extractFromBranchWeightMD(ProfileData, Weights);
200 extractFromBranchWeightMD(ProfileData, Weights);
207 extractFromBranchWeightMD(ProfileData, Weights);
213 auto *ProfileData =
I.getMetadata(LLVMContext::MD_prof);
219 assert((
I.getOpcode() == Instruction::Br ||
220 I.getOpcode() == Instruction::Select) &&
221 "Looking for branch weights on something besides branch, select, or "
225 auto *ProfileData =
I.getMetadata(LLVMContext::MD_prof);
229 if (Weights.
size() > 2)
232 TrueVal = Weights[0];
233 FalseVal = Weights[1];
250 TotalVal += V->getValue().getZExtValue();
272 LLVMContext::MD_prof,
274 {MDB.createString(MDProfLabels::UnknownBranchWeightsMarker),
275 MDB.createString(PassName)}));
280 if (std::optional<Function::ProfileCount> EC =
F.getEntryCount();
281 EC && EC->getCount() > 0)
288 LLVMContext::MD_prof,
290 {MDB.createString(MDProfLabels::UnknownBranchWeightsMarker),
291 MDB.createString(PassName)}));
301 auto *MD =
I.getMetadata(LLVMContext::MD_prof);
308 bool IsExpected,
bool ElideAllZero) {
311 I.setMetadata(LLVMContext::MD_prof,
nullptr);
317 I.setMetadata(LLVMContext::MD_prof, BranchWeights);
321 bool IsExpected,
bool ElideAllZero) {
326 std::optional<uint64_t> KnownMaxCount) {
327 uint64_t MaxCount = KnownMaxCount.has_value() ? KnownMaxCount.value()
329 assert(MaxCount > 0 &&
"Bad max count");
332 for (
const auto &ECI : Weights)
334 return DownscaledWeights;
338 assert(
T != 0 &&
"Caller should guarantee");
339 auto *ProfileData =
I.getMetadata(LLVMContext::MD_prof);
340 if (ProfileData ==
nullptr)
357 APInt APS(128, S), APT(128,
T);
370 for (
unsigned Idx = 1; Idx < ProfileData->
getNumOperands(); Idx += 2) {
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file contains the declarations for profiling metadata utility functions.
This file defines the SmallVector class.
static const char PassName[]
Class for arbitrary precision integers.
LLVM_ABI APInt udiv(const APInt &RHS) const
Unsigned division operation.
unsigned getActiveBits() const
Compute the number of active bits in the value.
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const
If this value is smaller than the specified limit, return it, otherwise return the limit value.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
This is the shared class of boolean and integer constants.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
const APInt & getValue() const
Return the constant as an APInt value reference.
This is an important class for using LLVM in a threaded context.
LLVM_ABI ConstantAsMetadata * createConstant(Constant *C)
Return the given constant as metadata.
LLVM_ABI MDNode * createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight, bool IsExpected=false)
Return metadata containing two branch weights.
const MDOperand & getOperand(unsigned I) const
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
unsigned getNumOperands() const
Return number of MDNode operands.
bool equalsStr(StringRef Str) const
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
LLVM Value Representation.
@ C
The default llvm calling convention, compatible with C.
initializer< Ty > init(const Ty &Val)
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract(Y &&MD)
Extract a Value from Metadata, if any.
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
This is an optimization pass for GlobalISel generic memory operations.
cl::opt< bool > ElideAllZeroBranchWeights("elide-all-zero-branch-weights", cl::init(true))
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
LLVM_ABI bool extractProfTotalWeight(const MDNode *ProfileData, uint64_t &TotalWeights)
Retrieve the total of all weights from MD_prof data.
LLVM_ABI unsigned getBranchWeightOffset(const MDNode *ProfileData)
Return the offset to the first branch weight data.
LLVM_ABI bool isBranchWeightMD(const MDNode *ProfileData)
Checks if an MDNode contains Branch Weight Metadata.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
LLVM_ABI void setExplicitlyUnknownBranchWeightsIfProfiled(Instruction &I, Function &F, StringRef PassName)
Like setExplicitlyUnknownBranchWeights(...), but only sets unknown branch weights in the new instruct...
LLVM_ABI bool isExplicitlyUnknownProfileMetadata(const MDNode &MD)
LLVM_ABI MDNode * getBranchWeightMDNode(const Instruction &I)
Get the branch weights metadata node.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
LLVM_ABI void setExplicitlyUnknownBranchWeights(Instruction &I, StringRef PassName)
Specify that the branch weights for this terminator cannot be known at compile time.
LLVM_ABI bool hasBranchWeightOrigin(const Instruction &I)
Check if Branch Weight Metadata has an "expected" field from an llvm.expect* intrinsic.
LLVM_ABI void setBranchWeights(Instruction &I, ArrayRef< uint32_t > Weights, bool IsExpected, bool ElideAllZero=false)
Create a new branch_weights metadata node and add or overwrite a prof metadata reference to instructi...
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
LLVM_ABI MDNode * getValidBranchWeightMDNode(const Instruction &I)
Get the valid branch weights metadata node.
LLVM_ABI bool hasValidBranchWeightMD(const Instruction &I)
Checks if an instructions has valid Branch Weight Metadata.
FunctionAddr VTableAddr Count
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
LLVM_ABI void setExplicitlyUnknownFunctionEntryCount(Function &F, StringRef PassName)
Analogous to setExplicitlyUnknownBranchWeights, but for functions and their entry counts.
LLVM_ABI bool isValueProfileMD(const MDNode *ProfileData)
Checks if an MDNode contains value profiling Metadata.
LLVM_ABI unsigned getNumBranchWeights(const MDNode &ProfileData)
LLVM_ABI void extractFromBranchWeightMD32(const MDNode *ProfileData, SmallVectorImpl< uint32_t > &Weights)
Faster version of extractBranchWeights() that skips checks and must only be called with "branch_weigh...
LLVM_ABI bool hasExplicitlyUnknownBranchWeights(const Instruction &I)
LLVM_ABI bool hasProfMD(const Instruction &I)
Checks if an Instruction has MD_prof Metadata.
auto max_element(R &&Range)
Provide wrappers to std::max_element which take ranges instead of having to pass begin/end explicitly...
LLVM_ABI bool extractBranchWeights(const MDNode *ProfileData, SmallVectorImpl< uint32_t > &Weights)
Extract branch weights from MD_prof metadata.
LLVM_ABI const char * LLVMLoopEstimatedTripCount
Profile-based loop metadata that should be accessed only by using llvm::getLoopEstimatedTripCount and...
uint32_t scaleBranchCount(uint64_t Count, uint64_t Scale)
Scale an individual branch count.
LLVM_ABI bool hasBranchWeightMD(const Instruction &I)
Checks if an instructions has Branch Weight Metadata.
LLVM_ABI void setFittedBranchWeights(Instruction &I, ArrayRef< uint64_t > Weights, bool IsExpected, bool ElideAllZero=false)
Variant of setBranchWeights where the Weights will be fit first to uint32_t by shifting right.
uint64_t calculateCountScale(uint64_t MaxCount)
Calculate what to divide by to scale counts.
LLVM_ABI SmallVector< uint32_t > downscaleWeights(ArrayRef< uint64_t > Weights, std::optional< uint64_t > KnownMaxCount=std::nullopt)
downscale the given weights preserving the ratio.
const uint64_t NOMORE_ICP_MAGICNUM
Magic number in the value profile metadata showing a target has been promoted for the instruction and...
LLVM_ABI void scaleProfData(Instruction &I, uint64_t S, uint64_t T)
Scaling the profile data attached to 'I' using the ratio of S/T.
static bool hasCountTypeMD(const Instruction &I)
LLVM_ABI void extractFromBranchWeightMD64(const MDNode *ProfileData, SmallVectorImpl< uint64_t > &Weights)
Faster version of extractBranchWeights() that skips checks and must only be called with "branch_weigh...
static LLVM_ABI const char * ExpectedBranchWeights
static LLVM_ABI const char * SyntheticFunctionEntryCount
static LLVM_ABI const char * BranchWeights
static LLVM_ABI const char * FunctionEntryCount
static LLVM_ABI const char * UnknownBranchWeightsMarker
static LLVM_ABI const char * ValueProfile