Thanks to visit codestin.com
Credit goes to llvm.org

LLVM 22.0.0git
ProfDataUtils.cpp
Go to the documentation of this file.
1//===- ProfDataUtils.cpp - Utility functions for MD_prof Metadata ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements utilities for working with Profiling Metadata.
10//
11//===----------------------------------------------------------------------===//
12
14
15#include "llvm/ADT/STLExtras.h"
17#include "llvm/IR/Constants.h"
18#include "llvm/IR/Function.h"
20#include "llvm/IR/LLVMContext.h"
21#include "llvm/IR/MDBuilder.h"
22#include "llvm/IR/Metadata.h"
24
25using namespace llvm;
26
27namespace {
28
29// MD_prof nodes have the following layout
30//
31// In general:
32// { String name, Array of i32 }
33//
34// In terms of Types:
35// { MDString, [i32, i32, ...]}
36//
37// Concretely for Branch Weights
38// { "branch_weights", [i32 1, i32 10000]}
39//
40// We maintain some constants here to ensure that we access the branch weights
41// correctly, and can change the behavior in the future if the layout changes
42
43// the minimum number of operands for MD_prof nodes with branch weights
44constexpr unsigned MinBWOps = 3;
45
46// the minimum number of operands for MD_prof nodes with value profiles
47constexpr unsigned MinVPOps = 5;
48
49// We may want to add support for other MD_prof types, so provide an abstraction
50// for checking the metadata type.
51bool isTargetMD(const MDNode *ProfData, const char *Name, unsigned MinOps) {
52 // TODO: This routine may be simplified if MD_prof used an enum instead of a
53 // string to differentiate the types of MD_prof nodes.
54 if (!ProfData || !Name || MinOps < 2)
55 return false;
56
57 unsigned NOps = ProfData->getNumOperands();
58 if (NOps < MinOps)
59 return false;
60
61 auto *ProfDataName = dyn_cast<MDString>(ProfData->getOperand(0));
62 if (!ProfDataName)
63 return false;
64
65 return ProfDataName->getString() == Name;
66}
67
68template <typename T,
69 typename = typename std::enable_if<std::is_arithmetic_v<T>>>
70static void extractFromBranchWeightMD(const MDNode *ProfileData,
71 SmallVectorImpl<T> &Weights) {
72 assert(isBranchWeightMD(ProfileData) && "wrong metadata");
73
74 unsigned NOps = ProfileData->getNumOperands();
75 unsigned WeightsIdx = getBranchWeightOffset(ProfileData);
76 assert(WeightsIdx < NOps && "Weights Index must be less than NOps.");
77 Weights.resize(NOps - WeightsIdx);
78
79 for (unsigned Idx = WeightsIdx, E = NOps; Idx != E; ++Idx) {
80 ConstantInt *Weight =
82 assert(Weight && "Malformed branch_weight in MD_prof node");
83 assert(Weight->getValue().getActiveBits() <= (sizeof(T) * 8) &&
84 "Too many bits for MD_prof branch_weight");
85 Weights[Idx - WeightsIdx] = Weight->getZExtValue();
86 }
87}
88
89/// Push the weights right to fit in uint32_t.
90static SmallVector<uint32_t> fitWeights(ArrayRef<uint64_t> Weights) {
92 Ret.reserve(Weights.size());
93 uint64_t Max = *llvm::max_element(Weights);
94 if (Max > UINT_MAX) {
95 unsigned Offset = 32 - llvm::countl_zero(Max);
96 for (const uint64_t &Value : Weights)
97 Ret.push_back(static_cast<uint32_t>(Value >> Offset));
98 } else {
99 append_range(Ret, Weights);
100 }
101 return Ret;
102}
103
104} // namespace
105
106namespace llvm {
107cl::opt<bool> ElideAllZeroBranchWeights("elide-all-zero-branch-weights",
108#if defined(LLVM_ENABLE_PROFCHECK)
109 cl::init(false)
110#else
111 cl::init(true)
112#endif
113);
114const char *MDProfLabels::BranchWeights = "branch_weights";
115const char *MDProfLabels::ExpectedBranchWeights = "expected";
116const char *MDProfLabels::ValueProfile = "VP";
117const char *MDProfLabels::FunctionEntryCount = "function_entry_count";
119 "synthetic_function_entry_count";
120const char *MDProfLabels::UnknownBranchWeightsMarker = "unknown";
121const char *LLVMLoopEstimatedTripCount = "llvm.loop.estimated_trip_count";
122
123bool hasProfMD(const Instruction &I) {
124 return I.hasMetadata(LLVMContext::MD_prof);
125}
126
127bool isBranchWeightMD(const MDNode *ProfileData) {
128 return isTargetMD(ProfileData, MDProfLabels::BranchWeights, MinBWOps);
129}
130
131bool isValueProfileMD(const MDNode *ProfileData) {
132 return isTargetMD(ProfileData, MDProfLabels::ValueProfile, MinVPOps);
133}
134
136 auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);
137 return isBranchWeightMD(ProfileData);
138}
139
140static bool hasCountTypeMD(const Instruction &I) {
141 auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);
142 // Value profiles record count-type information.
143 if (isValueProfileMD(ProfileData))
144 return true;
145 // Conservatively assume non CallBase instruction only get taken/not-taken
146 // branch probability, so not interpret them as count.
147 return isa<CallBase>(I) && !isBranchWeightMD(ProfileData);
148}
149
153
155 auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);
156 return hasBranchWeightOrigin(ProfileData);
157}
158
159bool hasBranchWeightOrigin(const MDNode *ProfileData) {
160 if (!isBranchWeightMD(ProfileData))
161 return false;
162 auto *ProfDataName = dyn_cast<MDString>(ProfileData->getOperand(1));
163 // NOTE: if we ever have more types of branch weight provenance,
164 // we need to check the string value is "expected". For now, we
165 // supply a more generic API, and avoid the spurious comparisons.
166 assert(ProfDataName == nullptr ||
167 ProfDataName->getString() == MDProfLabels::ExpectedBranchWeights);
168 return ProfDataName != nullptr;
169}
170
171unsigned getBranchWeightOffset(const MDNode *ProfileData) {
172 return hasBranchWeightOrigin(ProfileData) ? 2 : 1;
173}
174
175unsigned getNumBranchWeights(const MDNode &ProfileData) {
176 return ProfileData.getNumOperands() - getBranchWeightOffset(&ProfileData);
177}
178
180 auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);
181 if (!isBranchWeightMD(ProfileData))
182 return nullptr;
183 return ProfileData;
184}
185
187 auto *ProfileData = getBranchWeightMDNode(I);
188 if (ProfileData && getNumBranchWeights(*ProfileData) == I.getNumSuccessors())
189 return ProfileData;
190 return nullptr;
191}
192
193void extractFromBranchWeightMD32(const MDNode *ProfileData,
194 SmallVectorImpl<uint32_t> &Weights) {
195 extractFromBranchWeightMD(ProfileData, Weights);
196}
197
198void extractFromBranchWeightMD64(const MDNode *ProfileData,
199 SmallVectorImpl<uint64_t> &Weights) {
200 extractFromBranchWeightMD(ProfileData, Weights);
201}
202
203bool extractBranchWeights(const MDNode *ProfileData,
204 SmallVectorImpl<uint32_t> &Weights) {
205 if (!isBranchWeightMD(ProfileData))
206 return false;
207 extractFromBranchWeightMD(ProfileData, Weights);
208 return true;
209}
210
212 SmallVectorImpl<uint32_t> &Weights) {
213 auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);
214 return extractBranchWeights(ProfileData, Weights);
215}
216
218 uint64_t &FalseVal) {
219 assert((I.getOpcode() == Instruction::Br ||
220 I.getOpcode() == Instruction::Select) &&
221 "Looking for branch weights on something besides branch, select, or "
222 "switch");
223
225 auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);
226 if (!extractBranchWeights(ProfileData, Weights))
227 return false;
228
229 if (Weights.size() > 2)
230 return false;
231
232 TrueVal = Weights[0];
233 FalseVal = Weights[1];
234 return true;
235}
236
237bool extractProfTotalWeight(const MDNode *ProfileData, uint64_t &TotalVal) {
238 TotalVal = 0;
239 if (!ProfileData)
240 return false;
241
242 auto *ProfDataName = dyn_cast<MDString>(ProfileData->getOperand(0));
243 if (!ProfDataName)
244 return false;
245
246 if (ProfDataName->getString() == MDProfLabels::BranchWeights) {
247 unsigned Offset = getBranchWeightOffset(ProfileData);
248 for (unsigned Idx = Offset; Idx < ProfileData->getNumOperands(); ++Idx) {
249 auto *V = mdconst::extract<ConstantInt>(ProfileData->getOperand(Idx));
250 TotalVal += V->getValue().getZExtValue();
251 }
252 return true;
253 }
254
255 if (ProfDataName->getString() == MDProfLabels::ValueProfile &&
256 ProfileData->getNumOperands() > 3) {
257 TotalVal = mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(2))
258 ->getValue()
259 .getZExtValue();
260 return true;
261 }
262 return false;
263}
264
266 return extractProfTotalWeight(I.getMetadata(LLVMContext::MD_prof), TotalVal);
267}
268
270 MDBuilder MDB(I.getContext());
271 I.setMetadata(
272 LLVMContext::MD_prof,
273 MDNode::get(I.getContext(),
274 {MDB.createString(MDProfLabels::UnknownBranchWeightsMarker),
275 MDB.createString(PassName)}));
276}
277
280 if (std::optional<Function::ProfileCount> EC = F.getEntryCount();
281 EC && EC->getCount() > 0)
283}
284
286 MDBuilder MDB(F.getContext());
287 F.setMetadata(
288 LLVMContext::MD_prof,
289 MDNode::get(F.getContext(),
290 {MDB.createString(MDProfLabels::UnknownBranchWeightsMarker),
291 MDB.createString(PassName)}));
292}
293
295 if (MD.getNumOperands() != 2)
296 return false;
298}
299
301 auto *MD = I.getMetadata(LLVMContext::MD_prof);
302 if (!MD)
303 return false;
305}
306
308 bool IsExpected, bool ElideAllZero) {
309 if ((ElideAllZeroBranchWeights && ElideAllZero) &&
310 llvm::all_of(Weights, [](uint32_t V) { return V == 0; })) {
311 I.setMetadata(LLVMContext::MD_prof, nullptr);
312 return;
313 }
314
315 MDBuilder MDB(I.getContext());
316 MDNode *BranchWeights = MDB.createBranchWeights(Weights, IsExpected);
317 I.setMetadata(LLVMContext::MD_prof, BranchWeights);
318}
319
321 bool IsExpected, bool ElideAllZero) {
322 setBranchWeights(I, fitWeights(Weights), IsExpected, ElideAllZero);
323}
324
326 std::optional<uint64_t> KnownMaxCount) {
327 uint64_t MaxCount = KnownMaxCount.has_value() ? KnownMaxCount.value()
328 : *llvm::max_element(Weights);
329 assert(MaxCount > 0 && "Bad max count");
330 uint64_t Scale = calculateCountScale(MaxCount);
331 SmallVector<uint32_t> DownscaledWeights;
332 for (const auto &ECI : Weights)
333 DownscaledWeights.push_back(scaleBranchCount(ECI, Scale));
334 return DownscaledWeights;
335}
336
338 assert(T != 0 && "Caller should guarantee");
339 auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);
340 if (ProfileData == nullptr)
341 return;
342
343 auto *ProfDataName = dyn_cast<MDString>(ProfileData->getOperand(0));
344 if (!ProfDataName ||
345 (ProfDataName->getString() != MDProfLabels::BranchWeights &&
346 ProfDataName->getString() != MDProfLabels::ValueProfile))
347 return;
348
349 if (!hasCountTypeMD(I))
350 return;
351
352 LLVMContext &C = I.getContext();
353
354 MDBuilder MDB(C);
356 Vals.push_back(ProfileData->getOperand(0));
357 APInt APS(128, S), APT(128, T);
358 if (ProfDataName->getString() == MDProfLabels::BranchWeights &&
359 ProfileData->getNumOperands() > 0) {
360 // Using APInt::div may be expensive, but most cases should fit 64 bits.
361 APInt Val(128,
363 ProfileData->getOperand(getBranchWeightOffset(ProfileData)))
364 ->getValue()
365 .getZExtValue());
366 Val *= APS;
367 Vals.push_back(MDB.createConstant(ConstantInt::get(
368 Type::getInt32Ty(C), Val.udiv(APT).getLimitedValue(UINT32_MAX))));
369 } else if (ProfDataName->getString() == MDProfLabels::ValueProfile)
370 for (unsigned Idx = 1; Idx < ProfileData->getNumOperands(); Idx += 2) {
371 // The first value is the key of the value profile, which will not change.
372 Vals.push_back(ProfileData->getOperand(Idx));
374 mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(Idx + 1))
375 ->getValue()
376 .getZExtValue();
377 // Don't scale the magic number.
378 if (Count == NOMORE_ICP_MAGICNUM) {
379 Vals.push_back(ProfileData->getOperand(Idx + 1));
380 continue;
381 }
382 // Using APInt::div may be expensive, but most cases should fit 64 bits.
383 APInt Val(128, Count);
384 Val *= APS;
385 Vals.push_back(MDB.createConstant(ConstantInt::get(
386 Type::getInt64Ty(C), Val.udiv(APT).getLimitedValue())));
387 }
388 I.setMetadata(LLVMContext::MD_prof, MDNode::get(C, Vals));
389}
390
391} // namespace llvm
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
#define F(x, y, z)
Definition MD5.cpp:55
#define I(x, y, z)
Definition MD5.cpp:58
This file contains the declarations for metadata subclasses.
#define T
This file contains the declarations for profiling metadata utility functions.
This file contains some templates that are useful if you are working with the STL at all.
This file defines the SmallVector class.
static const char PassName[]
Class for arbitrary precision integers.
Definition APInt.h:78
LLVM_ABI APInt udiv(const APInt &RHS) const
Unsigned division operation.
Definition APInt.cpp:1573
unsigned getActiveBits() const
Compute the number of active bits in the value.
Definition APInt.h:1512
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const
If this value is smaller than the specified limit, return it, otherwise return the limit value.
Definition APInt.h:475
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:41
size_t size() const
size - Get the array size.
Definition ArrayRef.h:147
This is the shared class of boolean and integer constants.
Definition Constants.h:87
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:163
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition Constants.h:154
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:68
LLVM_ABI ConstantAsMetadata * createConstant(Constant *C)
Return the given constant as metadata.
Definition MDBuilder.cpp:25
LLVM_ABI MDNode * createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight, bool IsExpected=false)
Return metadata containing two branch weights.
Definition MDBuilder.cpp:38
Metadata node.
Definition Metadata.h:1078
const MDOperand & getOperand(unsigned I) const
Definition Metadata.h:1442
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1569
unsigned getNumOperands() const
Return number of MDNode operands.
Definition Metadata.h:1448
bool equalsStr(StringRef Str) const
Definition Metadata.h:922
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void resize(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
Definition StringRef.h:55
static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)
Definition Type.cpp:298
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:297
LLVM Value Representation.
Definition Value.h:75
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
initializer< Ty > init(const Ty &Val)
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract(Y &&MD)
Extract a Value from Metadata, if any.
Definition Metadata.h:695
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
Definition Metadata.h:667
This is an optimization pass for GlobalISel generic memory operations.
@ Offset
Definition DWP.cpp:477
cl::opt< bool > ElideAllZeroBranchWeights("elide-all-zero-branch-weights", cl::init(true))
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1705
LLVM_ABI bool extractProfTotalWeight(const MDNode *ProfileData, uint64_t &TotalWeights)
Retrieve the total of all weights from MD_prof data.
LLVM_ABI unsigned getBranchWeightOffset(const MDNode *ProfileData)
Return the offset to the first branch weight data.
LLVM_ABI bool isBranchWeightMD(const MDNode *ProfileData)
Checks if an MDNode contains Branch Weight Metadata.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:644
LLVM_ABI void setExplicitlyUnknownBranchWeightsIfProfiled(Instruction &I, Function &F, StringRef PassName)
Like setExplicitlyUnknownBranchWeights(...), but only sets unknown branch weights in the new instruct...
LLVM_ABI bool isExplicitlyUnknownProfileMetadata(const MDNode &MD)
LLVM_ABI MDNode * getBranchWeightMDNode(const Instruction &I)
Get the branch weights metadata node.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition STLExtras.h:2116
LLVM_ABI void setExplicitlyUnknownBranchWeights(Instruction &I, StringRef PassName)
Specify that the branch weights for this terminator cannot be known at compile time.
LLVM_ABI bool hasBranchWeightOrigin(const Instruction &I)
Check if Branch Weight Metadata has an "expected" field from an llvm.expect* intrinsic.
LLVM_ABI void setBranchWeights(Instruction &I, ArrayRef< uint32_t > Weights, bool IsExpected, bool ElideAllZero=false)
Create a new branch_weights metadata node and add or overwrite a prof metadata reference to instructi...
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
Definition bit.h:222
LLVM_ABI MDNode * getValidBranchWeightMDNode(const Instruction &I)
Get the valid branch weights metadata node.
LLVM_ABI bool hasValidBranchWeightMD(const Instruction &I)
Checks if an instructions has valid Branch Weight Metadata.
FunctionAddr VTableAddr Count
Definition InstrProf.h:139
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:548
LLVM_ABI void setExplicitlyUnknownFunctionEntryCount(Function &F, StringRef PassName)
Analogous to setExplicitlyUnknownBranchWeights, but for functions and their entry counts.
LLVM_ABI bool isValueProfileMD(const MDNode *ProfileData)
Checks if an MDNode contains value profiling Metadata.
LLVM_ABI unsigned getNumBranchWeights(const MDNode &ProfileData)
LLVM_ABI void extractFromBranchWeightMD32(const MDNode *ProfileData, SmallVectorImpl< uint32_t > &Weights)
Faster version of extractBranchWeights() that skips checks and must only be called with "branch_weigh...
LLVM_ABI bool hasExplicitlyUnknownBranchWeights(const Instruction &I)
LLVM_ABI bool hasProfMD(const Instruction &I)
Checks if an Instruction has MD_prof Metadata.
auto max_element(R &&Range)
Provide wrappers to std::max_element which take ranges instead of having to pass begin/end explicitly...
Definition STLExtras.h:2010
LLVM_ABI bool extractBranchWeights(const MDNode *ProfileData, SmallVectorImpl< uint32_t > &Weights)
Extract branch weights from MD_prof metadata.
LLVM_ABI const char * LLVMLoopEstimatedTripCount
Profile-based loop metadata that should be accessed only by using llvm::getLoopEstimatedTripCount and...
uint32_t scaleBranchCount(uint64_t Count, uint64_t Scale)
Scale an individual branch count.
LLVM_ABI bool hasBranchWeightMD(const Instruction &I)
Checks if an instructions has Branch Weight Metadata.
LLVM_ABI void setFittedBranchWeights(Instruction &I, ArrayRef< uint64_t > Weights, bool IsExpected, bool ElideAllZero=false)
Variant of setBranchWeights where the Weights will be fit first to uint32_t by shifting right.
uint64_t calculateCountScale(uint64_t MaxCount)
Calculate what to divide by to scale counts.
LLVM_ABI SmallVector< uint32_t > downscaleWeights(ArrayRef< uint64_t > Weights, std::optional< uint64_t > KnownMaxCount=std::nullopt)
downscale the given weights preserving the ratio.
const uint64_t NOMORE_ICP_MAGICNUM
Magic number in the value profile metadata showing a target has been promoted for the instruction and...
Definition Metadata.h:59
LLVM_ABI void scaleProfData(Instruction &I, uint64_t S, uint64_t T)
Scaling the profile data attached to 'I' using the ratio of S/T.
static bool hasCountTypeMD(const Instruction &I)
LLVM_ABI void extractFromBranchWeightMD64(const MDNode *ProfileData, SmallVectorImpl< uint64_t > &Weights)
Faster version of extractBranchWeights() that skips checks and must only be called with "branch_weigh...
static LLVM_ABI const char * ExpectedBranchWeights
static LLVM_ABI const char * SyntheticFunctionEntryCount
static LLVM_ABI const char * BranchWeights
static LLVM_ABI const char * FunctionEntryCount
static LLVM_ABI const char * UnknownBranchWeightsMarker
static LLVM_ABI const char * ValueProfile