23#define DEBUG_TYPE "memory-profile-info"
29 cl::desc(
"Report total allocation sizes of hinted allocations"));
36 cl::desc(
"Keep all non-cold contexts (increases cloning overheads)"));
40 cl::desc(
"Min percent of cold bytes to hint alloc cold during cloning"));
48 cl::desc(
"Min percent of cold bytes at a callsite to discard non-cold "
55 cl::desc(
"Min percent of max cold bytes for critical cold context"));
97 if (MDS->getString() ==
"cold") {
99 }
else if (MDS->getString() ==
"hot") {
117 assert(
false &&
"Unexpected alloc type");
124 assert(NumAllocTypes != 0);
125 return NumAllocTypes == 1;
148 std::vector<ContextTotalSize> ContextSizeInfo) {
150 CallStackTrieNode *Curr =
nullptr;
151 for (
auto StackId : StackIds) {
156 assert(AllocStackId == StackId);
157 Alloc->addAllocType(AllocType);
159 AllocStackId = StackId;
160 Alloc =
new CallStackTrieNode(AllocType);
166 auto [
Next, Inserted] = Curr->Callers.try_emplace(StackId);
169 Curr->addAllocType(AllocType);
173 auto *New =
new CallStackTrieNode(AllocType);
183 BuiltFromExistingMetadata =
true;
188 for (
const auto &MIBStackIter : StackMD->
operands()) {
191 CallStack.push_back(StackId->getZExtValue());
193 std::vector<ContextTotalSize> ContextSizeInfo;
205 ContextSizeInfo.push_back({FullStackId, TotalSize});
215 bool BuiltFromExistingMetadata,
222 if (ContextSizeInfo.
empty()) {
231 for (
const auto &[FullStackId, TotalSize] : ContextSizeInfo) {
232 TotalBytes += TotalSize;
233 bool LargeColdContext =
false;
235 ColdBytes += TotalSize;
241 LargeColdContext =
true;
252 auto *ContextSizeMD =
MDNode::get(Ctx, {FullStackIdMD, TotalSizeMD});
260void CallStackTrie::collectContextSizeInfo(
261 CallStackTrieNode *Node, std::vector<ContextTotalSize> &ContextSizeInfo) {
263 for (
auto &Caller :
Node->Callers)
264 collectContextSizeInfo(
Caller.second, ContextSizeInfo);
267void CallStackTrie::convertHotToNotCold(CallStackTrieNode *Node) {
272 for (
auto &Caller :
Node->Callers)
273 convertHotToNotCold(
Caller.second);
279 std::vector<Metadata *> &SavedMIBNodes,
280 unsigned CallerContextLength,
282 bool BuiltFromExistingMetadata) {
283 const bool MostlyCold =
302 auto EmitMessageForRemovedContexts = [](
const MDNode *MIBMD,
StringRef Tag,
314 errs() <<
"MemProf hinting: Total size for " << Tag
315 <<
" non-cold full allocation context hash " << FullStackId
316 << Extra <<
": " << TS <<
"\n";
324 auto NewColdMIBNodes =
331 const float PercentCold = ColdBytes * 100.0 / TotalBytes;
332 std::string PercentStr;
334 OS <<
format(
" for %5.2f%% cold bytes", PercentCold);
335 EmitMessageForRemovedContexts(MIBMD,
"discarded", OS.
str());
339 for (
auto *M : NewColdMIBNodes)
340 SavedMIBNodes.push_back(M);
371 bool LongerNotColdContextKept =
false;
372 for (
auto *MIB : NewMIBNodes) {
379 LongerNotColdContextKept =
true;
385 bool KeepFirstNewNotCold = !LongerNotColdContextKept;
397 if (KeepFirstNewNotCold) {
398 KeepFirstNewNotCold =
false;
402 EmitMessageForRemovedContexts(MIBMD,
"pruned",
"");
407 for (
auto *M : NewColdMIBNodes)
408 SavedMIBNodes.push_back(M);
415bool CallStackTrie::buildMIBNodes(CallStackTrieNode *Node, LLVMContext &Ctx,
416 std::vector<uint64_t> &MIBCallStack,
417 std::vector<Metadata *> &MIBNodes,
418 bool CalleeHasAmbiguousCallerContext,
419 uint64_t &TotalBytes, uint64_t &ColdBytes) {
423 std::vector<ContextTotalSize> ContextSizeInfo;
424 collectContextSizeInfo(Node, ContextSizeInfo);
427 MaxColdSize, BuiltFromExistingMetadata, TotalBytes, ColdBytes));
433 if (!
Node->Callers.empty()) {
434 bool NodeHasAmbiguousCallerContext =
Node->Callers.size() > 1;
435 bool AddedMIBNodesForAllCallerContexts =
true;
439 std::vector<Metadata *> NewMIBNodes;
442 uint64_t CallerTotalBytes = 0;
443 uint64_t CallerColdBytes = 0;
444 for (
auto &Caller :
Node->Callers) {
445 MIBCallStack.push_back(
Caller.first);
446 AddedMIBNodesForAllCallerContexts &= buildMIBNodes(
447 Caller.second, Ctx, MIBCallStack, NewMIBNodes,
448 NodeHasAmbiguousCallerContext, CallerTotalBytes, CallerColdBytes);
450 MIBCallStack.pop_back();
455 CallerTotalBytes, CallerColdBytes,
456 BuiltFromExistingMetadata);
457 TotalBytes += CallerTotalBytes;
458 ColdBytes += CallerColdBytes;
460 if (AddedMIBNodesForAllCallerContexts)
464 assert(!NodeHasAmbiguousCallerContext);
477 if (!CalleeHasAmbiguousCallerContext)
479 std::vector<ContextTotalSize> ContextSizeInfo;
480 collectContextSizeInfo(Node, ContextSizeInfo);
483 BuiltFromExistingMetadata, TotalBytes, ColdBytes));
496 std::vector<ContextTotalSize> ContextSizeInfo;
497 collectContextSizeInfo(Alloc, ContextSizeInfo);
498 for (
const auto &[FullStackId, TotalSize] : ContextSizeInfo) {
499 errs() <<
"MemProf hinting: Total size for full allocation context hash "
500 << FullStackId <<
" and " << Descriptor <<
" alloc type "
506 <<
ore::NV(
"AllocationCall", CI) <<
" in function "
508 <<
" marked with memprof allocation attribute "
509 <<
ore::NV(
"Attribute", AllocTypeString));
529 convertHotToNotCold(Alloc);
538 std::vector<uint64_t> MIBCallStack;
539 MIBCallStack.push_back(AllocStackId);
540 std::vector<Metadata *> MIBNodes;
543 assert(!Alloc->Callers.empty() &&
"addCallStack has not been called yet");
547 if (buildMIBNodes(Alloc, Ctx, MIBCallStack, MIBNodes,
550 assert(MIBCallStack.size() == 1 &&
551 "Should only be left with Alloc's location in stack");
571 Iter = End ?
N->op_end() :
N->op_begin();
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static MDNode * createMIBNode(LLVMContext &Ctx, ArrayRef< uint64_t > MIBCallStack, AllocationType AllocType, ArrayRef< ContextTotalSize > ContextSizeInfo, const uint64_t MaxColdSize, bool BuiltFromExistingMetadata, uint64_t &TotalBytes, uint64_t &ColdBytes)
static void saveFilteredNewMIBNodes(std::vector< Metadata * > &NewMIBNodes, std::vector< Metadata * > &SavedMIBNodes, unsigned CallerContextLength, uint64_t TotalBytes, uint64_t ColdBytes, bool BuiltFromExistingMetadata)
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
bool empty() const
empty - Check if the array is empty.
static LLVM_ABI Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
LLVM_ABI StringRef getValueAsString() const
Return the attribute's value as a string.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
void addFnAttr(Attribute::AttrKind Kind)
Adds the attribute to the function.
bool hasFnAttr(Attribute::AttrKind Kind) const
Determine whether this call has the given attribute.
Attribute getFnAttr(StringRef Kind) const
Get the attribute of a given kind for the function.
void removeFnAttr(Attribute::AttrKind Kind)
Removes the attribute from the function.
This is the shared class of boolean and integer constants.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
This is an important class for using LLVM in a threaded context.
static LLVM_ABI MDNode * getMergedCallsiteMetadata(MDNode *A, MDNode *B)
const MDOperand & getOperand(unsigned I) const
ArrayRef< MDOperand > operands() const
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
unsigned getNumOperands() const
Return number of MDNode operands.
LLVM_ABI MDNode(LLVMContext &Context, unsigned ID, StorageType Storage, ArrayRef< Metadata * > Ops1, ArrayRef< Metadata * > Ops2={})
static LLVM_ABI MDNode * getMergedMemProfMetadata(MDNode *A, MDNode *B)
static LLVM_ABI MDString * get(LLVMContext &Context, StringRef Str)
void push_back(Metadata *MD)
Append an element to the tuple. This will resize the node.
void reserve(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
The instances of the Type class are immutable: once they are created, they are never changed.
static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)
LLVM_ABI LLVMContext & getContext() const
All values hold a context through their type.
LLVM_ABI void addCallStack(AllocationType AllocType, ArrayRef< uint64_t > StackIds, std::vector< ContextTotalSize > ContextSizeInfo={})
Add a call stack context with the given allocation type to the Trie.
LLVM_ABI void addSingleAllocTypeAttribute(CallBase *CI, AllocationType AT, StringRef Descriptor)
Add an attribute for the given allocation type to the call instruction.
LLVM_ABI bool buildAndAttachMIBMetadata(CallBase *CI)
Build and attach the minimal necessary MIB metadata.
Helper class to iterate through stack ids in both metadata (memprof MIB and callsite) and the corresp...
A raw_ostream that writes to an std::string.
std::string & str()
Returns the string's reference.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
initializer< Ty > init(const Ty &Val)
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract(Y &&MD)
Extract a Value from Metadata, if any.
LLVM_ABI MDNode * buildCallstackMetadata(ArrayRef< uint64_t > CallStack, LLVMContext &Ctx)
Build callstack metadata from the provided list of call stack ids.
LLVM_ABI bool recordContextSizeInfoForAnalysis()
Whether we need to record the context size info in the alloc trie used to build metadata.
LLVM_ABI bool metadataIncludesAllContextSizeInfo()
Whether the alloc memeprof metadata will include context size info for all MIBs.
LLVM_ABI AllocationType getMIBAllocType(const MDNode *MIB)
Returns the allocation type from an MIB metadata node.
LLVM_ABI bool metadataMayIncludeContextSizeInfo()
Whether the alloc memprof metadata may include context size info for some MIBs (but possibly not all)...
LLVM_ABI bool hasSingleAllocType(uint8_t AllocTypes)
True if the AllocTypes bitmask contains just a single type.
LLVM_ABI std::string getAllocTypeAttributeString(AllocationType Type)
Returns the string to use in attributes with the given type.
LLVM_ABI MDNode * getMIBStackNode(const MDNode *MIB)
Returns the stack node from an MIB metadata node.
LLVM_ABI void removeAnyExistingAmbiguousAttribute(CallBase *CB)
Removes any existing "ambiguous" memprof attribute.
LLVM_ABI void addAmbiguousAttribute(CallBase *CB)
Adds an "ambiguous" memprof attribute to call with a matched allocation profile but that we haven't y...
DiagnosticInfoOptimizationBase::Argument NV
NodeAddr< NodeBase * > Node
This is an optimization pass for GlobalISel generic memory operations.
cl::opt< unsigned > MinClonedColdBytePercent("memprof-cloning-cold-threshold", cl::init(100), cl::Hidden, cl::desc("Min percent of cold bytes to hint alloc cold during cloning"))
cl::opt< bool > MemProfReportHintedSizes("memprof-report-hinted-sizes", cl::init(false), cl::Hidden, cl::desc("Report total allocation sizes of hinted allocations"))
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
LLVM_ABI cl::opt< bool > MemProfKeepAllNotColdContexts("memprof-keep-all-not-cold-contexts", cl::init(false), cl::Hidden, cl::desc("Keep all non-cold contexts (increases cloning overheads)"))
cl::opt< unsigned > MinCallsiteColdBytePercent("memprof-callsite-cold-threshold", cl::init(100), cl::Hidden, cl::desc("Min percent of cold bytes at a callsite to discard non-cold " "contexts"))
iterator_range< filter_iterator< detail::IterOfRange< RangeT >, PredicateT > > make_filter_range(RangeT &&Range, PredicateT Pred)
Convenience function that takes a range of elements and a predicate, and return a new filter_iterator...
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
FunctionAddr VTableAddr Next
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
cl::opt< unsigned > MinPercentMaxColdSize("memprof-min-percent-max-cold-size", cl::init(100), cl::Hidden, cl::desc("Min percent of max cold bytes for critical cold context"))
int popcount(T Value) noexcept
Count the number of set bits in a value.
CallStackIterator(const NodeT *N, bool End)