MemProfUse.cpp File Reference

#include "llvm/Transforms/Instrumentation/MemProfUse.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Analysis/MemoryProfileInfo.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Module.h"
#include "llvm/ProfileData/DataAccessProf.h"
#include "llvm/ProfileData/InstrProf.h"
#include "llvm/ProfileData/InstrProfReader.h"
#include "llvm/ProfileData/MemProfCommon.h"
#include "llvm/Support/BLAKE3.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/HashBuilder.h"
#include "llvm/Support/VirtualFileSystem.h"
#include "llvm/Transforms/Utils/LongestCommonSequence.h"
#include <map>
#include <set>

Go to the source code of this file.

Classes
struct	AllocMatchInfo
struct	CallSiteEntry
struct	CallSiteEntryHash

Namespaces
namespace	llvm
	This is an optimization pass for GlobalISel generic memory operations.

Macros
#define	DEBUG_TYPE   "memprof"

Functions
	STATISTIC (NumOfMemProfMissing, "Number of functions without memory profile.")
	STATISTIC (NumOfMemProfMismatch, "Number of functions having mismatched memory profile hash.")
	STATISTIC (NumOfMemProfFunc, "Number of functions having valid memory profile.")
	STATISTIC (NumOfMemProfAllocContextProfiles, "Number of alloc contexts in memory profile.")
	STATISTIC (NumOfMemProfCallSiteProfiles, "Number of callsites in memory profile.")
	STATISTIC (NumOfMemProfMatchedAllocContexts, "Number of matched memory profile alloc contexts.")
	STATISTIC (NumOfMemProfMatchedAllocs, "Number of matched memory profile allocs.")
	STATISTIC (NumOfMemProfMatchedCallSites, "Number of matched memory profile callsites.")
	STATISTIC (NumOfMemProfHotGlobalVars, "Number of global vars annotated with 'hot' section prefix.")
	STATISTIC (NumOfMemProfColdGlobalVars, "Number of global vars annotated with 'unlikely' section prefix.")
	STATISTIC (NumOfMemProfUnknownGlobalVars, "Number of global vars with unknown hotness (no section prefix).")
	STATISTIC (NumOfMemProfExplicitSectionGlobalVars, "Number of global vars with user-specified section (not annotated).")
static void	addCallsiteMetadata (Instruction &I, ArrayRef< uint64_t > InlinedCallStack, LLVMContext &Ctx)
static uint64_t	computeStackId (GlobalValue::GUID Function, uint32_t LineOffset, uint32_t Column)
static uint64_t	computeStackId (const memprof::Frame &Frame)
static AllocationType	addCallStack (CallStackTrie &AllocTrie, const AllocationInfo *AllocInfo, uint64_t FullStackId)
static bool	stackFrameIncludesInlinedCallStack (ArrayRef< Frame > ProfileCallStack, ArrayRef< uint64_t > InlinedCallStack)
static bool	isAllocationWithHotColdVariant (const Function *Callee, const TargetLibraryInfo &TLI)
static void	undriftMemProfRecord (const DenseMap< uint64_t, LocToLocMap > &UndriftMaps, memprof::MemProfRecord &MemProfRec)
static void	addVPMetadata (Module &M, Instruction &I, ArrayRef< GlobalValue::GUID > CalleeGuids)
static void	handleAllocSite (Instruction &I, CallBase *CI, ArrayRef< uint64_t > InlinedCallStack, LLVMContext &Ctx, OptimizationRemarkEmitter &ORE, uint64_t MaxColdSize, const std::set< const AllocationInfo * > &AllocInfoSet, std::map< std::pair< uint64_t, unsigned >, AllocMatchInfo > &FullStackIdToAllocMatchInfo)
static void	handleCallSite (Instruction &I, const Function *CalledFunction, ArrayRef< uint64_t > InlinedCallStack, const std::unordered_set< CallSiteEntry, CallSiteEntryHash > &CallSiteEntries, Module &M, std::set< std::vector< uint64_t > > &MatchedCallSites)
static void	readMemprof (Module &M, Function &F, IndexedInstrProfReader *MemProfReader, const TargetLibraryInfo &TLI, std::map< std::pair< uint64_t, unsigned >, AllocMatchInfo > &FullStackIdToAllocMatchInfo, std::set< std::vector< uint64_t > > &MatchedCallSites, DenseMap< uint64_t, LocToLocMap > &UndriftMaps, OptimizationRemarkEmitter &ORE, uint64_t MaxColdSize)
static bool	hasExplicitSectionName (const GlobalVariable &GVar)

Variables
cl::opt< bool >	llvm::PGOWarnMissing
cl::opt< bool >	llvm::NoPGOWarnMismatchComdatWeak
static cl::opt< bool >	ClMemProfMatchHotColdNew ("memprof-match-hot-cold-new", cl::desc("Match allocation profiles onto existing hot/cold operator new calls"), cl::Hidden, cl::init(false))
static cl::opt< bool >	ClPrintMemProfMatchInfo ("memprof-print-match-info", cl::desc("Print matching stats for each allocation " "context in this module's profiles"), cl::Hidden, cl::init(false))
static cl::opt< bool >	SalvageStaleProfile ("memprof-salvage-stale-profile", cl::desc("Salvage stale MemProf profile"), cl::init(false), cl::Hidden)
static cl::opt< bool >	ClMemProfAttachCalleeGuids ("memprof-attach-calleeguids", cl::desc("Attach calleeguids as value profile metadata for indirect calls."), cl::init(true), cl::Hidden)
static cl::opt< unsigned >	MinMatchedColdBytePercent ("memprof-matching-cold-threshold", cl::init(100), cl::Hidden, cl::desc("Min percent of cold bytes matched to hint allocation cold"))
static cl::opt< bool >	AnnotateStaticDataSectionPrefix ("memprof-annotate-static-data-prefix", cl::init(false), cl::Hidden, cl::desc("If true, annotate the static data section prefix"))

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "memprof"

Definition at line 41 of file MemProfUse.cpp.

Function Documentation

addCallsiteMetadata()

void addCallsiteMetadata ( Instruction & I, ArrayRef< uint64_t > InlinedCallStack, LLVMContext & Ctx )

static

Definition at line 107 of file MemProfUse.cpp.

References llvm::memprof::buildCallstackMetadata(), and I.

Referenced by handleAllocSite(), and handleCallSite().

addCallStack()

AllocationType addCallStack ( CallStackTrie & AllocTrie, const AllocationInfo * AllocInfo, uint64_t FullStackId )

static

Definition at line 129 of file MemProfUse.cpp.

References llvm::memprof::CallStackTrie::addCallStack(), assert(), computeStackId(), llvm::memprof::getAllocType(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::memprof::recordContextSizeInfoForAnalysis().

Referenced by handleAllocSite().

addVPMetadata()

void addVPMetadata ( Module & M, Instruction & I, ArrayRef< GlobalValue::GUID > CalleeGuids )

static

Definition at line 338 of file MemProfUse.cpp.

References llvm::annotateValueSite(), ClMemProfAttachCalleeGuids, llvm::ArrayRef< T >::empty(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::getValueProfDataFromInst(), I, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::SmallVectorTemplateCommon< T, typename >::size().

Referenced by handleCallSite().

computeStackId() [1/2]

uint64_t computeStackId ( const memprof::Frame & Frame )

static

Definition at line 125 of file MemProfUse.cpp.

References llvm::memprof::Frame::Column, computeStackId(), llvm::memprof::Frame::Function, and llvm::memprof::Frame::LineOffset.

computeStackId() [2/2]

uint64_t computeStackId ( GlobalValue::GUID Function, uint32_t LineOffset, uint32_t Column )

static

Definition at line 114 of file MemProfUse.cpp.

References llvm::HashBuilder< HasherT, Endianness >::add(), llvm::HashBuilderBase< HasherT >::final(), and llvm::little.

Referenced by addCallStack(), computeStackId(), and readMemprof().

handleAllocSite()

void handleAllocSite ( Instruction & I, CallBase * CI, ArrayRef< uint64_t > InlinedCallStack, LLVMContext & Ctx, OptimizationRemarkEmitter & ORE, uint64_t MaxColdSize, const std::set< const AllocationInfo * > & AllocInfoSet, std::map< std::pair< uint64_t, unsigned >, AllocMatchInfo > & FullStackIdToAllocMatchInfo )

static

Definition at line 378 of file MemProfUse.cpp.

References addCallsiteMetadata(), addCallStack(), llvm::memprof::CallStackTrie::addSingleAllocTypeAttribute(), assert(), llvm::memprof::CallStackTrie::buildAndAttachMIBMetadata(), ClPrintMemProfMatchInfo, llvm::Cold, llvm::memprof::computeFullStackId(), llvm::memprof::CallStackTrie::empty(), I, MinMatchedColdBytePercent, llvm::memprof::recordContextSizeInfoForAnalysis(), llvm::ArrayRef< T >::size(), and stackFrameIncludesInlinedCallStack().

Referenced by readMemprof().

handleCallSite()

void handleCallSite ( Instruction & I, const Function * CalledFunction, ArrayRef< uint64_t > InlinedCallStack, const std::unordered_set< CallSiteEntry, CallSiteEntryHash > & CallSiteEntries, Module & M, std::set< std::vector< uint64_t > > & MatchedCallSites )

static

Definition at line 470 of file MemProfUse.cpp.

References addCallsiteMetadata(), addVPMetadata(), llvm::append_range(), CallSiteEntry::CalleeGuids, ClPrintMemProfMatchInfo, CallSiteEntry::Frames, I, and stackFrameIncludesInlinedCallStack().

Referenced by readMemprof().

hasExplicitSectionName()

bool hasExplicitSectionName ( const GlobalVariable & GVar )

static

Definition at line 783 of file MemProfUse.cpp.

References llvm::GlobalVariable::getAttributes(), and llvm::GlobalObject::hasSection().

isAllocationWithHotColdVariant()

bool isAllocationWithHotColdVariant ( const Function * Callee, const TargetLibraryInfo & TLI )

static

Definition at line 162 of file MemProfUse.cpp.

References ClMemProfMatchHotColdNew, and llvm::TargetLibraryInfo::getLibFunc().

Referenced by llvm::memprof::extractCallsFromIR(), and readMemprof().

readMemprof()

void readMemprof ( Module & M, Function & F, IndexedInstrProfReader * MemProfReader, const TargetLibraryInfo & TLI, std::map< std::pair< uint64_t, unsigned >, AllocMatchInfo > & FullStackIdToAllocMatchInfo, std::set< std::vector< uint64_t > > & MatchedCallSites, DenseMap< uint64_t, LocToLocMap > & UndriftMaps, OptimizationRemarkEmitter & ORE, uint64_t MaxColdSize )

static

Definition at line 502 of file MemProfUse.cpp.

References llvm::ArrayRef(), assert(), llvm::GlobalValue::AvailableExternallyLinkage, computeStackId(), llvm::dbgs(), llvm::ArrayRef< T >::drop_front(), llvm::DS_Warning, llvm::dyn_cast(), llvm::SmallVectorTemplateCommon< T, typename >::end(), F, llvm::InstrProfError::get(), llvm::GlobalValue::getGUIDAssumingExternalLinkage(), llvm::handleAllErrors(), handleAllocSite(), handleCallSite(), llvm::hash_mismatch, I, isAllocationWithHotColdVariant(), LLVM_DEBUG, llvm::InstrProfError::message(), llvm::NoPGOWarnMismatch, llvm::NoPGOWarnMismatchComdatWeak, llvm::PGOWarnMissing, llvm::SmallVectorTemplateBase< T, bool >::push_back(), SalvageStaleProfile, undriftMemProfRecord(), and llvm::unknown_function.

Referenced by llvm::MemProfUsePass::run().

stackFrameIncludesInlinedCallStack()

bool stackFrameIncludesInlinedCallStack ( ArrayRef< Frame > ProfileCallStack, ArrayRef< uint64_t > InlinedCallStack )

static

Definition at line 153 of file MemProfUse.cpp.

References llvm::equal(), F, llvm::ArrayRef< T >::size(), and llvm::ArrayRef< T >::take_front().

Referenced by handleAllocSite(), and handleCallSite().

STATISTIC() [1/12]

STATISTIC	(	NumOfMemProfAllocContextProfiles	,
		"Number of alloc contexts in memory profile."	)

STATISTIC() [2/12]

STATISTIC	(	NumOfMemProfCallSiteProfiles	,
		"Number of callsites in memory profile."	)

STATISTIC() [3/12]

STATISTIC	(	NumOfMemProfColdGlobalVars	,
		"Number of global vars annotated with 'unlikely' section prefix."	)

STATISTIC() [4/12]

STATISTIC	(	NumOfMemProfExplicitSectionGlobalVars	,
		"Number of global vars with user-specified section (not annotated)."	)

STATISTIC() [5/12]

STATISTIC	(	NumOfMemProfFunc	,
		"Number of functions having valid memory profile."	)

STATISTIC() [6/12]

STATISTIC	(	NumOfMemProfHotGlobalVars	,
		"Number of global vars annotated with 'hot' section prefix."	)

STATISTIC() [7/12]

STATISTIC	(	NumOfMemProfMatchedAllocContexts	,
		"Number of matched memory profile alloc contexts."	)

STATISTIC() [8/12]

STATISTIC	(	NumOfMemProfMatchedAllocs	,
		"Number of matched memory profile allocs."	)

STATISTIC() [9/12]

STATISTIC	(	NumOfMemProfMatchedCallSites	,
		"Number of matched memory profile callsites."	)

STATISTIC() [10/12]

STATISTIC	(	NumOfMemProfMismatch	,
		"Number of functions having mismatched memory profile hash."	)

STATISTIC() [11/12]

STATISTIC	(	NumOfMemProfMissing	,
		"Number of functions without memory profile."	)

STATISTIC() [12/12]

STATISTIC	(	NumOfMemProfUnknownGlobalVars	,
		"Number of global vars with unknown hotness (no section prefix)."	)

undriftMemProfRecord()

void undriftMemProfRecord ( const DenseMap< uint64_t, LocToLocMap > & UndriftMaps, memprof::MemProfRecord & MemProfRec )

static

Definition at line 313 of file MemProfUse.cpp.

References llvm::memprof::MemProfRecord::AllocSites, llvm::memprof::MemProfRecord::CallSites, llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::end(), F, llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::find(), and I.

Referenced by readMemprof().

Variable Documentation

AnnotateStaticDataSectionPrefix

cl::opt< bool > AnnotateStaticDataSectionPrefix("memprof-annotate-static-data-prefix", cl::init(false), cl::Hidden, cl::desc("If true, annotate the static data section prefix")) ( "memprof-annotate-static-data-prefix" , cl::init(false) , cl::Hidden , cl::desc("If true, annotate the static data section prefix")  )

static

ClMemProfAttachCalleeGuids

cl::opt< bool > ClMemProfAttachCalleeGuids("memprof-attach-calleeguids", cl::desc( "Attach calleeguids as value profile metadata for indirect calls."), cl::init(true), cl::Hidden) ( "memprof-attach-calleeguids" , cl::desc( "Attach calleeguids as value profile metadata for indirect calls.") , cl::init(true) , cl::Hidden  )

static

Referenced by addVPMetadata().

ClMemProfMatchHotColdNew

cl::opt< bool > ClMemProfMatchHotColdNew("memprof-match-hot-cold-new", cl::desc( "Match allocation profiles onto existing hot/cold operator new calls"), cl::Hidden, cl::init(false)) ( "memprof-match-hot-cold-new" , cl::desc( "Match allocation profiles onto existing hot/cold operator new calls") , cl::Hidden , cl::init(false)  )

static

Referenced by isAllocationWithHotColdVariant().

ClPrintMemProfMatchInfo

cl::opt< bool > ClPrintMemProfMatchInfo("memprof-print-match-info", cl::desc("Print matching stats for each allocation " "context in this module's profiles"), cl::Hidden, cl::init(false)) ( "memprof-print-match-info" , cl::desc("Print matching stats for each allocation " "context in this module's profiles") , cl::Hidden , cl::init(false)  )

static

Referenced by handleAllocSite(), handleCallSite(), and llvm::MemProfUsePass::run().

MinMatchedColdBytePercent

cl::opt< unsigned > MinMatchedColdBytePercent("memprof-matching-cold-threshold", cl::init(100), cl::Hidden, cl::desc("Min percent of cold bytes matched to hint allocation cold")) ( "memprof-matching-cold-threshold" , cl::init(100) , cl::Hidden , cl::desc("Min percent of cold bytes matched to hint allocation cold")  )

static

Referenced by handleAllocSite().

SalvageStaleProfile

cl::opt< bool > SalvageStaleProfile("memprof-salvage-stale-profile", cl::desc("Salvage stale MemProf profile"), cl::init(false), cl::Hidden) ( "memprof-salvage-stale-profile" , cl::desc("Salvage stale MemProf profile") , cl::init(false) , cl::Hidden  )

static

Referenced by readMemprof(), llvm::MemProfUsePass::run(), and llvm::SampleProfileMatcher::runOnModule().