AtomicExpandPass.cpp File Reference

#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLFunctionalExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Analysis/InstSimplifyFolder.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/CodeGen/AtomicExpand.h"
#include "llvm/CodeGen/AtomicExpandUtils.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/MemoryModelRelaxationAnnotations.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Transforms/Utils/LowerAtomic.h"
#include <cassert>
#include <cstdint>
#include <iterator>

Go to the source code of this file.

Macros
#define	DEBUG_TYPE   "atomic-expand"

Functions
	INITIALIZE_PASS_BEGIN (AtomicExpandLegacy, DEBUG_TYPE, "Expand Atomic instructions", false, false) INITIALIZE_PASS_END(AtomicExpandLegacy
Expand Atomic static false unsigned	getAtomicOpSize (LoadInst *LI)
static unsigned	getAtomicOpSize (StoreInst *SI)
static unsigned	getAtomicOpSize (AtomicRMWInst *RMWI)
static unsigned	getAtomicOpSize (AtomicCmpXchgInst *CASI)
static void	copyMetadataForAtomic (Instruction &Dest, const Instruction &Source)
	Copy metadata that's safe to preserve when widening atomics.
template<typename Inst>
static bool	atomicSizeSupported (const TargetLowering *TLI, Inst *I)
static void	createCmpXchgInstFun (IRBuilderBase &Builder, Value *Addr, Value *Loaded, Value *NewVal, Align AddrAlign, AtomicOrdering MemOpOrder, SyncScope::ID SSID, Value *&Success, Value *&NewLoaded, Instruction *MetadataSrc)
static PartwordMaskValues	createMaskInstrs (IRBuilderBase &Builder, Instruction *I, Type *ValueType, Value *Addr, Align AddrAlign, unsigned MinWordSize)
	This is a helper function which builds instructions to provide values necessary for partword atomic operations.
static Value *	extractMaskedValue (IRBuilderBase &Builder, Value *WideWord, const PartwordMaskValues &PMV)
static Value *	insertMaskedValue (IRBuilderBase &Builder, Value *WideWord, Value *Updated, const PartwordMaskValues &PMV)
static Value *	performMaskedAtomicOp (AtomicRMWInst::BinOp Op, IRBuilderBase &Builder, Value *Loaded, Value *Shifted_Inc, Value *Inc, const PartwordMaskValues &PMV)
	Emit IR to implement a masked version of a given atomicrmw operation.
static bool	canUseSizedAtomicCall (unsigned Size, Align Alignment, const DataLayout &DL)
static ArrayRef< RTLIB::Libcall >	GetRMWLibcall (AtomicRMWInst::BinOp Op)

Variables
	DEBUG_TYPE
Expand Atomic	instructions
Expand Atomic	false

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "atomic-expand"

Definition at line 59 of file AtomicExpandPass.cpp.

Function Documentation

atomicSizeSupported()

template<typename Inst>

bool atomicSizeSupported ( const TargetLowering * TLI, Inst * I )

static

Definition at line 243 of file AtomicExpandPass.cpp.

References getAtomicOpSize(), llvm::TargetLoweringBase::getMaxAtomicSizeInBitsSupported(), I, and Size.

canUseSizedAtomicCall()

bool canUseSizedAtomicCall ( unsigned Size, Align Alignment, const DataLayout & DL )

static

Definition at line 1743 of file AtomicExpandPass.cpp.

References DL, and Size.

copyMetadataForAtomic()

void copyMetadataForAtomic ( Instruction & Dest, const Instruction & Source )

static

Copy metadata that's safe to preserve when widening atomics.

Definition at line 207 of file AtomicExpandPass.cpp.

References llvm::Value::getContext(), N, and llvm::Instruction::setMetadata().

Referenced by createCmpXchgInstFun().

createCmpXchgInstFun()

void createCmpXchgInstFun ( IRBuilderBase & Builder, Value * Addr, Value * Loaded, Value * NewVal, Align AddrAlign, AtomicOrdering MemOpOrder, SyncScope::ID SSID, Value *& Success, Value *& NewLoaded, Instruction * MetadataSrc )

static

Definition at line 651 of file AtomicExpandPass.cpp.

References assert(), copyMetadataForAtomic(), llvm::Type::getIntNTy(), llvm::Type::getPrimitiveSizeInBits(), llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(), llvm::Value::getType(), llvm::Type::isFloatingPointTy(), llvm::Type::isPointerTy(), llvm::Type::isVectorTy(), and Success.

createMaskInstrs()

PartwordMaskValues createMaskInstrs ( IRBuilderBase & Builder, Instruction * I, Type * ValueType, Value * Addr, Align AddrAlign, unsigned MinWordSize )

static

This is a helper function which builds instructions to provide values necessary for partword atomic operations.

It takes an incoming address, Addr, and ValueType, and constructs the address, shift-amounts and masks needed to work with a larger value of size WordSize.

AlignedAddr: Addr rounded down to a multiple of WordSize

ShiftAmt: Number of bits to right-shift a WordSize value loaded from AlignAddr for it to have the same value as if ValueType was loaded from Addr.

Mask: Value to mask with the value loaded from AlignAddr to include only the part that would've been loaded from Addr.

Inv_Mask: The inverse of Mask.

Definition at line 818 of file AtomicExpandPass.cpp.

References assert(), llvm::cast(), DL, llvm::Type::getIntNTy(), llvm::Constant::getNullValue(), llvm::Value::getType(), I, llvm::Type::isFloatingPointTy(), and llvm::Type::isVectorTy().

extractMaskedValue()

Value * extractMaskedValue ( IRBuilderBase & Builder, Value * WideWord, const PartwordMaskValues & PMV )

static

Definition at line 885 of file AtomicExpandPass.cpp.

References assert(), and llvm::Value::getType().

Referenced by performMaskedAtomicOp().

getAtomicOpSize() [1/4]

unsigned getAtomicOpSize ( AtomicCmpXchgInst * CASI )

static

Definition at line 201 of file AtomicExpandPass.cpp.

References DL, llvm::AtomicCmpXchgInst::getCompareOperand(), llvm::Instruction::getDataLayout(), and llvm::Value::getType().

getAtomicOpSize() [2/4]

unsigned getAtomicOpSize ( AtomicRMWInst * RMWI )

static

Definition at line 196 of file AtomicExpandPass.cpp.

References DL, llvm::Instruction::getDataLayout(), llvm::Value::getType(), and llvm::AtomicRMWInst::getValOperand().

getAtomicOpSize() [3/4]

Expand Atomic static false unsigned getAtomicOpSize ( LoadInst * LI )

static

Definition at line 186 of file AtomicExpandPass.cpp.

References DL, and getAtomicOpSize().

Referenced by atomicSizeSupported(), and getAtomicOpSize().

getAtomicOpSize() [4/4]

unsigned getAtomicOpSize ( StoreInst * SI )

static

Definition at line 191 of file AtomicExpandPass.cpp.

References DL.

GetRMWLibcall()

ArrayRef< RTLIB::Libcall > GetRMWLibcall ( AtomicRMWInst::BinOp Op )

static

Definition at line 1799 of file AtomicExpandPass.cpp.

References llvm::AtomicRMWInst::Add, llvm::AtomicRMWInst::And, llvm::ArrayRef(), llvm::AtomicRMWInst::BAD_BINOP, llvm::AtomicRMWInst::FAdd, llvm::AtomicRMWInst::FMax, llvm::AtomicRMWInst::FMaximum, llvm::AtomicRMWInst::FMin, llvm::AtomicRMWInst::FMinimum, llvm::AtomicRMWInst::FSub, llvm_unreachable, llvm::AtomicRMWInst::Max, llvm::AtomicRMWInst::Min, llvm::AtomicRMWInst::Nand, llvm::AtomicRMWInst::Or, llvm::AtomicRMWInst::Sub, llvm::AtomicRMWInst::UDecWrap, llvm::AtomicRMWInst::UIncWrap, llvm::AtomicRMWInst::UMax, llvm::AtomicRMWInst::UMin, llvm::AtomicRMWInst::USubCond, llvm::AtomicRMWInst::USubSat, llvm::AtomicRMWInst::Xchg, and llvm::AtomicRMWInst::Xor.

INITIALIZE_PASS_BEGIN()

INITIALIZE_PASS_BEGIN	(	AtomicExpandLegacy	,
		DEBUG_TYPE	,
		"Expand Atomic instructions"	,
		false	,
		false	)

References DEBUG_TYPE, and INITIALIZE_PASS_DEPENDENCY.

insertMaskedValue()

Value * insertMaskedValue ( IRBuilderBase & Builder, Value * WideWord, Value * Updated, const PartwordMaskValues & PMV )

static

Definition at line 896 of file AtomicExpandPass.cpp.

References assert(), and llvm::Value::getType().

Referenced by performMaskedAtomicOp().

performMaskedAtomicOp()

Value * performMaskedAtomicOp ( AtomicRMWInst::BinOp Op, IRBuilderBase & Builder, Value * Loaded, Value * Shifted_Inc, Value * Inc, const PartwordMaskValues & PMV )

static

Emit IR to implement a masked version of a given atomicrmw operation.

(That is, only the bits under the Mask should be affected by the operation)

Definition at line 916 of file AtomicExpandPass.cpp.

References llvm::AtomicRMWInst::Add, llvm::AtomicRMWInst::And, llvm::buildAtomicRMWValue(), extractMaskedValue(), llvm::AtomicRMWInst::FAdd, llvm::AtomicRMWInst::FMax, llvm::AtomicRMWInst::FMaximum, llvm::AtomicRMWInst::FMin, llvm::AtomicRMWInst::FMinimum, llvm::AtomicRMWInst::FSub, insertMaskedValue(), llvm_unreachable, llvm::AtomicRMWInst::Max, llvm::AtomicRMWInst::Min, llvm::AtomicRMWInst::Nand, llvm::AtomicRMWInst::Or, llvm::AtomicRMWInst::Sub, llvm::AtomicRMWInst::UDecWrap, llvm::AtomicRMWInst::UIncWrap, llvm::AtomicRMWInst::UMax, llvm::AtomicRMWInst::UMin, llvm::AtomicRMWInst::USubCond, llvm::AtomicRMWInst::USubSat, llvm::AtomicRMWInst::Xchg, and llvm::AtomicRMWInst::Xor.

Variable Documentation

DEBUG_TYPE

DEBUG_TYPE

Definition at line 182 of file AtomicExpandPass.cpp.

false

Expand Atomic false

Definition at line 183 of file AtomicExpandPass.cpp.

instructions

Expand Atomic instructions

Definition at line 183 of file AtomicExpandPass.cpp.

Referenced by addBoundsChecking(), llvm::SystemZTTIImpl::adjustInliningThreshold(), llvm::coro::Shape::analyze(), llvm::X86TTIImpl::areInlineCompatible(), bitTrackingDCE(), canTRE(), checkFunctionMemoryAccess(), collectDbgVariableRecords(), collectMemAccessInfo(), collectPreserveStaticOffsetCalls(), llvm::coro::collectSpillsAndAllocasFromInsts(), convertAnnotation2Metadata(), llvm::coro::doRematerializations(), eliminateDeadCode(), eraseDebugIntrinsicsWithNonLocalRefs(), explicifyGuards(), FixupDebugInfoForOutlinedFunction(), fixupDebugInfoPostExtraction(), functionWillReturn(), getEdgeKind(), llvm::GenericUniformityAnalysisImpl< ContextT >::initialize(), INITIALIZE_PASS(), insertCallAtAllFunctionExitPoints(), lowerAllowChecks(), llvm::MLInlineAdvisor::MLInlineAdvisor(), llvm::InstDeleterIRStrategy::mutate(), llvm::ScalarEvolution::print(), llvm::StackSafetyGlobalInfo::print(), processDbgDeclares(), removeTailCallAttribute(), llvm::replaceCreatedSSACopys(), llvm::coro::BaseCloner::replaceEntryBlock(), rescheduleLexographically(), llvm::AliasSetsPrinterPass::run(), llvm::AMDGPULowerKernelAttributesPass::run(), llvm::AssumeBuilderPass::run(), llvm::DevirtSCCRepeatedPass::run(), llvm::DSEPass::run(), llvm::InlinerPass::run(), llvm::KCFIPass::run(), llvm::MemDerefPrinterPass::run(), llvm::ModuleInlinerPass::run(), llvm::MustBeExecutedContextPrinterPass::run(), llvm::PAEvalPass::run(), llvm::StackLifetimePrinterPass::run(), runImpl(), runImpl(), runImpl(), runImpl(), runImpl(), runNVVMIntrRange(), llvm::RewriteStatepointsForGC::runOnFunction(), runPass(), sinkLifetimeStartMarkers(), stripDeadDebugInfoImpl(), stripGCRelocates(), stripNonValidDataFromBody(), stripTBAA(), tagInvariantLoads(), updateCGAndAnalysisManagerForPass(), and usesMSVCFloatingPoint().