30#include "llvm/ADT/SmallSet.h"
31#include "llvm/BinaryFormat/Dwarf.h"
32#include "llvm/Frontend/OpenMP/OMPConstants.h"
33#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
34#include "llvm/IR/Constants.h"
35#include "llvm/IR/DebugInfoMetadata.h"
36#include "llvm/IR/Instructions.h"
37#include "llvm/IR/IntrinsicInst.h"
38#include "llvm/IR/Metadata.h"
39#include "llvm/Support/AtomicOrdering.h"
40#include "llvm/Support/Debug.h"
44using namespace llvm::omp;
46#define TTL_CODEGEN_TYPE "target-teams-loop-codegen"
57 for (
const auto *
C : S.clauses()) {
59 if (
const auto *PreInit =
60 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
61 for (
const auto *I : PreInit->decls()) {
62 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
65 CodeGenFunction::AutoVarEmission Emission =
74 CodeGenFunction::OMPPrivateScope InlinedShareds;
76 static bool isCapturedVar(CodeGenFunction &CGF,
const VarDecl *VD) {
85 CodeGenFunction &CGF,
const OMPExecutableDirective &S,
86 const std::optional<OpenMPDirectiveKind> CapturedRegion = std::nullopt,
87 const bool EmitPreInitStmt =
true)
91 emitPreInitStmt(CGF, S);
94 assert(S.hasAssociatedStmt() &&
95 "Expected associated statement for inlined directive.");
96 const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);
98 if (
C.capturesVariable() ||
C.capturesVariableByCopy()) {
99 auto *VD =
C.getCapturedVar();
101 "Canonical decl must be captured.");
105 InlinedShareds.isGlobalVarCaptured(VD)),
110 (void)InlinedShareds.Privatize();
116class OMPParallelScope final :
public OMPLexicalScope {
117 bool EmitPreInitStmt(
const OMPExecutableDirective &S) {
125 OMPParallelScope(CodeGenFunction &CGF,
const OMPExecutableDirective &S)
126 : OMPLexicalScope(CGF, S, std::nullopt,
127 EmitPreInitStmt(S)) {}
132class OMPTeamsScope final :
public OMPLexicalScope {
133 bool EmitPreInitStmt(
const OMPExecutableDirective &S) {
140 OMPTeamsScope(CodeGenFunction &CGF,
const OMPExecutableDirective &S)
141 : OMPLexicalScope(CGF, S, std::nullopt,
142 EmitPreInitStmt(S)) {}
148 void emitPreInitStmt(CodeGenFunction &CGF,
const OMPLoopBasedDirective &S) {
149 const Stmt *PreInits;
150 CodeGenFunction::OMPMapVars PreCondVars;
151 if (
auto *LD = dyn_cast<OMPLoopDirective>(&S)) {
152 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
153 for (
const auto *E : LD->counters()) {
160 for (
const auto *
C : LD->getClausesOfKind<OMPPrivateClause>()) {
161 for (
const Expr *IRef :
C->varlist()) {
164 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
165 QualType OrigVDTy = OrigVD->getType().getNonReferenceType();
175 (void)PreCondVars.
apply(CGF);
177 (void)OMPLoopBasedDirective::doForAllLoops(
178 LD->getInnermostCapturedStmt()->getCapturedStmt(),
179 true, LD->getLoopsNumber(),
180 [&CGF](
unsigned Cnt,
const Stmt *CurStmt) {
181 if (const auto *CXXFor = dyn_cast<CXXForRangeStmt>(CurStmt)) {
182 if (const Stmt *Init = CXXFor->getInit())
184 CGF.EmitStmt(CXXFor->getRangeStmt());
185 CGF.EmitStmt(CXXFor->getEndStmt());
189 PreInits = LD->getPreInits();
190 }
else if (
const auto *
Tile = dyn_cast<OMPTileDirective>(&S)) {
191 PreInits =
Tile->getPreInits();
192 }
else if (
const auto *Stripe = dyn_cast<OMPStripeDirective>(&S)) {
193 PreInits = Stripe->getPreInits();
194 }
else if (
const auto *Unroll = dyn_cast<OMPUnrollDirective>(&S)) {
195 PreInits = Unroll->getPreInits();
196 }
else if (
const auto *Reverse = dyn_cast<OMPReverseDirective>(&S)) {
197 PreInits = Reverse->getPreInits();
198 }
else if (
const auto *Interchange =
199 dyn_cast<OMPInterchangeDirective>(&S)) {
200 PreInits = Interchange->getPreInits();
202 llvm_unreachable(
"Unknown loop-based directive kind.");
204 doEmitPreinits(PreInits);
211 const Stmt *PreInits;
212 if (
const auto *Fuse = dyn_cast<OMPFuseDirective>(&S)) {
213 PreInits = Fuse->getPreInits();
216 "Unknown canonical loop sequence transform directive kind.");
218 doEmitPreinits(PreInits);
221 void doEmitPreinits(
const Stmt *PreInits) {
227 if (
auto *PreInitCompound = dyn_cast<CompoundStmt>(PreInits))
228 llvm::append_range(PreInitStmts, PreInitCompound->body());
230 PreInitStmts.push_back(PreInits);
232 for (
const Stmt *S : PreInitStmts) {
235 if (
auto *PreInitDecl = dyn_cast<DeclStmt>(S)) {
236 for (
Decl *I : PreInitDecl->decls())
248 emitPreInitStmt(CGF, S);
253 emitPreInitStmt(CGF, S);
258 CodeGenFunction::OMPPrivateScope InlinedShareds;
260 static bool isCapturedVar(CodeGenFunction &CGF,
const VarDecl *VD) {
268 OMPSimdLexicalScope(CodeGenFunction &CGF,
const OMPExecutableDirective &S)
270 InlinedShareds(CGF) {
271 for (
const auto *
C : S.clauses()) {
272 if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
273 if (const auto *PreInit =
274 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
275 for (const auto *I : PreInit->decls()) {
276 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
277 CGF.EmitVarDecl(cast<VarDecl>(*I));
279 CodeGenFunction::AutoVarEmission Emission =
280 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
281 CGF.EmitAutoVarCleanups(Emission);
285 }
else if (
const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(
C)) {
286 for (
const Expr *E : UDP->varlist()) {
288 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
291 }
else if (
const auto *UDP = dyn_cast<OMPUseDeviceAddrClause>(
C)) {
292 for (
const Expr *E : UDP->varlist()) {
294 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
301 if (
const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
302 if (
const Expr *E = TG->getReductionRef())
307 llvm::DenseSet<CanonicalDeclPtr<const Decl>> CopyArrayTemps;
308 for (
const auto *
C : S.getClausesOfKind<OMPReductionClause>()) {
309 if (C->getModifier() != OMPC_REDUCTION_inscan)
311 for (const Expr *E : C->copy_array_temps())
312 CopyArrayTemps.insert(cast<DeclRefExpr>(E)->getDecl());
314 const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
317 if (
C.capturesVariable() ||
C.capturesVariableByCopy()) {
318 auto *VD =
C.getCapturedVar();
319 if (CopyArrayTemps.contains(VD))
322 "Canonical decl must be captured.");
323 DeclRefExpr DRE(CGF.
getContext(),
const_cast<VarDecl *
>(VD),
324 isCapturedVar(CGF, VD) ||
326 InlinedShareds.isGlobalVarCaptured(VD)),
334 (void)InlinedShareds.Privatize();
345 if (Kind != OMPD_loop)
350 BindKind =
C->getBindKind();
353 case OMPC_BIND_parallel:
355 case OMPC_BIND_teams:
356 return OMPD_distribute;
357 case OMPC_BIND_thread:
369 if (
const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
370 if (
const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
371 OrigVD = OrigVD->getCanonicalDecl();
377 OrigDRE->getType(),
VK_LValue, OrigDRE->getExprLoc());
386 llvm::Value *Size =
nullptr;
387 auto SizeInChars =
C.getTypeSizeInChars(Ty);
388 if (SizeInChars.isZero()) {
396 SizeInChars =
C.getTypeSizeInChars(Ty);
397 if (SizeInChars.isZero())
398 return llvm::ConstantInt::get(
SizeTy, 0);
399 return Builder.CreateNUWMul(Size,
CGM.getSize(SizeInChars));
401 return CGM.getSize(SizeInChars);
411 I != E; ++I, ++CurField, ++CurCap) {
412 if (CurField->hasCapturedVLAType()) {
415 CapturedVars.push_back(Val);
416 }
else if (CurCap->capturesThis()) {
417 CapturedVars.push_back(CXXThisValue);
418 }
else if (CurCap->capturesVariableByCopy()) {
423 if (!CurField->getType()->isAnyPointerType()) {
427 Twine(CurCap->getCapturedVar()->getName(),
".casted"));
443 CapturedVars.push_back(CV);
445 assert(CurCap->capturesVariable() &&
"Expected capture by reference.");
466 if (
T->isLValueReferenceType())
467 return C.getLValueReferenceType(
470 if (
T->isPointerType())
472 if (
const ArrayType *A =
T->getAsArrayTypeUnsafe()) {
473 if (
const auto *VLA = dyn_cast<VariableArrayType>(A))
475 if (!A->isVariablyModifiedType())
476 return C.getCanonicalType(
T);
478 return C.getCanonicalParamType(
T);
483struct FunctionOptions {
485 const CapturedStmt *S =
nullptr;
488 const bool UIntPtrCastRequired =
true;
491 const bool RegisterCastedArgsOnly =
false;
493 const StringRef FunctionName;
496 const bool IsDeviceKernel =
false;
497 explicit FunctionOptions(
const CapturedStmt *S,
bool UIntPtrCastRequired,
498 bool RegisterCastedArgsOnly, StringRef FunctionName,
499 SourceLocation Loc,
bool IsDeviceKernel)
500 : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
501 RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
502 FunctionName(FunctionName), Loc(Loc), IsDeviceKernel(IsDeviceKernel) {}
508 llvm::MapVector<
const Decl *, std::pair<const VarDecl *, Address>>
510 llvm::DenseMap<
const Decl *, std::pair<const Expr *, llvm::Value *>>
512 llvm::Value *&CXXThisValue,
const FunctionOptions &FO) {
515 assert(CD->
hasBody() &&
"missing CapturedDecl body");
517 CXXThisValue =
nullptr;
529 if (!FO.UIntPtrCastRequired) {
549 if (FO.UIntPtrCastRequired &&
550 ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
551 I->capturesVariableArrayType()))
554 if (I->capturesVariable() || I->capturesVariableByCopy()) {
555 CapVar = I->getCapturedVar();
557 }
else if (I->capturesThis()) {
560 assert(I->capturesVariableArrayType());
563 if (ArgType->isVariablyModifiedType())
570 }
else if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
572 Ctx, DebugFunctionDecl,
573 CapVar ? CapVar->
getBeginLoc() : FD->getBeginLoc(),
574 CapVar ? CapVar->
getLocation() : FD->getLocation(), II, ArgType,
580 Args.emplace_back(Arg);
582 TargetArgs.emplace_back(
583 FO.UIntPtrCastRequired
604 llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
608 F->setDoesNotThrow();
609 F->setDoesNotRecurse();
613 F->removeFnAttr(llvm::Attribute::NoInline);
614 F->addFnAttr(llvm::Attribute::AlwaysInline);
619 FO.UIntPtrCastRequired ? FO.Loc : FO.S->
getBeginLoc(),
620 FO.UIntPtrCastRequired ? FO.Loc
627 if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
635 if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
636 const VarDecl *CurVD = I->getCapturedVar();
637 if (!FO.RegisterCastedArgsOnly)
638 LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
646 if (FD->hasCapturedVLAType()) {
647 if (FO.UIntPtrCastRequired) {
650 Args[Cnt]->getName(), ArgLVal),
655 VLASizes.try_emplace(Args[Cnt], VAT->
getSizeExpr(), ExprArg);
656 }
else if (I->capturesVariable()) {
657 const VarDecl *Var = I->getCapturedVar();
659 Address ArgAddr = ArgLVal.getAddress();
660 if (ArgLVal.getType()->isLValueReferenceType()) {
663 assert(ArgLVal.getType()->isPointerType());
665 ArgAddr, ArgLVal.getType()->castAs<
PointerType>());
667 if (!FO.RegisterCastedArgsOnly) {
671 }
else if (I->capturesVariableByCopy()) {
672 assert(!FD->getType()->isAnyPointerType() &&
673 "Not expecting a captured pointer.");
674 const VarDecl *Var = I->getCapturedVar();
675 LocalAddrs.insert({Args[Cnt],
676 {Var, FO.UIntPtrCastRequired
678 CGF, I->getLocation(), FD->getType(),
679 Args[Cnt]->getName(), ArgLVal)
680 : ArgLVal.getAddress()}});
683 assert(I->capturesThis());
685 LocalAddrs.insert({Args[Cnt], {
nullptr, ArgLVal.getAddress()}});
699 "CapturedStmtInfo should be set when generating the captured function");
702 bool NeedWrapperFunction =
705 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs,
707 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes,
710 llvm::raw_svector_ostream Out(Buffer);
713 bool IsDeviceKernel =
CGM.getOpenMPRuntime().isGPU() &&
715 D.getCapturedStmt(OMPD_target) == &S;
716 CodeGenFunction WrapperCGF(
CGM,
true);
717 llvm::Function *WrapperF =
nullptr;
718 if (NeedWrapperFunction) {
721 FunctionOptions WrapperFO(&S,
true,
728 WrapperCGF.CXXThisValue, WrapperFO);
731 FunctionOptions FO(&S, !NeedWrapperFunction,
false,
732 Out.str(), Loc, !NeedWrapperFunction && IsDeviceKernel);
734 *
this, WrapperArgs, WrapperLocalAddrs, WrapperVLASizes, CXXThisValue, FO);
736 for (
const auto &LocalAddrPair : WrapperLocalAddrs) {
737 if (LocalAddrPair.second.first) {
738 LocalScope.addPrivate(LocalAddrPair.second.first,
739 LocalAddrPair.second.second);
742 (void)LocalScope.Privatize();
743 for (
const auto &VLASizePair : WrapperVLASizes)
744 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
747 LocalScope.ForceCleanup();
749 if (!NeedWrapperFunction)
753 WrapperF->removeFromParent();
754 F->getParent()->getFunctionList().insertAfter(F->getIterator(), WrapperF);
757 auto *PI = F->arg_begin();
758 for (
const auto *Arg : Args) {
760 auto I = LocalAddrs.find(Arg);
761 if (I != LocalAddrs.end()) {
764 I->second.first ? I->second.first->getType() : Arg->getType(),
770 auto EI = VLASizes.find(Arg);
771 if (EI != VLASizes.end()) {
783 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, Loc, F, CallArgs);
799 llvm::Value *NumElements =
emitArrayLength(ArrayTy, ElementTy, DestAddr);
806 DestBegin, NumElements);
811 llvm::Value *IsEmpty =
812 Builder.CreateICmpEQ(DestBegin, DestEnd,
"omp.arraycpy.isempty");
813 Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
816 llvm::BasicBlock *EntryBB =
Builder.GetInsertBlock();
821 llvm::PHINode *SrcElementPHI =
822 Builder.CreatePHI(SrcBegin->getType(), 2,
"omp.arraycpy.srcElementPast");
823 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
828 llvm::PHINode *DestElementPHI =
Builder.CreatePHI(
829 DestBegin->getType(), 2,
"omp.arraycpy.destElementPast");
830 DestElementPHI->addIncoming(DestBegin, EntryBB);
836 CopyGen(DestElementCurrent, SrcElementCurrent);
839 llvm::Value *DestElementNext =
841 1,
"omp.arraycpy.dest.element");
842 llvm::Value *SrcElementNext =
844 1,
"omp.arraycpy.src.element");
847 Builder.CreateICmpEQ(DestElementNext, DestEnd,
"omp.arraycpy.done");
848 Builder.CreateCondBr(Done, DoneBB, BodyBB);
849 DestElementPHI->addIncoming(DestElementNext,
Builder.GetInsertBlock());
850 SrcElementPHI->addIncoming(SrcElementNext,
Builder.GetInsertBlock());
860 const auto *BO = dyn_cast<BinaryOperator>(
Copy);
861 if (BO && BO->getOpcode() == BO_Assign) {
870 DestAddr, SrcAddr, OriginalType,
898 bool DeviceConstTarget =
getLangOpts().OpenMPIsTargetDevice &&
900 bool FirstprivateIsLastprivate =
false;
901 llvm::DenseMap<const VarDecl *, OpenMPLastprivateModifier> Lastprivates;
903 for (
const auto *D :
C->varlist())
904 Lastprivates.try_emplace(
908 llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
913 bool MustEmitFirstprivateCopy =
914 CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
916 const auto *IRef =
C->varlist_begin();
917 const auto *InitsRef =
C->inits().begin();
918 for (
const Expr *IInit :
C->private_copies()) {
920 bool ThisFirstprivateIsLastprivate =
921 Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
924 if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
926 (!VD || !VD->
hasAttr<OMPAllocateDeclAttr>())) {
927 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
934 if (DeviceConstTarget && OrigVD->getType().isConstant(
getContext()) &&
936 (!VD || !VD->
hasAttr<OMPAllocateDeclAttr>())) {
937 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
942 FirstprivateIsLastprivate =
943 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
944 if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
964 assert(!CE &&
"Expected non-constant firstprivate.");
971 if (
Type->isArrayType()) {
987 RunCleanupsScope InitScope(*this);
989 setAddrOfLocalVar(VDInit, SrcElement);
990 EmitAnyExprToMem(Init, DestElement,
991 Init->getType().getQualifiers(),
993 LocalDeclMap.erase(VDInit);
1004 setAddrOfLocalVar(VDInit, OriginalAddr);
1006 LocalDeclMap.erase(VDInit);
1008 if (ThisFirstprivateIsLastprivate &&
1009 Lastprivates[OrigVD->getCanonicalDecl()] ==
1010 OMPC_LASTPRIVATE_conditional) {
1015 (*IRef)->getExprLoc());
1016 VDAddr =
CGM.getOpenMPRuntime().emitLastprivateConditionalInit(
1020 LocalDeclMap.erase(VD);
1021 setAddrOfLocalVar(VD, VDAddr);
1023 IsRegistered = PrivateScope.
addPrivate(OrigVD, VDAddr);
1025 assert(IsRegistered &&
1026 "firstprivate var already registered as private");
1034 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
1042 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
1044 auto IRef =
C->varlist_begin();
1045 for (
const Expr *IInit :
C->private_copies()) {
1047 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
1053 assert(IsRegistered &&
"private var already registered as private");
1069 llvm::DenseSet<const VarDecl *> CopiedVars;
1070 llvm::BasicBlock *CopyBegin =
nullptr, *CopyEnd =
nullptr;
1072 auto IRef =
C->varlist_begin();
1073 auto ISrcRef =
C->source_exprs().begin();
1074 auto IDestRef =
C->destination_exprs().begin();
1075 for (
const Expr *AssignOp :
C->assignment_ops()) {
1084 getContext().getTargetInfo().isTLSSupported()) {
1086 "Copyin threadprivates should have been captured!");
1090 LocalDeclMap.erase(VD);
1094 :
CGM.GetAddrOfGlobal(VD),
1095 CGM.getTypes().ConvertTypeForMem(VD->
getType()),
1100 if (CopiedVars.size() == 1) {
1106 auto *MasterAddrInt =
Builder.CreatePtrToInt(
1108 auto *PrivateAddrInt =
Builder.CreatePtrToInt(
1111 Builder.CreateICmpNE(MasterAddrInt, PrivateAddrInt), CopyBegin,
1117 const auto *DestVD =
1138 bool HasAtLeastOneLastprivate =
false;
1140 llvm::DenseSet<const VarDecl *> SIMDLCVs;
1143 for (
const Expr *
C : LoopDirective->counters()) {
1148 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1150 HasAtLeastOneLastprivate =
true;
1153 const auto *IRef =
C->varlist_begin();
1154 const auto *IDestRef =
C->destination_exprs().begin();
1155 for (
const Expr *IInit :
C->private_copies()) {
1161 if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
1162 const auto *DestVD =
1167 (*IRef)->getType(),
VK_LValue, (*IRef)->getExprLoc());
1172 if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
1175 if (
C->getKind() == OMPC_LASTPRIVATE_conditional) {
1176 VDAddr =
CGM.getOpenMPRuntime().emitLastprivateConditionalInit(
1178 setAddrOfLocalVar(VD, VDAddr);
1184 bool IsRegistered = PrivateScope.
addPrivate(OrigVD, VDAddr);
1185 assert(IsRegistered &&
1186 "lastprivate var already registered as private");
1194 return HasAtLeastOneLastprivate;
1199 llvm::Value *IsLastIterCond) {
1208 llvm::BasicBlock *ThenBB =
nullptr;
1209 llvm::BasicBlock *DoneBB =
nullptr;
1210 if (IsLastIterCond) {
1216 return C->getKind() == OMPC_LASTPRIVATE_conditional;
1218 CGM.getOpenMPRuntime().emitBarrierCall(*
this, D.getBeginLoc(),
1225 Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
1228 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1229 llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1230 if (
const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
1231 auto IC = LoopDirective->counters().begin();
1232 for (
const Expr *F : LoopDirective->finals()) {
1236 AlreadyEmittedVars.insert(D);
1238 LoopCountersAndUpdates[D] = F;
1243 auto IRef =
C->varlist_begin();
1244 auto ISrcRef =
C->source_exprs().begin();
1245 auto IDestRef =
C->destination_exprs().begin();
1246 for (
const Expr *AssignOp :
C->assignment_ops()) {
1247 const auto *PrivateVD =
1250 const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1251 if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1255 if (
const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
1259 const auto *DestVD =
1263 if (
const auto *RefTy = PrivateVD->getType()->getAs<
ReferenceType>())
1265 Builder.CreateLoad(PrivateAddr),
1266 CGM.getTypes().ConvertTypeForMem(RefTy->getPointeeType()),
1267 CGM.getNaturalTypeAlignment(RefTy->getPointeeType()));
1269 if (
C->getKind() == OMPC_LASTPRIVATE_conditional)
1270 CGM.getOpenMPRuntime().emitLastprivateConditionalFinalUpdate(
1272 (*IRef)->getExprLoc());
1275 EmitOMPCopy(
Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1281 if (
const Expr *PostUpdate =
C->getPostUpdateExpr())
1302 if (ForInscan != (
C->getModifier() == OMPC_REDUCTION_inscan))
1304 Shareds.append(
C->varlist_begin(),
C->varlist_end());
1305 Privates.append(
C->privates().begin(),
C->privates().end());
1306 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
1307 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
1308 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
1309 if (
C->getModifier() == OMPC_REDUCTION_task) {
1310 Data.ReductionVars.append(
C->privates().begin(),
C->privates().end());
1311 Data.ReductionOrigs.append(
C->varlist_begin(),
C->varlist_end());
1312 Data.ReductionCopies.append(
C->privates().begin(),
C->privates().end());
1313 Data.ReductionOps.append(
C->reduction_ops().begin(),
1314 C->reduction_ops().end());
1315 TaskLHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
1316 TaskRHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
1321 auto *ILHS = LHSs.begin();
1322 auto *IRHS = RHSs.begin();
1323 auto *IPriv = Privates.begin();
1324 for (
const Expr *IRef : Shareds) {
1332 [&Emission](CodeGenFunction &CGF) {
1333 CGF.EmitAutoVarInit(Emission);
1341 assert(IsRegistered &&
"private var already registered as private");
1349 if (isaOMPArraySectionExpr &&
Type->isVariablyModifiedType()) {
1354 }
else if ((isaOMPArraySectionExpr &&
Type->isScalarType()) ||
1372 PrivateScope.
addPrivate(LHSVD, OriginalAddr);
1383 if (!
Data.ReductionVars.empty()) {
1385 Data.IsReductionWithTaskMod =
true;
1387 llvm::Value *ReductionDesc =
CGM.getOpenMPRuntime().emitTaskReductionInit(
1388 *
this, D.getBeginLoc(), TaskLHSs, TaskRHSs,
Data);
1389 const Expr *TaskRedRef =
nullptr;
1400 case OMPD_parallel_for:
1403 case OMPD_parallel_master:
1407 case OMPD_parallel_sections:
1411 case OMPD_target_parallel:
1415 case OMPD_target_parallel_for:
1419 case OMPD_distribute_parallel_for:
1423 case OMPD_teams_distribute_parallel_for:
1425 .getTaskReductionRefExpr();
1427 case OMPD_target_teams_distribute_parallel_for:
1429 .getTaskReductionRefExpr();
1437 case OMPD_parallel_for_simd:
1439 case OMPD_taskyield:
1443 case OMPD_taskgroup:
1451 case OMPD_cancellation_point:
1453 case OMPD_target_data:
1454 case OMPD_target_enter_data:
1455 case OMPD_target_exit_data:
1457 case OMPD_taskloop_simd:
1458 case OMPD_master_taskloop:
1459 case OMPD_master_taskloop_simd:
1460 case OMPD_parallel_master_taskloop:
1461 case OMPD_parallel_master_taskloop_simd:
1462 case OMPD_distribute:
1463 case OMPD_target_update:
1464 case OMPD_distribute_parallel_for_simd:
1465 case OMPD_distribute_simd:
1466 case OMPD_target_parallel_for_simd:
1467 case OMPD_target_simd:
1468 case OMPD_teams_distribute:
1469 case OMPD_teams_distribute_simd:
1470 case OMPD_teams_distribute_parallel_for_simd:
1471 case OMPD_target_teams:
1472 case OMPD_target_teams_distribute:
1473 case OMPD_target_teams_distribute_parallel_for_simd:
1474 case OMPD_target_teams_distribute_simd:
1475 case OMPD_declare_target:
1476 case OMPD_end_declare_target:
1477 case OMPD_threadprivate:
1479 case OMPD_declare_reduction:
1480 case OMPD_declare_mapper:
1481 case OMPD_declare_simd:
1483 case OMPD_declare_variant:
1484 case OMPD_begin_declare_variant:
1485 case OMPD_end_declare_variant:
1488 llvm_unreachable(
"Unexpected directive with task reductions.");
1494 false, TaskRedRef->
getType());
1507 bool HasAtLeastOneReduction =
false;
1508 bool IsReductionWithTaskMod =
false;
1511 if (
C->getModifier() == OMPC_REDUCTION_inscan)
1513 HasAtLeastOneReduction =
true;
1514 Privates.append(
C->privates().begin(),
C->privates().end());
1515 LHSExprs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
1516 RHSExprs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
1517 IsPrivateVarReduction.append(
C->private_var_reduction_flags().begin(),
1518 C->private_var_reduction_flags().end());
1519 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
1520 IsReductionWithTaskMod =
1521 IsReductionWithTaskMod ||
C->getModifier() == OMPC_REDUCTION_task;
1523 if (HasAtLeastOneReduction) {
1525 if (IsReductionWithTaskMod) {
1526 CGM.getOpenMPRuntime().emitTaskReductionFini(
1529 bool TeamsLoopCanBeParallel =
false;
1530 if (
auto *TTLD = dyn_cast<OMPTargetTeamsGenericLoopDirective>(&D))
1531 TeamsLoopCanBeParallel = TTLD->canBeParallelFor();
1534 TeamsLoopCanBeParallel || ReductionKind == OMPD_simd;
1535 bool SimpleReduction = ReductionKind == OMPD_simd;
1538 CGM.getOpenMPRuntime().emitReduction(
1539 *
this, D.getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
1540 {WithNowait, SimpleReduction, IsPrivateVarReduction, ReductionKind});
1549 llvm::BasicBlock *DoneBB =
nullptr;
1551 if (
const Expr *PostUpdate =
C->getPostUpdateExpr()) {
1553 if (llvm::Value *
Cond = CondGen(CGF)) {
1574 const OMPExecutableDirective &,
1575 llvm::SmallVectorImpl<llvm::Value *> &)>
1576 CodeGenBoundParametersTy;
1584 llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> PrivateDecls;
1586 for (
const Expr *Ref :
C->varlist()) {
1587 if (!Ref->getType()->isScalarType())
1589 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1597 for (
const Expr *Ref :
C->varlist()) {
1598 if (!Ref->getType()->isScalarType())
1600 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1608 for (
const Expr *Ref :
C->varlist()) {
1609 if (!Ref->getType()->isScalarType())
1611 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1623 for (
const Expr *Ref :
C->varlist()) {
1624 if (!Ref->getType()->isScalarType())
1626 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1633 CGF, S, PrivateDecls);
1639 const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1640 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1641 llvm::Value *NumThreads =
nullptr;
1650 llvm::Function *OutlinedFn =
1657 NumThreads = CGF.
EmitScalarExpr(NumThreadsClause->getNumThreads(),
1659 Modifier = NumThreadsClause->getModifier();
1661 Message = MessageClause->getMessageString();
1662 MessageLoc = MessageClause->getBeginLoc();
1665 Severity = SeverityClause->getSeverityKind();
1666 SeverityLoc = SeverityClause->getBeginLoc();
1669 CGF, NumThreads, NumThreadsClause->getBeginLoc(), Modifier, Severity,
1670 SeverityLoc, Message, MessageLoc);
1675 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
1677 const Expr *IfCond =
nullptr;
1678 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
1679 if (
C->getNameModifier() == OMPD_unknown ||
1680 C->getNameModifier() == OMPD_parallel) {
1681 IfCond =
C->getCondition();
1686 OMPParallelScope
Scope(CGF, S);
1692 CodeGenBoundParameters(CGF, S, CapturedVars);
1695 CapturedVars, IfCond, NumThreads,
1696 Modifier, Severity, Message);
1701 if (!CVD->
hasAttr<OMPAllocateDeclAttr>())
1703 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
1705 return !((AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc ||
1706 AA->getAllocatorType() == OMPAllocateDeclAttr::OMPNullMemAlloc) &&
1707 !AA->getAllocator());
1722 CGF, S.getBeginLoc(), OMPD_unknown,
false,
1728 CodeGenFunction &CGF,
const VarDecl *VD) {
1730 auto &OMPBuilder =
CGM.getOpenMPRuntime().getOMPBuilder();
1742 Size = CGF.
Builder.CreateNUWAdd(
1744 Size = CGF.
Builder.CreateUDiv(Size,
CGM.getSize(Align));
1745 Size = CGF.
Builder.CreateNUWMul(Size,
CGM.getSize(Align));
1751 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
1752 assert(AA->getAllocator() &&
1753 "Expected allocator expression for non-default allocator.");
1757 if (Allocator->getType()->isIntegerTy())
1758 Allocator = CGF.
Builder.CreateIntToPtr(Allocator,
CGM.VoidPtrTy);
1759 else if (Allocator->getType()->isPointerTy())
1763 llvm::Value *
Addr = OMPBuilder.createOMPAlloc(
1766 llvm::CallInst *FreeCI =
1767 OMPBuilder.createOMPFree(CGF.
Builder,
Addr, Allocator);
1781 if (
CGM.getLangOpts().OpenMPUseTLS &&
1782 CGM.getContext().getTargetInfo().isTLSSupported())
1785 llvm::OpenMPIRBuilder &OMPBuilder =
CGM.getOpenMPRuntime().getOMPBuilder();
1790 llvm::ConstantInt *Size =
CGM.getSize(
CGM.GetTargetTypeStoreSize(VarTy));
1792 llvm::Twine CacheName = Twine(
CGM.getMangledName(VD)).concat(Suffix);
1794 llvm::CallInst *ThreadPrivateCacheCall =
1795 OMPBuilder.createCachedThreadPrivate(CGF.
Builder,
Data, Size, CacheName);
1803 llvm::raw_svector_ostream OS(Buffer);
1804 StringRef Sep = FirstSeparator;
1805 for (StringRef Part : Parts) {
1809 return OS.str().str();
1817 llvm::BasicBlock *FiniBB = splitBBWithSuffix(
Builder,
false,
1818 "." + RegionName +
".after");
1834 llvm::BasicBlock *FiniBB = splitBBWithSuffix(
Builder,
false,
1835 "." + RegionName +
".after");
1847 if (
CGM.getLangOpts().OpenMPIRBuilder) {
1848 llvm::OpenMPIRBuilder &OMPBuilder =
CGM.getOpenMPRuntime().getOMPBuilder();
1850 llvm::Value *IfCond =
nullptr;
1855 llvm::Value *NumThreads =
nullptr;
1860 ProcBindKind ProcBind = OMP_PROC_BIND_default;
1862 ProcBind = ProcBindClause->getProcBindKind();
1864 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1868 auto FiniCB = [
this](InsertPointTy IP) {
1870 return llvm::Error::success();
1877 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
1878 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
1889 auto BodyGenCB = [&,
this](InsertPointTy AllocaIP,
1890 InsertPointTy CodeGenIP) {
1892 *
this, ParallelRegionBodyStmt, AllocaIP, CodeGenIP,
"parallel");
1893 return llvm::Error::success();
1898 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
1900 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
1901 OMPBuilder.createParallel(
Builder, AllocaIP, BodyGenCB, PrivCB, FiniCB,
1902 IfCond, NumThreads, ProcBind, S.hasCancel()));
1916 CGF.
EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
1925 [](CodeGenFunction &) {
return nullptr; });
1937class OMPTransformDirectiveScopeRAII {
1938 OMPLoopScope *
Scope =
nullptr;
1942 OMPTransformDirectiveScopeRAII(
const OMPTransformDirectiveScopeRAII &) =
1944 OMPTransformDirectiveScopeRAII &
1945 operator=(
const OMPTransformDirectiveScopeRAII &) =
delete;
1949 if (
const auto *Dir = dyn_cast<OMPLoopBasedDirective>(S)) {
1950 Scope =
new OMPLoopScope(CGF, *Dir);
1954 if (
const auto *Dir =
1955 dyn_cast<OMPCanonicalLoopSequenceTransformationDirective>(S)) {
1959 Scope =
new OMPLoopScope(CGF, *Dir);
1964 ~OMPTransformDirectiveScopeRAII() {
1975 int MaxLevel,
int Level = 0) {
1976 assert(Level < MaxLevel &&
"Too deep lookup during loop body codegen.");
1978 if (
const auto *CS = dyn_cast<CompoundStmt>(SimplifiedS)) {
1981 "LLVM IR generation of compound statement ('{}')");
1985 for (
const Stmt *CurStmt : CS->body())
1986 emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level);
1989 if (SimplifiedS == NextLoop) {
1990 if (
auto *Dir = dyn_cast<OMPLoopTransformationDirective>(SimplifiedS))
1991 SimplifiedS = Dir->getTransformedStmt();
1992 if (
const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(SimplifiedS))
1993 SimplifiedS = CanonLoop->getLoopStmt();
1994 if (
const auto *For = dyn_cast<ForStmt>(SimplifiedS)) {
1998 "Expected canonical for loop or range-based for loop.");
2000 CGF.
EmitStmt(CXXFor->getLoopVarStmt());
2001 S = CXXFor->getBody();
2003 if (Level + 1 < MaxLevel) {
2004 NextLoop = OMPLoopDirective::tryToFindNextInnerLoop(
2006 emitBody(CGF, S, NextLoop, MaxLevel, Level + 1);
2017 for (
const Expr *UE : D.updates())
2025 for (
const Expr *UE :
C->updates())
2032 BreakContinueStack.push_back(BreakContinue(D,
LoopExit, Continue));
2033 for (
const Expr *E : D.finals_conditions()) {
2046 bool IsInscanRegion = InscanScope.
Privatize();
2047 if (IsInscanRegion) {
2057 if (EKind != OMPD_simd && !
getLangOpts().OpenMPSimd)
2066 D.getInnermostCapturedStmt()->getCapturedStmt()->IgnoreContainers();
2069 OMPLoopBasedDirective::tryToFindNextInnerLoop(
2071 D.getLoopsNumber());
2079 BreakContinueStack.pop_back();
2090 std::unique_ptr<CodeGenFunction::CGCapturedStmtInfo> CSI =
2091 std::make_unique<CodeGenFunction::CGCapturedStmtInfo>(*S);
2095 return {F, CapStruct.getPointer(
ParentCGF)};
2099static llvm::CallInst *
2104 EffectiveArgs.reserve(Args.size() + 1);
2105 llvm::append_range(EffectiveArgs, Args);
2106 EffectiveArgs.push_back(Cap.second);
2111llvm::CanonicalLoopInfo *
2113 assert(Depth == 1 &&
"Nested loops with OpenMPIRBuilder not yet implemented");
2139 const Stmt *SyntacticalLoop = S->getLoopStmt();
2150 const Stmt *BodyStmt;
2151 if (
const auto *For = dyn_cast<ForStmt>(SyntacticalLoop)) {
2152 if (
const Stmt *InitStmt = For->getInit())
2154 BodyStmt = For->getBody();
2155 }
else if (
const auto *RangeFor =
2156 dyn_cast<CXXForRangeStmt>(SyntacticalLoop)) {
2157 if (
const DeclStmt *RangeStmt = RangeFor->getRangeStmt())
2159 if (
const DeclStmt *BeginStmt = RangeFor->getBeginStmt())
2161 if (
const DeclStmt *EndStmt = RangeFor->getEndStmt())
2163 if (
const DeclStmt *LoopVarStmt = RangeFor->getLoopVarStmt())
2165 BodyStmt = RangeFor->getBody();
2167 llvm_unreachable(
"Expected for-stmt or range-based for-stmt");
2170 const CapturedStmt *DistanceFunc = S->getDistanceFunc();
2183 llvm::Value *DistVal =
Builder.CreateLoad(CountAddr,
".count");
2186 llvm::OpenMPIRBuilder &OMPBuilder =
CGM.getOpenMPRuntime().getOMPBuilder();
2187 auto BodyGen = [&,
this](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP,
2188 llvm::Value *IndVar) {
2193 const DeclRefExpr *LoopVarRef = S->getLoopVarRef();
2201 return llvm::Error::success();
2204 llvm::CanonicalLoopInfo *
CL =
2205 cantFail(OMPBuilder.createCanonicalLoop(
Builder, BodyGen, DistVal));
2217 const Expr *IncExpr,
2218 const llvm::function_ref<
void(CodeGenFunction &)> BodyGen,
2219 const llvm::function_ref<
void(CodeGenFunction &)> PostIncGen) {
2229 const CapturedStmt *ICS = OMPED.getInnermostCapturedStmt();
2243 llvm::BasicBlock *ExitBlock =
LoopExit.getBlock();
2244 if (RequiresCleanup)
2251 if (ExitBlock !=
LoopExit.getBlock()) {
2261 BreakContinueStack.push_back(BreakContinue(S,
LoopExit, Continue));
2269 BreakContinueStack.pop_back();
2280 bool HasLinears =
false;
2285 if (
const auto *Ref =
2304 if (
const auto *CS = cast_or_null<BinaryOperator>(
C->getCalcStep()))
2316 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
2319 llvm::BasicBlock *DoneBB =
nullptr;
2322 auto IC =
C->varlist_begin();
2323 for (
const Expr *F :
C->finals()) {
2325 if (llvm::Value *
Cond = CondGen(*
this)) {
2337 (*IC)->getType(),
VK_LValue, (*IC)->getExprLoc());
2345 if (
const Expr *PostUpdate =
C->getPostUpdateExpr())
2357 llvm::APInt ClauseAlignment(64, 0);
2358 if (
const Expr *AlignmentExpr = Clause->getAlignment()) {
2361 ClauseAlignment = AlignmentCI->getValue();
2363 for (
const Expr *E : Clause->varlist()) {
2364 llvm::APInt Alignment(ClauseAlignment);
2365 if (Alignment == 0) {
2372 E->getType()->getPointeeType()))
2375 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
2376 "alignment is not power of 2");
2377 if (Alignment != 0) {
2391 auto I = S.private_counters().begin();
2392 for (
const Expr *E : S.counters()) {
2398 LocalDeclMap.erase(PrivateVD);
2404 E->getType(),
VK_LValue, E->getExprLoc());
2413 if (!
C->getNumForLoops())
2415 for (
unsigned I = S.getLoopsNumber(), E =
C->getLoopNumIterations().size();
2421 if (DRE->refersToEnclosingVariableOrCapture()) {
2430 const Expr *
Cond, llvm::BasicBlock *TrueBlock,
2431 llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
2439 for (
const Expr *I : S.inits()) {
2446 for (
const Expr *E : S.dependent_counters()) {
2449 assert(!E->getType().getNonReferenceType()->isRecordType() &&
2450 "dependent counter must not be an iterator.");
2454 (void)PreCondVars.
setVarAddr(CGF, VD, CounterAddr);
2456 (void)PreCondVars.
apply(CGF);
2457 for (
const Expr *E : S.dependent_inits()) {
2471 llvm::DenseSet<const VarDecl *> SIMDLCVs;
2475 for (
const Expr *
C : LoopDirective->counters()) {
2481 auto CurPrivate =
C->privates().begin();
2482 for (
const Expr *E :
C->varlist()) {
2484 const auto *PrivateVD =
2491 assert(IsRegistered &&
"linear var already registered as private");
2579 if (
const auto *CS = dyn_cast<CapturedStmt>(S))
2597 if (HasOrderedDirective)
2605 const Stmt *AssociatedStmt = D.getAssociatedStmt();
2609 if (
C->getKind() == OMPC_ORDER_concurrent)
2612 if ((EKind == OMPD_simd ||
2616 return C->getModifier() == OMPC_REDUCTION_inscan;
2624 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
2627 llvm::BasicBlock *DoneBB =
nullptr;
2628 auto IC = D.counters().begin();
2629 auto IPC = D.private_counters().begin();
2630 for (
const Expr *F : D.finals()) {
2633 const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
2635 OrigVD->hasGlobalStorage() || CED) {
2637 if (llvm::Value *
Cond = CondGen(*
this)) {
2685 auto &&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](
CodeGenFunction &CGF,
2699 const Expr *IfCond =
nullptr;
2702 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
2704 (
C->getNameModifier() == OMPD_unknown ||
2705 C->getNameModifier() == OMPD_simd)) {
2706 IfCond =
C->getCondition();
2722 OMPLoopScope PreInitScope(CGF, S);
2744 llvm::BasicBlock *ContBlock =
nullptr;
2751 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
2758 const Expr *IVExpr = S.getIterationVariable();
2766 if (
const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
2781 CGF, S, CGF.
EmitLValue(S.getIterationVariable()));
2796 emitOMPLoopBodyWithStopPoint(CGF, S,
2797 CodeGenFunction::JumpDest());
2803 if (HasLastprivateClause)
2832 if (
const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(S.getRawStmt())) {
2833 if (
const Stmt *SyntacticalLoop = CanonLoop->getLoopStmt()) {
2834 for (
const Stmt *SubStmt : SyntacticalLoop->
children()) {
2837 if (
const CompoundStmt *CS = dyn_cast<CompoundStmt>(SubStmt)) {
2852static llvm::MapVector<llvm::Value *, llvm::Value *>
2854 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars;
2856 llvm::APInt ClauseAlignment(64, 0);
2857 if (
const Expr *AlignmentExpr = Clause->getAlignment()) {
2860 ClauseAlignment = AlignmentCI->getValue();
2862 for (
const Expr *E : Clause->varlist()) {
2863 llvm::APInt Alignment(ClauseAlignment);
2864 if (Alignment == 0) {
2871 E->getType()->getPointeeType()))
2874 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
2875 "alignment is not power of 2");
2877 AlignedVars[PtrValue] = CGF.
Builder.getInt64(Alignment.getSExtValue());
2887 bool UseOMPIRBuilder =
2889 if (UseOMPIRBuilder) {
2893 if (UseOMPIRBuilder) {
2894 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars =
2897 const Stmt *Inner = S.getRawStmt();
2898 llvm::CanonicalLoopInfo *CLI =
2899 CGF.EmitOMPCollapsedCanonicalLoopNest(Inner, 1);
2901 llvm::OpenMPIRBuilder &OMPBuilder =
2904 llvm::ConstantInt *Simdlen =
nullptr;
2911 llvm::ConstantInt *Safelen =
nullptr;
2918 llvm::omp::OrderKind Order = llvm::omp::OrderKind::OMP_ORDER_unknown;
2920 if (
C->getKind() == OpenMPOrderClauseKind::OMPC_ORDER_concurrent) {
2921 Order = llvm::omp::OrderKind::OMP_ORDER_concurrent;
2926 OMPBuilder.applySimd(CLI, AlignedVars,
2927 nullptr, Order, Simdlen, Safelen);
2934 OMPLexicalScope
Scope(CGF, S, OMPD_unknown);
2949 OMPLexicalScope
Scope(CGF, S, OMPD_unknown);
2962 OMPTransformDirectiveScopeRAII TileScope(*
this, &S);
2968 OMPTransformDirectiveScopeRAII StripeScope(*
this, &S);
2974 OMPTransformDirectiveScopeRAII ReverseScope(*
this, &S);
2981 OMPTransformDirectiveScopeRAII InterchangeScope(*
this, &S);
2987 OMPTransformDirectiveScopeRAII FuseScope(*
this, &S);
2992 bool UseOMPIRBuilder =
CGM.getLangOpts().OpenMPIRBuilder;
2994 if (UseOMPIRBuilder) {
2996 const Stmt *Inner = S.getRawStmt();
3004 llvm::OpenMPIRBuilder &OMPBuilder =
CGM.getOpenMPRuntime().getOMPBuilder();
3007 llvm::CanonicalLoopInfo *UnrolledCLI =
nullptr;
3011 OMPBuilder.unrollLoopFull(DL, CLI);
3013 uint64_t Factor = 0;
3014 if (
Expr *FactorExpr = PartialClause->getFactor()) {
3015 Factor = FactorExpr->EvaluateKnownConstInt(
getContext()).getZExtValue();
3016 assert(Factor >= 1 &&
"Only positive factors are valid");
3018 OMPBuilder.unrollLoopPartial(DL, CLI, Factor,
3019 NeedsUnrolledCLI ? &UnrolledCLI :
nullptr);
3021 OMPBuilder.unrollLoopHeuristic(DL, CLI);
3024 assert((!NeedsUnrolledCLI || UnrolledCLI) &&
3025 "NeedsUnrolledCLI implies UnrolledCLI to be set");
3042 if (
Expr *FactorExpr = PartialClause->getFactor()) {
3044 FactorExpr->EvaluateKnownConstInt(
getContext()).getZExtValue();
3045 assert(Factor >= 1 &&
"Only positive factors are valid");
3053void CodeGenFunction::EmitOMPOuterLoop(
3056 const CodeGenFunction::OMPLoopArguments &LoopArgs,
3061 const Expr *IVExpr = S.getIterationVariable();
3075 llvm::Value *BoolCondVal =
nullptr;
3076 if (!DynamicOrOrdered) {
3087 RT.
emitForNext(*
this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,
3088 LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
3093 llvm::BasicBlock *ExitBlock =
LoopExit.getBlock();
3098 Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
3099 if (ExitBlock !=
LoopExit.getBlock()) {
3107 if (DynamicOrOrdered)
3112 BreakContinueStack.push_back(BreakContinue(S, LoopExit, Continue));
3117 [&S, IsMonotonic, EKind](CodeGenFunction &CGF, PrePostActionTy &) {
3122 if (
const auto *
C = S.getSingleClause<OMPOrderClause>())
3123 if (
C->getKind() == OMPC_ORDER_concurrent)
3129 [&S, &LoopArgs, LoopExit, &CodeGenLoop, IVSize, IVSigned, &CodeGenOrdered,
3130 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
3131 SourceLocation Loc = S.getBeginLoc();
3137 CGF.EmitOMPInnerLoop(
3139 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
3140 CodeGenLoop(CGF, S, LoopExit);
3142 [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
3143 CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
3148 BreakContinueStack.pop_back();
3149 if (!DynamicOrOrdered) {
3162 auto &&CodeGen = [DynamicOrOrdered, &S, &LoopArgs](CodeGenFunction &CGF) {
3163 if (!DynamicOrOrdered)
3164 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
3167 OMPCancelStack.emitExit(*
this, EKind, CodeGen);
3170void CodeGenFunction::EmitOMPForOuterLoop(
3171 const OpenMPScheduleTy &ScheduleKind,
bool IsMonotonic,
3173 const OMPLoopArguments &LoopArgs,
3175 CGOpenMPRuntime &RT =
CGM.getOpenMPRuntime();
3181 LoopArgs.Chunk !=
nullptr)) &&
3182 "static non-chunked schedule does not need outer loop");
3236 const Expr *IVExpr = S.getIterationVariable();
3240 if (DynamicOrOrdered) {
3241 const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
3242 CGDispatchBounds(*
this, S, LoopArgs.LB, LoopArgs.UB);
3243 llvm::Value *LBVal = DispatchBounds.first;
3244 llvm::Value *UBVal = DispatchBounds.second;
3245 CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
3248 IVSigned, Ordered, DipatchRTInputValues);
3250 CGOpenMPRuntime::StaticRTInput StaticInit(
3251 IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
3252 LoopArgs.ST, LoopArgs.Chunk);
3258 auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
3259 const unsigned IVSize,
3260 const bool IVSigned) {
3267 OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
3268 LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
3269 OuterLoopArgs.IncExpr = S.getInc();
3270 OuterLoopArgs.Init = S.getInit();
3271 OuterLoopArgs.Cond = S.getCond();
3272 OuterLoopArgs.NextLB = S.getNextLowerBound();
3273 OuterLoopArgs.NextUB = S.getNextUpperBound();
3274 OuterLoopArgs.DKind = LoopArgs.DKind;
3275 EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
3277 if (DynamicOrOrdered) {
3283 const unsigned IVSize,
const bool IVSigned) {}
3285void CodeGenFunction::EmitOMPDistributeOuterLoop(
3290 CGOpenMPRuntime &RT =
CGM.getOpenMPRuntime();
3297 const Expr *IVExpr = S.getIterationVariable();
3302 CGOpenMPRuntime::StaticRTInput StaticInit(
3303 IVSize, IVSigned,
false, LoopArgs.IL, LoopArgs.LB,
3304 LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
3311 IncExpr = S.getDistInc();
3313 IncExpr = S.getInc();
3318 OMPLoopArguments OuterLoopArgs;
3319 OuterLoopArgs.LB = LoopArgs.LB;
3320 OuterLoopArgs.UB = LoopArgs.UB;
3321 OuterLoopArgs.ST = LoopArgs.ST;
3322 OuterLoopArgs.IL = LoopArgs.IL;
3323 OuterLoopArgs.Chunk = LoopArgs.Chunk;
3325 ? S.getCombinedEnsureUpperBound()
3326 : S.getEnsureUpperBound();
3327 OuterLoopArgs.IncExpr = IncExpr;
3329 ? S.getCombinedInit()
3332 ? S.getCombinedCond()
3335 ? S.getCombinedNextLowerBound()
3336 : S.getNextLowerBound();
3338 ? S.getCombinedNextUpperBound()
3339 : S.getNextUpperBound();
3340 OuterLoopArgs.DKind = OMPD_distribute;
3342 EmitOMPOuterLoop(
false,
false, S,
3343 LoopScope, OuterLoopArgs, CodeGenLoopContent,
3347static std::pair<LValue, LValue>
3362 LValue PrevLB = CGF.
EmitLValue(LS.getPrevLowerBoundVariable());
3363 LValue PrevUB = CGF.
EmitLValue(LS.getPrevUpperBoundVariable());
3365 PrevLB, LS.getPrevLowerBoundVariable()->getExprLoc());
3367 PrevLBVal, LS.getPrevLowerBoundVariable()->getType(),
3368 LS.getIterationVariable()->getType(),
3369 LS.getPrevLowerBoundVariable()->getExprLoc());
3371 PrevUB, LS.getPrevUpperBoundVariable()->getExprLoc());
3373 PrevUBVal, LS.getPrevUpperBoundVariable()->getType(),
3374 LS.getIterationVariable()->getType(),
3375 LS.getPrevUpperBoundVariable()->getExprLoc());
3390static std::pair<llvm::Value *, llvm::Value *>
3395 const Expr *IVExpr = LS.getIterationVariable();
3401 llvm::Value *LBVal =
3403 llvm::Value *UBVal =
3405 return {LBVal, UBVal};
3414 llvm::Value *LBCast = CGF.
Builder.CreateIntCast(
3416 CapturedVars.push_back(LBCast);
3420 llvm::Value *UBCast = CGF.
Builder.CreateIntCast(
3422 CapturedVars.push_back(UBCast);
3433 bool HasCancel =
false;
3435 if (
const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
3436 HasCancel = D->hasCancel();
3437 else if (
const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
3438 HasCancel = D->hasCancel();
3439 else if (
const auto *D =
3440 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
3441 HasCancel = D->hasCancel();
3451 CGInlinedWorksharingLoop,
3461 OMPLexicalScope
Scope(*
this, S, OMPD_parallel);
3462 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_distribute,
CodeGen);
3471 OMPLexicalScope
Scope(*
this, S, OMPD_parallel);
3472 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_distribute,
CodeGen);
3480 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
3481 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_simd,
CodeGen);
3491 llvm::Constant *
Addr;
3493 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
3495 assert(Fn &&
Addr &&
"Target device function emission failed.");
3507struct ScheduleKindModifiersTy {
3514 : Kind(Kind), M1(M1), M2(M2) {}
3530 if (
const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
3538 bool HasLastprivateClause;
3541 OMPLoopScope PreInitScope(*
this, S);
3546 llvm::BasicBlock *ContBlock =
nullptr;
3553 emitPreCond(*
this, S, S.getPreCond(), ThenBlock, ContBlock,
3560 bool Ordered =
false;
3562 if (OrderedClause->getNumForLoops())
3572 std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*
this, S);
3573 LValue LB = Bounds.first;
3574 LValue UB = Bounds.second;
3588 CGM.getOpenMPRuntime().emitBarrierCall(
3589 *
this, S.getBeginLoc(), OMPD_unknown,
false,
3594 *
this, S,
EmitLValue(S.getIterationVariable()));
3601 CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*
this, S);
3604 const Expr *ChunkExpr =
nullptr;
3607 ScheduleKind.
Schedule =
C->getScheduleKind();
3608 ScheduleKind.
M1 =
C->getFirstScheduleModifier();
3609 ScheduleKind.
M2 =
C->getSecondScheduleModifier();
3610 ChunkExpr =
C->getChunkSize();
3613 CGM.getOpenMPRuntime().getDefaultScheduleAndChunk(
3614 *
this, S, ScheduleKind.
Schedule, ChunkExpr);
3616 bool HasChunkSizeOne =
false;
3617 llvm::Value *Chunk =
nullptr;
3621 S.getIterationVariable()->getType(),
3625 llvm::APSInt EvaluatedChunk =
Result.Val.getInt();
3626 HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);
3635 bool StaticChunkedOne =
3637 Chunk !=
nullptr) &&
3641 (ScheduleKind.
Schedule == OMPC_SCHEDULE_static &&
3642 !(ScheduleKind.
M1 == OMPC_SCHEDULE_MODIFIER_nonmonotonic ||
3643 ScheduleKind.
M2 == OMPC_SCHEDULE_MODIFIER_nonmonotonic)) ||
3644 ScheduleKind.
M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
3645 ScheduleKind.
M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
3647 Chunk !=
nullptr) ||
3648 StaticChunkedOne) &&
3658 if (
C->getKind() == OMPC_ORDER_concurrent)
3662 [IVSize, IVSigned, Ordered, IL, LB, UB, ST, StaticChunkedOne, Chunk,
3671 IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(),
3672 UB.getAddress(), ST.getAddress(),
3673 StaticChunkedOne ? Chunk :
nullptr);
3675 CGF, S.getBeginLoc(), EKind, ScheduleKind, StaticInit);
3677 if (!StaticChunkedOne)
3696 StaticChunkedOne ? S.getCombinedParForInDistCond()
3698 StaticChunkedOne ? S.getDistInc() : S.getInc(),
3699 [&S,
LoopExit](CodeGenFunction &CGF) {
3700 emitOMPLoopBodyWithStopPoint(CGF, S, LoopExit);
3702 [](CodeGenFunction &) {});
3706 auto &&
CodeGen = [&S](CodeGenFunction &CGF) {
3710 OMPCancelStack.emitExit(*
this, EKind,
CodeGen);
3717 LoopArguments.DKind = OMPD_for;
3718 EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
3719 LoopArguments, CGDispatchBounds);
3723 return CGF.
Builder.CreateIsNotNull(
3729 ? OMPD_parallel_for_simd
3733 *
this, S, [IL, &S](CodeGenFunction &CGF) {
3734 return CGF.
Builder.CreateIsNotNull(
3738 if (HasLastprivateClause)
3744 return CGF.
Builder.CreateIsNotNull(
3755 return HasLastprivateClause;
3761static std::pair<LValue, LValue>
3775static std::pair<llvm::Value *, llvm::Value *>
3779 const Expr *IVExpr = LS.getIterationVariable();
3781 llvm::Value *LBVal = CGF.
Builder.getIntN(IVSize, 0);
3783 return {LBVal, UBVal};
3795 llvm::function_ref<llvm::Value *(
CodeGenFunction &)> NumIteratorsGen) {
3796 llvm::Value *OMPScanNumIterations = CGF.
Builder.CreateIntCast(
3797 NumIteratorsGen(CGF), CGF.
SizeTy,
false);
3803 assert(
C->getModifier() == OMPC_REDUCTION_inscan &&
3804 "Only inscan reductions are expected.");
3805 Shareds.append(
C->varlist_begin(),
C->varlist_end());
3806 Privates.append(
C->privates().begin(),
C->privates().end());
3807 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
3808 CopyArrayTemps.append(
C->copy_array_temps().begin(),
3809 C->copy_array_temps().end());
3817 auto *ITA = CopyArrayTemps.begin();
3818 for (
const Expr *IRef : Privates) {
3822 if (PrivateVD->getType()->isVariablyModifiedType()) {
3847 llvm::function_ref<llvm::Value *(
CodeGenFunction &)> NumIteratorsGen) {
3848 llvm::Value *OMPScanNumIterations = CGF.
Builder.CreateIntCast(
3849 NumIteratorsGen(CGF), CGF.
SizeTy,
false);
3857 assert(
C->getModifier() == OMPC_REDUCTION_inscan &&
3858 "Only inscan reductions are expected.");
3859 Shareds.append(
C->varlist_begin(),
C->varlist_end());
3860 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
3861 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
3862 Privates.append(
C->privates().begin(),
C->privates().end());
3863 CopyOps.append(
C->copy_ops().begin(),
C->copy_ops().end());
3864 CopyArrayElems.append(
C->copy_array_elems().begin(),
3865 C->copy_array_elems().end());
3869 llvm::Value *OMPLast = CGF.
Builder.CreateNSWSub(
3870 OMPScanNumIterations,
3871 llvm::ConstantInt::get(CGF.
SizeTy, 1,
false));
3872 for (
unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
3873 const Expr *PrivateExpr = Privates[I];
3874 const Expr *OrigExpr = Shareds[I];
3875 const Expr *CopyArrayElem = CopyArrayElems[I];
3882 LValue SrcLVal = CGF.
EmitLValue(CopyArrayElem);
3884 PrivateExpr->
getType(), DestLVal.getAddress(), SrcLVal.getAddress(),
3914 llvm::Value *OMPScanNumIterations = CGF.
Builder.CreateIntCast(
3915 NumIteratorsGen(CGF), CGF.
SizeTy,
false);
3922 assert(
C->getModifier() == OMPC_REDUCTION_inscan &&
3923 "Only inscan reductions are expected.");
3924 Privates.append(
C->privates().begin(),
C->privates().end());
3925 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
3926 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
3927 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
3928 CopyArrayElems.append(
C->copy_array_elems().begin(),
3929 C->copy_array_elems().end());
3944 auto &&
CodeGen = [&S, OMPScanNumIterations, &LHSs, &RHSs, &CopyArrayElems,
3951 llvm::BasicBlock *InputBB = CGF.Builder.GetInsertBlock();
3952 llvm::BasicBlock *LoopBB = CGF.createBasicBlock(
"omp.outer.log.scan.body");
3953 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(
"omp.outer.log.scan.exit");
3955 CGF.CGM.getIntrinsic(llvm::Intrinsic::log2, CGF.DoubleTy);
3957 CGF.Builder.CreateUIToFP(OMPScanNumIterations, CGF.DoubleTy);
3958 llvm::Value *LogVal = CGF.EmitNounwindRuntimeCall(F, Arg);
3959 F = CGF.CGM.getIntrinsic(llvm::Intrinsic::ceil, CGF.DoubleTy);
3960 LogVal = CGF.EmitNounwindRuntimeCall(F, LogVal);
3961 LogVal = CGF.Builder.CreateFPToUI(LogVal, CGF.IntTy);
3962 llvm::Value *NMin1 = CGF.Builder.CreateNUWSub(
3963 OMPScanNumIterations, llvm::ConstantInt::get(CGF.SizeTy, 1));
3965 CGF.EmitBlock(LoopBB);
3966 auto *Counter = CGF.Builder.CreatePHI(CGF.IntTy, 2);
3968 auto *Pow2K = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
3969 Counter->addIncoming(llvm::ConstantInt::get(CGF.IntTy, 0), InputBB);
3970 Pow2K->addIncoming(llvm::ConstantInt::get(CGF.SizeTy, 1), InputBB);
3973 llvm::BasicBlock *InnerLoopBB =
3974 CGF.createBasicBlock(
"omp.inner.log.scan.body");
3975 llvm::BasicBlock *InnerExitBB =
3976 CGF.createBasicBlock(
"omp.inner.log.scan.exit");
3977 llvm::Value *CmpI = CGF.Builder.CreateICmpUGE(NMin1, Pow2K);
3978 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
3979 CGF.EmitBlock(InnerLoopBB);
3980 auto *IVal = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
3981 IVal->addIncoming(NMin1, LoopBB);
3984 auto *ILHS = LHSs.begin();
3985 auto *IRHS = RHSs.begin();
3986 for (
const Expr *CopyArrayElem : CopyArrayElems) {
3996 LHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress();
4001 llvm::Value *OffsetIVal = CGF.Builder.CreateNUWSub(IVal, Pow2K);
4007 RHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress();
4014 CGF.CGM.getOpenMPRuntime().emitReduction(
4015 CGF, S.getEndLoc(), Privates, LHSs, RHSs, ReductionOps,
4019 llvm::Value *NextIVal =
4020 CGF.Builder.CreateNUWSub(IVal, llvm::ConstantInt::get(CGF.SizeTy, 1));
4021 IVal->addIncoming(NextIVal, CGF.Builder.GetInsertBlock());
4022 CmpI = CGF.Builder.CreateICmpUGE(NextIVal, Pow2K);
4023 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
4024 CGF.EmitBlock(InnerExitBB);
4026 CGF.Builder.CreateNUWAdd(Counter, llvm::ConstantInt::get(CGF.IntTy, 1));
4027 Counter->addIncoming(
Next, CGF.Builder.GetInsertBlock());
4029 llvm::Value *NextPow2K =
4030 CGF.Builder.CreateShl(Pow2K, 1,
"",
true);
4031 Pow2K->addIncoming(NextPow2K, CGF.Builder.GetInsertBlock());
4032 llvm::Value *Cmp = CGF.Builder.CreateICmpNE(
Next, LogVal);
4033 CGF.Builder.CreateCondBr(Cmp, LoopBB, ExitBB);
4035 CGF.EmitBlock(ExitBB);
4041 CGF, S.getBeginLoc(), OMPD_unknown,
false,
4044 RegionCodeGenTy RCG(CodeGen);
4055 bool HasLastprivates;
4059 return C->getModifier() == OMPC_REDUCTION_inscan;
4063 OMPLoopScope LoopScope(CGF, S);
4066 const auto &&FirstGen = [&S, HasCancel, EKind](
CodeGenFunction &CGF) {
4075 const auto &&SecondGen = [&S, HasCancel, EKind,
4093 return HasLastprivates;
4106 if (
auto *SC = dyn_cast<OMPScheduleClause>(
C)) {
4111 switch (SC->getScheduleKind()) {
4112 case OMPC_SCHEDULE_auto:
4113 case OMPC_SCHEDULE_dynamic:
4114 case OMPC_SCHEDULE_runtime:
4115 case OMPC_SCHEDULE_guided:
4116 case OMPC_SCHEDULE_static:
4129static llvm::omp::ScheduleKind
4131 switch (ScheduleClauseKind) {
4133 return llvm::omp::OMP_SCHEDULE_Default;
4134 case OMPC_SCHEDULE_auto:
4135 return llvm::omp::OMP_SCHEDULE_Auto;
4136 case OMPC_SCHEDULE_dynamic:
4137 return llvm::omp::OMP_SCHEDULE_Dynamic;
4138 case OMPC_SCHEDULE_guided:
4139 return llvm::omp::OMP_SCHEDULE_Guided;
4140 case OMPC_SCHEDULE_runtime:
4141 return llvm::omp::OMP_SCHEDULE_Runtime;
4142 case OMPC_SCHEDULE_static:
4143 return llvm::omp::OMP_SCHEDULE_Static;
4145 llvm_unreachable(
"Unhandled schedule kind");
4152 bool HasLastprivates =
false;
4155 auto &&
CodeGen = [&S, &
CGM, HasCancel, &HasLastprivates,
4158 if (UseOMPIRBuilder) {
4161 llvm::omp::ScheduleKind SchedKind = llvm::omp::OMP_SCHEDULE_Default;
4162 llvm::Value *ChunkSize =
nullptr;
4166 if (
const Expr *ChunkSizeExpr = SchedClause->getChunkSize())
4171 const Stmt *Inner = S.getRawStmt();
4172 llvm::CanonicalLoopInfo *CLI =
4175 llvm::OpenMPIRBuilder &OMPBuilder =
4177 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
4179 cantFail(OMPBuilder.applyWorkshareLoop(
4180 CGF.
Builder.getCurrentDebugLocation(), CLI, AllocaIP, NeedsBarrier,
4181 SchedKind, ChunkSize,
false,
4192 OMPLexicalScope
Scope(CGF, S, OMPD_unknown);
4197 if (!UseOMPIRBuilder) {
4211 bool HasLastprivates =
false;
4212 auto &&
CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
4219 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
4220 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_simd,
CodeGen);
4225 CGM.getOpenMPRuntime().emitBarrierCall(*
this, S.getBeginLoc(), OMPD_for);
4232 llvm::Value *
Init =
nullptr) {
4239void CodeGenFunction::EmitSections(
const OMPExecutableDirective &S) {
4240 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
4241 const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);
4242 bool HasLastprivates =
false;
4244 auto &&CodeGen = [&S, CapturedStmt, CS, EKind,
4245 &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) {
4246 const ASTContext &
C = CGF.getContext();
4247 QualType KmpInt32Ty =
4248 C.getIntTypeForBitwidth(32, 1);
4251 CGF.Builder.getInt32(0));
4252 llvm::ConstantInt *GlobalUBVal = CS !=
nullptr
4253 ? CGF.Builder.getInt32(CS->size() - 1)
4254 : CGF.Builder.getInt32(0);
4258 CGF.Builder.getInt32(1));
4260 CGF.Builder.getInt32(0));
4263 OpaqueValueExpr IVRefExpr(S.getBeginLoc(), KmpInt32Ty,
VK_LValue);
4264 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
4265 OpaqueValueExpr UBRefExpr(S.getBeginLoc(), KmpInt32Ty,
VK_LValue);
4266 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
4270 S.getBeginLoc(), FPOptionsOverride());
4274 S.getBeginLoc(),
true, FPOptionsOverride());
4275 auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
4287 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(
".omp.sections.exit");
4288 llvm::SwitchInst *SwitchStmt =
4289 CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.
getBeginLoc()),
4290 ExitBB, CS ==
nullptr ? 1 : CS->size());
4292 unsigned CaseNumber = 0;
4293 for (
const Stmt *SubStmt : CS->
children()) {
4294 auto CaseBB = CGF.createBasicBlock(
".omp.sections.case");
4295 CGF.EmitBlock(CaseBB);
4296 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
4297 CGF.EmitStmt(SubStmt);
4298 CGF.EmitBranch(ExitBB);
4302 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(
".omp.sections.case");
4303 CGF.EmitBlock(CaseBB);
4304 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
4305 CGF.EmitStmt(CapturedStmt);
4306 CGF.EmitBranch(ExitBB);
4308 CGF.EmitBlock(ExitBB,
true);
4311 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
4312 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
4316 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
4320 CGF.EmitOMPPrivateClause(S, LoopScope);
4321 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(CGF, S, IV);
4322 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4323 CGF.EmitOMPReductionClauseInit(S, LoopScope);
4324 (void)LoopScope.Privatize();
4326 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4329 OpenMPScheduleTy ScheduleKind;
4330 ScheduleKind.
Schedule = OMPC_SCHEDULE_static;
4331 CGOpenMPRuntime::StaticRTInput StaticInit(
4332 32,
true,
false, IL.getAddress(),
4333 LB.getAddress(), UB.getAddress(), ST.getAddress());
4334 CGF.CGM.getOpenMPRuntime().emitForStaticInit(CGF, S.
getBeginLoc(), EKind,
4335 ScheduleKind, StaticInit);
4337 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.
getBeginLoc());
4338 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
4339 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
4340 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
4342 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.
getBeginLoc()), IV);
4344 CGF.EmitOMPInnerLoop(S,
false,
Cond, Inc, BodyGen,
4345 [](CodeGenFunction &) {});
4347 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
4348 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.
getEndLoc(),
4351 CGF.OMPCancelStack.emitExit(CGF, EKind, CodeGen);
4352 CGF.EmitOMPReductionClauseFinal(S, OMPD_parallel);
4355 return CGF.
Builder.CreateIsNotNull(
4360 if (HasLastprivates)
4367 bool HasCancel =
false;
4368 if (
auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
4369 HasCancel = OSD->hasCancel();
4370 else if (
auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
4371 HasCancel = OPSD->hasCancel();
4373 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_sections, CodeGen,
4378 if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
4396 CGF.
EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
4401 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
4402 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_scope,
CodeGen);
4406 CGM.getOpenMPRuntime().emitBarrierCall(*
this, S.getBeginLoc(), OMPD_scope);
4413 if (
CGM.getLangOpts().OpenMPIRBuilder) {
4414 llvm::OpenMPIRBuilder &OMPBuilder =
CGM.getOpenMPRuntime().getOMPBuilder();
4415 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4416 using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
4418 auto FiniCB = [](InsertPointTy IP) {
4421 return llvm::Error::success();
4424 const CapturedStmt *ICS = S.getInnermostCapturedStmt();
4430 auto SectionCB = [
this, SubStmt](InsertPointTy AllocaIP,
4431 InsertPointTy CodeGenIP) {
4433 *
this, SubStmt, AllocaIP, CodeGenIP,
"section");
4434 return llvm::Error::success();
4436 SectionCBVector.push_back(SectionCB);
4439 auto SectionCB = [
this,
CapturedStmt](InsertPointTy AllocaIP,
4440 InsertPointTy CodeGenIP) {
4443 return llvm::Error::success();
4445 SectionCBVector.push_back(SectionCB);
4452 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
4453 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
4463 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
4465 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
4466 cantFail(OMPBuilder.createSections(
4467 Builder, AllocaIP, SectionCBVector, PrivCB, FiniCB, S.hasCancel(),
4475 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
4480 CGM.getOpenMPRuntime().emitBarrierCall(*
this, S.getBeginLoc(),
4488 if (
CGM.getLangOpts().OpenMPIRBuilder) {
4489 llvm::OpenMPIRBuilder &OMPBuilder =
CGM.getOpenMPRuntime().getOMPBuilder();
4490 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4492 const Stmt *SectionRegionBodyStmt = S.getAssociatedStmt();
4493 auto FiniCB = [
this](InsertPointTy IP) {
4495 return llvm::Error::success();
4498 auto BodyGenCB = [SectionRegionBodyStmt,
this](InsertPointTy AllocaIP,
4499 InsertPointTy CodeGenIP) {
4501 *
this, SectionRegionBodyStmt, AllocaIP, CodeGenIP,
"section");
4502 return llvm::Error::success();
4507 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
4508 cantFail(OMPBuilder.createSection(
Builder, BodyGenCB, FiniCB));
4528 CopyprivateVars.append(
C->varlist_begin(),
C->varlist_end());
4529 DestExprs.append(
C->destination_exprs().begin(),
4530 C->destination_exprs().end());
4531 SrcExprs.append(
C->source_exprs().begin(),
C->source_exprs().end());
4532 AssignmentOps.append(
C->assignment_ops().begin(),
4533 C->assignment_ops().end());
4542 CGF.
EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
4547 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
4548 CGM.getOpenMPRuntime().emitSingleRegion(*
this,
CodeGen, S.getBeginLoc(),
4549 CopyprivateVars, DestExprs,
4550 SrcExprs, AssignmentOps);
4554 if (!S.getSingleClause<
OMPNowaitClause>() && CopyprivateVars.empty()) {
4555 CGM.getOpenMPRuntime().emitBarrierCall(
4556 *
this, S.getBeginLoc(),
4572 if (
CGM.getLangOpts().OpenMPIRBuilder) {
4573 llvm::OpenMPIRBuilder &OMPBuilder =
CGM.getOpenMPRuntime().getOMPBuilder();
4574 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4576 const Stmt *MasterRegionBodyStmt = S.getAssociatedStmt();
4578 auto FiniCB = [
this](InsertPointTy IP) {
4580 return llvm::Error::success();
4583 auto BodyGenCB = [MasterRegionBodyStmt,
this](InsertPointTy AllocaIP,
4584 InsertPointTy CodeGenIP) {
4586 *
this, MasterRegionBodyStmt, AllocaIP, CodeGenIP,
"master");
4587 return llvm::Error::success();
4592 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
4593 cantFail(OMPBuilder.createMaster(
Builder, BodyGenCB, FiniCB));
4608 Expr *Filter =
nullptr;
4610 Filter = FilterClause->getThreadID();
4616 if (
CGM.getLangOpts().OpenMPIRBuilder) {
4617 llvm::OpenMPIRBuilder &OMPBuilder =
CGM.getOpenMPRuntime().getOMPBuilder();
4618 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4620 const Stmt *MaskedRegionBodyStmt = S.getAssociatedStmt();
4621 const Expr *Filter =
nullptr;
4623 Filter = FilterClause->getThreadID();
4624 llvm::Value *FilterVal = Filter
4626 : llvm::ConstantInt::get(
CGM.Int32Ty, 0);
4628 auto FiniCB = [
this](InsertPointTy IP) {
4630 return llvm::Error::success();
4633 auto BodyGenCB = [MaskedRegionBodyStmt,
this](InsertPointTy AllocaIP,
4634 InsertPointTy CodeGenIP) {
4636 *
this, MaskedRegionBodyStmt, AllocaIP, CodeGenIP,
"masked");
4637 return llvm::Error::success();
4642 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
4643 OMPBuilder.createMasked(
Builder, BodyGenCB, FiniCB, FilterVal));
4654 if (
CGM.getLangOpts().OpenMPIRBuilder) {
4655 llvm::OpenMPIRBuilder &OMPBuilder =
CGM.getOpenMPRuntime().getOMPBuilder();
4656 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4658 const Stmt *CriticalRegionBodyStmt = S.getAssociatedStmt();
4659 const Expr *Hint =
nullptr;
4660 if (
const auto *HintClause = S.getSingleClause<
OMPHintClause>())
4661 Hint = HintClause->getHint();
4666 llvm::Value *HintInst =
nullptr;
4671 auto FiniCB = [
this](InsertPointTy IP) {
4673 return llvm::Error::success();
4676 auto BodyGenCB = [CriticalRegionBodyStmt,
this](InsertPointTy AllocaIP,
4677 InsertPointTy CodeGenIP) {
4679 *
this, CriticalRegionBodyStmt, AllocaIP, CodeGenIP,
"critical");
4680 return llvm::Error::success();
4685 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
4686 cantFail(OMPBuilder.createCritical(
Builder, BodyGenCB, FiniCB,
4687 S.getDirectiveName().getAsString(),
4696 CGF.
EmitStmt(S.getAssociatedStmt());
4698 const Expr *Hint =
nullptr;
4699 if (
const auto *HintClause = S.getSingleClause<
OMPHintClause>())
4700 Hint = HintClause->getHint();
4703 CGM.getOpenMPRuntime().emitCriticalRegion(*
this,
4704 S.getDirectiveName().getAsString(),
4705 CodeGen, S.getBeginLoc(), Hint);
4709 const OMPParallelForDirective &S) {
4718 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
4722 OMPLoopScope LoopScope(CGF, S);
4727 return C->getModifier() == OMPC_REDUCTION_inscan;
4743 const OMPParallelForSimdDirective &S) {
4752 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
4756 OMPLoopScope LoopScope(CGF, S);
4761 return C->getModifier() == OMPC_REDUCTION_inscan;
4777 const OMPParallelMasterDirective &S) {
4797 [](CodeGenFunction &) {
return nullptr; });
4804 const OMPParallelMaskedDirective &S) {
4824 [](CodeGenFunction &) {
return nullptr; });
4831 const OMPParallelSectionsDirective &S) {
4837 CGF.EmitSections(S);
4851class CheckVarsEscapingUntiedTaskDeclContext final
4856 explicit CheckVarsEscapingUntiedTaskDeclContext() =
default;
4857 ~CheckVarsEscapingUntiedTaskDeclContext() =
default;
4858 void VisitDeclStmt(
const DeclStmt *S) {
4863 if (
const auto *VD = dyn_cast_or_null<VarDecl>(D))
4865 PrivateDecls.push_back(VD);
4869 void VisitCapturedStmt(
const CapturedStmt *) {}
4871 void VisitBlockExpr(
const BlockExpr *) {}
4872 void VisitStmt(
const Stmt *S) {
4875 for (
const Stmt *Child : S->
children())
4881 ArrayRef<const VarDecl *> getPrivateDecls()
const {
return PrivateDecls; }
4889 bool OmpAllMemory =
false;
4892 return C->getDependencyKind() == OMPC_DEPEND_outallmemory ||
4893 C->getDependencyKind() == OMPC_DEPEND_inoutallmemory;
4895 OmpAllMemory =
true;
4900 Data.Dependences.emplace_back(OMPC_DEPEND_outallmemory,
4909 if (Kind == OMPC_DEPEND_outallmemory || Kind == OMPC_DEPEND_inoutallmemory)
4911 if (OmpAllMemory && (Kind == OMPC_DEPEND_out || Kind == OMPC_DEPEND_inout))
4914 Data.Dependences.emplace_back(
C->getDependencyKind(),
C->getModifier());
4915 DD.
DepExprs.append(
C->varlist_begin(),
C->varlist_end());
4924 const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);
4926 auto PartId = std::next(I);
4927 auto TaskT = std::next(I, 4);
4932 const Expr *
Cond = Clause->getCondition();
4935 Data.Final.setInt(CondConstant);
4940 Data.Final.setInt(
false);
4944 const Expr *Prio = Clause->getPriority();
4945 Data.Priority.setInt(
true);
4953 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
4956 auto IRef =
C->varlist_begin();
4957 for (
const Expr *IInit :
C->private_copies()) {
4959 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4960 Data.PrivateVars.push_back(*IRef);
4961 Data.PrivateCopies.push_back(IInit);
4966 EmittedAsPrivate.clear();
4969 auto IRef =
C->varlist_begin();
4970 auto IElemInitRef =
C->inits().begin();
4971 for (
const Expr *IInit :
C->private_copies()) {
4973 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4974 Data.FirstprivateVars.push_back(*IRef);
4975 Data.FirstprivateCopies.push_back(IInit);
4976 Data.FirstprivateInits.push_back(*IElemInitRef);
4983 llvm::MapVector<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
4985 auto IRef =
C->varlist_begin();
4986 auto ID =
C->destination_exprs().begin();
4987 for (
const Expr *IInit :
C->private_copies()) {
4989 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4990 Data.LastprivateVars.push_back(*IRef);
4991 Data.LastprivateCopies.push_back(IInit);
4993 LastprivateDstsOrigs.insert(
5003 Data.ReductionVars.append(
C->varlist_begin(),
C->varlist_end());
5004 Data.ReductionOrigs.append(
C->varlist_begin(),
C->varlist_end());
5005 Data.ReductionCopies.append(
C->privates().begin(),
C->privates().end());
5006 Data.ReductionOps.append(
C->reduction_ops().begin(),
5007 C->reduction_ops().end());
5008 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
5009 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
5011 Data.Reductions =
CGM.getOpenMPRuntime().emitTaskReductionInit(
5012 *
this, S.getBeginLoc(), LHSs, RHSs,
Data);
5017 CheckVarsEscapingUntiedTaskDeclContext Checker;
5018 Checker.Visit(S.getInnermostCapturedStmt()->getCapturedStmt());
5019 Data.PrivateLocals.append(Checker.getPrivateDecls().begin(),
5020 Checker.getPrivateDecls().end());
5022 auto &&
CodeGen = [&
Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
5023 CapturedRegion](CodeGenFunction &CGF,
5025 llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
5026 std::pair<Address, Address>>
5031 if (
auto *DI = CGF.getDebugInfo()) {
5032 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields =
5033 CGF.CapturedStmtInfo->getCaptureFields();
5034 llvm::Value *ContextValue = CGF.CapturedStmtInfo->getContextValue();
5035 if (CaptureFields.size() && ContextValue) {
5036 unsigned CharWidth = CGF.getContext().getCharWidth();
5050 for (
auto It = CaptureFields.begin(); It != CaptureFields.end(); ++It) {
5051 const VarDecl *SharedVar = It->first;
5054 CGF.getContext().getASTRecordLayout(CaptureRecord);
5057 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
5058 (void)DI->EmitDeclareOfAutoVariable(SharedVar, ContextValue,
5059 CGF.Builder,
false);
5062 auto UpdateExpr = [](llvm::LLVMContext &Ctx,
auto *
Declare,
5067 Ops.push_back(llvm::dwarf::DW_OP_plus_uconst);
5068 Ops.push_back(Offset);
5070 Ops.push_back(llvm::dwarf::DW_OP_deref);
5071 Declare->setExpression(llvm::DIExpression::get(Ctx, Ops));
5073 llvm::Instruction &
Last = CGF.Builder.GetInsertBlock()->back();
5074 if (
auto DDI = dyn_cast<llvm::DbgVariableIntrinsic>(&
Last))
5075 UpdateExpr(DDI->getContext(), DDI, Offset);
5078 assert(!
Last.isTerminator() &&
"unexpected terminator");
5080 CGF.Builder.GetInsertBlock()->getTrailingDbgRecords()) {
5081 for (llvm::DbgVariableRecord &DVR : llvm::reverse(
5082 llvm::filterDbgVars(Marker->getDbgRecordRange()))) {
5083 UpdateExpr(
Last.getContext(), &DVR, Offset);
5091 if (!
Data.PrivateVars.empty() || !
Data.FirstprivateVars.empty() ||
5092 !
Data.LastprivateVars.empty() || !
Data.PrivateLocals.empty()) {
5093 enum { PrivatesParam = 2, CopyFnParam = 3 };
5094 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
5096 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
5102 CallArgs.push_back(PrivatesPtr);
5103 ParamTypes.push_back(PrivatesPtr->getType());
5104 for (
const Expr *E :
Data.PrivateVars) {
5107 CGF.getContext().getPointerType(E->
getType()),
".priv.ptr.addr");
5108 PrivatePtrs.emplace_back(VD, PrivatePtr);
5110 ParamTypes.push_back(PrivatePtr.
getType());
5112 for (
const Expr *E :
Data.FirstprivateVars) {
5115 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->
getType()),
5116 ".firstpriv.ptr.addr");
5117 PrivatePtrs.emplace_back(VD, PrivatePtr);
5118 FirstprivatePtrs.emplace_back(VD, PrivatePtr);
5120 ParamTypes.push_back(PrivatePtr.
getType());
5122 for (
const Expr *E :
Data.LastprivateVars) {
5125 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->
getType()),
5126 ".lastpriv.ptr.addr");
5127 PrivatePtrs.emplace_back(VD, PrivatePtr);
5129 ParamTypes.push_back(PrivatePtr.
getType());
5134 Ty = CGF.getContext().getPointerType(Ty);
5136 Ty = CGF.getContext().getPointerType(Ty);
5138 CGF.getContext().getPointerType(Ty),
".local.ptr.addr");
5139 auto Result = UntiedLocalVars.insert(
5142 if (
Result.second ==
false)
5143 *
Result.first = std::make_pair(
5146 ParamTypes.push_back(PrivatePtr.
getType());
5148 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),
5150 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
5151 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
5152 for (
const auto &Pair : LastprivateDstsOrigs) {
5156 CGF.CapturedStmtInfo->lookup(OrigVD) !=
nullptr,
5158 Pair.second->getExprLoc());
5159 Scope.addPrivate(Pair.first, CGF.EmitLValue(&DRE).getAddress());
5161 for (
const auto &Pair : PrivatePtrs) {
5163 CGF.Builder.CreateLoad(Pair.second),
5164 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
5165 CGF.getContext().getDeclAlign(Pair.first));
5166 Scope.addPrivate(Pair.first, Replacement);
5167 if (
auto *DI = CGF.getDebugInfo())
5168 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
5169 (void)DI->EmitDeclareOfAutoVariable(
5170 Pair.first, Pair.second.getBasePointer(), CGF.Builder,
5175 for (
auto &Pair : UntiedLocalVars) {
5176 QualType VDType = Pair.first->getType().getNonReferenceType();
5177 if (Pair.first->getType()->isLValueReferenceType())
5178 VDType = CGF.getContext().getPointerType(VDType);
5180 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
5183 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(VDType)),
5184 CGF.getPointerAlign());
5185 Pair.second.first = Replacement;
5186 Ptr = CGF.Builder.CreateLoad(Replacement);
5187 Replacement =
Address(Ptr, CGF.ConvertTypeForMem(VDType),
5188 CGF.getContext().getDeclAlign(Pair.first));
5189 Pair.second.second = Replacement;
5191 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
5192 Address Replacement(Ptr, CGF.ConvertTypeForMem(VDType),
5193 CGF.getContext().getDeclAlign(Pair.first));
5194 Pair.second.first = Replacement;
5198 if (
Data.Reductions) {
5200 for (
const auto &Pair : FirstprivatePtrs) {
5202 CGF.Builder.CreateLoad(Pair.second),
5203 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
5204 CGF.getContext().getDeclAlign(Pair.first));
5205 FirstprivateScope.
addPrivate(Pair.first, Replacement);
5208 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
5210 Data.ReductionCopies,
Data.ReductionOps);
5211 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
5213 for (
unsigned Cnt = 0, E =
Data.ReductionVars.size(); Cnt < E; ++Cnt) {
5219 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
5221 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
5224 CGF.EmitScalarConversion(Replacement.emitRawPointer(CGF),
5225 CGF.getContext().VoidPtrTy,
5226 CGF.getContext().getPointerType(
5227 Data.ReductionCopies[Cnt]->getType()),
5228 Data.ReductionCopies[Cnt]->getExprLoc()),
5229 CGF.ConvertTypeForMem(
Data.ReductionCopies[Cnt]->getType()),
5230 Replacement.getAlignment());
5236 (void)
Scope.Privatize();
5242 auto IPriv =
C->privates().begin();
5243 auto IRed =
C->reduction_ops().begin();
5244 auto ITD =
C->taskgroup_descriptors().begin();
5245 for (
const Expr *Ref :
C->varlist()) {
5246 InRedVars.emplace_back(Ref);
5247 InRedPrivs.emplace_back(*IPriv);
5248 InRedOps.emplace_back(*IRed);
5249 TaskgroupDescriptors.emplace_back(*ITD);
5250 std::advance(IPriv, 1);
5251 std::advance(IRed, 1);
5252 std::advance(ITD, 1);
5258 if (!InRedVars.empty()) {
5260 for (
unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
5268 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
5270 llvm::Value *ReductionsPtr;
5271 if (
const Expr *TRExpr = TaskgroupDescriptors[Cnt]) {
5272 ReductionsPtr = CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr),
5273 TRExpr->getExprLoc());
5275 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5277 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
5280 CGF.EmitScalarConversion(
5281 Replacement.emitRawPointer(CGF), CGF.getContext().VoidPtrTy,
5282 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
5283 InRedPrivs[Cnt]->getExprLoc()),
5284 CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()),
5285 Replacement.getAlignment());
5298 llvm::Function *OutlinedFn =
CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
5299 S, *I, *PartId, *TaskT, EKind,
CodeGen,
Data.Tied,
Data.NumberOfParts);
5300 OMPLexicalScope
Scope(*
this, S, std::nullopt,
5303 TaskGen(*
this, OutlinedFn,
Data);
5320 QualType ElemType =
C.getBaseElementType(Ty);
5330 Data.FirstprivateVars.emplace_back(OrigRef);
5331 Data.FirstprivateCopies.emplace_back(PrivateRef);
5332 Data.FirstprivateInits.emplace_back(InitRef);
5345 auto PartId = std::next(I);
5346 auto TaskT = std::next(I, 4);
5349 Data.Final.setInt(
false);
5352 auto IRef =
C->varlist_begin();
5353 auto IElemInitRef =
C->inits().begin();
5354 for (
auto *IInit :
C->private_copies()) {
5355 Data.FirstprivateVars.push_back(*IRef);
5356 Data.FirstprivateCopies.push_back(IInit);
5357 Data.FirstprivateInits.push_back(*IElemInitRef);
5365 Data.ReductionVars.append(
C->varlist_begin(),
C->varlist_end());
5366 Data.ReductionOrigs.append(
C->varlist_begin(),
C->varlist_end());
5367 Data.ReductionCopies.append(
C->privates().begin(),
C->privates().end());
5368 Data.ReductionOps.append(
C->reduction_ops().begin(),
5369 C->reduction_ops().end());
5370 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
5371 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
5386 getContext(),
Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
5388 getContext(),
Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
5400 if (!isa_and_nonnull<llvm::ConstantPointerNull>(
5403 getContext(),
Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
5410 auto &&
CodeGen = [&
Data, &S, CS, &BodyGen, BPVD, PVD, SVD, MVD, EKind,
5414 if (!
Data.FirstprivateVars.empty()) {
5415 enum { PrivatesParam = 2, CopyFnParam = 3 };
5416 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
5418 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
5424 CallArgs.push_back(PrivatesPtr);
5425 ParamTypes.push_back(PrivatesPtr->getType());
5426 for (
const Expr *E :
Data.FirstprivateVars) {
5429 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->
getType()),
5430 ".firstpriv.ptr.addr");
5431 PrivatePtrs.emplace_back(VD, PrivatePtr);
5433 ParamTypes.push_back(PrivatePtr.
getType());
5435 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),
5437 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
5438 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
5439 for (
const auto &Pair : PrivatePtrs) {
5441 CGF.Builder.CreateLoad(Pair.second),
5442 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
5443 CGF.getContext().getDeclAlign(Pair.first));
5444 Scope.addPrivate(Pair.first, Replacement);
5447 CGF.processInReduction(S,
Data, CGF, CS,
Scope);
5450 CGF.GetAddrOfLocalVar(BPVD), 0);
5452 CGF.GetAddrOfLocalVar(PVD), 0);
5453 InputInfo.
SizesArray = CGF.Builder.CreateConstArrayGEP(
5454 CGF.GetAddrOfLocalVar(SVD), 0);
5457 InputInfo.
MappersArray = CGF.Builder.CreateConstArrayGEP(
5458 CGF.GetAddrOfLocalVar(MVD), 0);
5462 OMPLexicalScope LexScope(CGF, S, OMPD_task,
false);
5464 if (CGF.CGM.getLangOpts().OpenMP >= 51 &&
5469 CGF.CGM.getOpenMPRuntime().emitThreadLimitClause(
5470 CGF, TL->getThreadLimit().front(), S.getBeginLoc());
5474 llvm::Function *OutlinedFn =
CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
5475 S, *I, *PartId, *TaskT, EKind,
CodeGen,
true,
5476 Data.NumberOfParts);
5477 llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<
OMPNowaitClause>() ? 1 : 0);
5481 CGM.getOpenMPRuntime().emitTaskCall(*
this, S.getBeginLoc(), S, OutlinedFn,
5482 SharedsTy, CapturedStruct, &IfCond,
Data);
5487 CodeGenFunction &CGF,
5491 if (
Data.Reductions) {
5493 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
5495 Data.ReductionCopies,
Data.ReductionOps);
5498 for (
unsigned Cnt = 0, E =
Data.ReductionVars.size(); Cnt < E; ++Cnt) {
5512 Data.ReductionCopies[Cnt]->getType()),
5513 Data.ReductionCopies[Cnt]->getExprLoc()),
5515 Replacement.getAlignment());
5520 (void)
Scope.Privatize();
5526 auto IPriv =
C->privates().begin();
5527 auto IRed =
C->reduction_ops().begin();
5528 auto ITD =
C->taskgroup_descriptors().begin();
5529 for (
const Expr *Ref :
C->varlist()) {
5530 InRedVars.emplace_back(Ref);
5531 InRedPrivs.emplace_back(*IPriv);
5532 InRedOps.emplace_back(*IRed);
5533 TaskgroupDescriptors.emplace_back(*ITD);
5534 std::advance(IPriv, 1);
5535 std::advance(IRed, 1);
5536 std::advance(ITD, 1);
5540 if (!InRedVars.empty()) {
5542 for (
unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
5550 llvm::Value *ReductionsPtr;
5551 if (
const Expr *TRExpr = TaskgroupDescriptors[Cnt]) {
5555 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
5563 InRedPrivs[Cnt]->getExprLoc()),
5565 Replacement.getAlignment());
5579 const Expr *IfCond =
nullptr;
5580 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
5581 if (
C->getNameModifier() == OMPD_unknown ||
5582 C->getNameModifier() == OMPD_task) {
5583 IfCond =
C->getCondition();
5594 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
5595 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
5598 SharedsTy, CapturedStruct, IfCond,
5607 const OMPTaskyieldDirective &S) {
5608 CGM.getOpenMPRuntime().emitTaskyieldCall(*
this, S.getBeginLoc());
5615 bool IsFatal =
false;
5618 CGM.getOpenMPRuntime().emitErrorCall(*
this, S.getBeginLoc(), ME, IsFatal);
5622 CGM.getOpenMPRuntime().emitBarrierCall(*
this, S.getBeginLoc(), OMPD_barrier);
5630 CGM.getOpenMPRuntime().emitTaskwaitCall(*
this, S.getBeginLoc(),
Data);
5634 return T.clauses().empty();
5638 const OMPTaskgroupDirective &S) {
5639 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
5641 llvm::OpenMPIRBuilder &OMPBuilder =
CGM.getOpenMPRuntime().getOMPBuilder();
5642 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
5646 auto BodyGenCB = [&,
this](InsertPointTy AllocaIP,
5647 InsertPointTy CodeGenIP) {
5649 EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
5650 return llvm::Error::success();
5655 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
5656 cantFail(OMPBuilder.createTaskgroup(
Builder, AllocaIP, BodyGenCB));
5662 if (
const Expr *E = S.getReductionRef()) {
5667 Data.ReductionVars.append(
C->varlist_begin(),
C->varlist_end());
5668 Data.ReductionOrigs.append(
C->varlist_begin(),
C->varlist_end());
5669 Data.ReductionCopies.append(
C->privates().begin(),
C->privates().end());
5670 Data.ReductionOps.append(
C->reduction_ops().begin(),
5671 C->reduction_ops().end());
5672 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
5673 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
5675 llvm::Value *ReductionDesc =
5683 CGF.
EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
5685 CGM.getOpenMPRuntime().emitTaskgroupRegion(*
this,
CodeGen, S.getBeginLoc());
5690 ? llvm::AtomicOrdering::NotAtomic
5691 : llvm::AtomicOrdering::AcquireRelease;
5692 CGM.getOpenMPRuntime().emitFlush(
5695 if (
const auto *FlushClause = S.getSingleClause<
OMPFlushClause>())
5697 FlushClause->varlist_end());
5700 S.getBeginLoc(), AO);
5710 for (
auto &Dep :
Data.Dependences) {
5711 Address DepAddr =
CGM.getOpenMPRuntime().emitDepobjDependClause(
5712 *
this, Dep, DC->getBeginLoc());
5718 CGM.getOpenMPRuntime().emitDestroyClause(*
this, DOLVal, DC->getBeginLoc());
5722 CGM.getOpenMPRuntime().emitUpdateClause(
5723 *
this, DOLVal, UC->getDependencyKind(), UC->getBeginLoc());
5742 if (
C->getModifier() != OMPC_REDUCTION_inscan)
5744 Shareds.append(
C->varlist_begin(),
C->varlist_end());
5745 Privates.append(
C->privates().begin(),
C->privates().end());
5746 LHSs.append(
C->lhs_exprs().begin(),
C->lhs_exprs().end());
5747 RHSs.append(
C->rhs_exprs().begin(),
C->rhs_exprs().end());
5748 ReductionOps.append(
C->reduction_ops().begin(),
C->reduction_ops().end());
5749 CopyOps.append(
C->copy_ops().begin(),
C->copy_ops().end());
5750 CopyArrayTemps.append(
C->copy_array_temps().begin(),
5751 C->copy_array_temps().end());
5752 CopyArrayElems.append(
C->copy_array_elems().begin(),
5753 C->copy_array_elems().end());
5755 if (ParentDir.getDirectiveKind() == OMPD_simd ||
5797 : BreakContinueStack.back().ContinueBlock.getBlock());
5808 for (
unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
5809 const Expr *PrivateExpr = Privates[I];
5810 const Expr *TempExpr = CopyArrayTemps[I];
5822 CGM.getOpenMPRuntime().emitReduction(
5823 *
this, ParentDir.getEndLoc(), Privates, LHSs, RHSs, ReductionOps,
5826 for (
unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
5827 const Expr *PrivateExpr = Privates[I];
5834 const Expr *TempExpr = CopyArrayTemps[I];
5846 ? BreakContinueStack.back().ContinueBlock.getBlock()
5852 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
5858 .getIterationVariable()
5859 ->IgnoreParenImpCasts();
5862 IdxVal = Builder.CreateIntCast(IdxVal,
SizeTy,
false);
5863 for (
unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
5864 const Expr *PrivateExpr = Privates[I];
5865 const Expr *OrigExpr = Shareds[I];
5866 const Expr *CopyArrayElem = CopyArrayElems[I];
5867 OpaqueValueMapping IdxMapping(
5880 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
5883 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
5889 .getIterationVariable()
5890 ->IgnoreParenImpCasts();
5894 llvm::BasicBlock *ExclusiveExitBB =
nullptr;
5898 llvm::Value *Cmp =
Builder.CreateIsNull(IdxVal);
5899 Builder.CreateCondBr(Cmp, ExclusiveExitBB, ContBB);
5902 IdxVal =
Builder.CreateNUWSub(IdxVal, llvm::ConstantInt::get(
SizeTy, 1));
5904 for (
unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
5905 const Expr *PrivateExpr = Privates[I];
5906 const Expr *OrigExpr = Shareds[I];
5907 const Expr *CopyArrayElem = CopyArrayElems[I];
5916 PrivateExpr->
getType(), DestLVal.getAddress(), SrcLVal.getAddress(),
5940 if (
const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
5948 bool HasLastprivateClause =
false;
5951 OMPLoopScope PreInitScope(*
this, S);
5956 llvm::BasicBlock *ContBlock =
nullptr;
5963 emitPreCond(*
this, S, S.getPreCond(), ThenBlock, ContBlock,
5977 ? S.getCombinedLowerBoundVariable()
5978 : S.getLowerBoundVariable())));
5982 ? S.getCombinedUpperBoundVariable()
5983 : S.getUpperBoundVariable())));
5994 CGM.getOpenMPRuntime().emitBarrierCall(
5995 *
this, S.getBeginLoc(), OMPD_unknown,
false,
6007 CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*
this, S);
6010 llvm::Value *Chunk =
nullptr;
6013 ScheduleKind =
C->getDistScheduleKind();
6014 if (
const Expr *Ch =
C->getChunkSize()) {
6017 S.getIterationVariable()->getType(),
6022 CGM.getOpenMPRuntime().getDefaultDistScheduleAndChunk(
6023 *
this, S, ScheduleKind, Chunk);
6036 bool StaticChunked =
6040 Chunk !=
nullptr) ||
6045 StaticChunked ? Chunk :
nullptr);
6052 ? S.getCombinedEnsureUpperBound()
6053 : S.getEnsureUpperBound());
6056 ? S.getCombinedInit()
6061 ? S.getCombinedCond()
6065 Cond = S.getCombinedDistCond();
6097 [&S, &LoopScope,
Cond, IncExpr,
LoopExit, &CodeGenLoop,
6100 S, LoopScope.requiresCleanups(),
Cond, IncExpr,
6101 [&S,
LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
6102 CodeGenLoop(CGF, S, LoopExit);
6104 [&S, StaticChunked](CodeGenFunction &CGF) {
6105 if (StaticChunked) {
6106 CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound());
6107 CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound());
6108 CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound());
6109 CGF.EmitIgnoredExpr(S.getCombinedInit());
6119 const OMPLoopArguments LoopArguments = {
6120 LB.getAddress(), UB.getAddress(), ST.getAddress(), IL.getAddress(),
6122 EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
6127 return CGF.
Builder.CreateIsNotNull(
6137 *
this, S, [IL, &S](CodeGenFunction &CGF) {
6138 return CGF.
Builder.CreateIsNotNull(
6143 if (HasLastprivateClause) {
6166 OMPLexicalScope
Scope(CGF, S, OMPD_unknown);
6175static llvm::Function *
6182 Fn->setDoesNotRecurse();
6186template <
typename T>
6188 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP,
6189 llvm::OpenMPIRBuilder &OMPBuilder) {
6191 unsigned NumLoops =
C->getNumLoops();
6195 for (
unsigned I = 0; I < NumLoops; I++) {
6196 const Expr *CounterVal =
C->getLoopData(I);
6201 StoreValues.emplace_back(StoreValue);
6203 OMPDoacrossKind<T> ODK;
6204 bool IsDependSource = ODK.isSource(
C);
6206 OMPBuilder.createOrderedDepend(CGF.
Builder, AllocaIP, NumLoops,
6207 StoreValues,
".cnt.addr", IsDependSource));
6211 if (
CGM.getLangOpts().OpenMPIRBuilder) {
6212 llvm::OpenMPIRBuilder &OMPBuilder =
CGM.getOpenMPRuntime().getOMPBuilder();
6213 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
6218 assert(!S.hasAssociatedStmt() &&
"No associated statement must be in "
6219 "ordered depend|doacross construct.");
6231 auto FiniCB = [
this](InsertPointTy IP) {
6233 return llvm::Error::success();
6236 auto BodyGenCB = [&S,
C,
this](InsertPointTy AllocaIP,
6237 InsertPointTy CodeGenIP) {
6242 llvm::BasicBlock *FiniBB = splitBBWithSuffix(
6243 Builder,
false,
".ordered.after");
6247 assert(S.getBeginLoc().isValid() &&
6248 "Outlined function call location must be valid.");
6251 OutlinedFn, CapturedVars);
6256 return llvm::Error::success();
6259 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
6260 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
6261 OMPBuilder.createOrderedThreadsSimd(
Builder, BodyGenCB, FiniCB, !
C));
6268 assert(!S.hasAssociatedStmt() &&
6269 "No associated statement must be in ordered depend construct.");
6271 CGM.getOpenMPRuntime().emitDoacrossOrdered(*
this, DC);
6275 assert(!S.hasAssociatedStmt() &&
6276 "No associated statement must be in ordered doacross construct.");
6278 CGM.getOpenMPRuntime().emitDoacrossOrdered(*
this, DC);
6282 auto &&
CodeGen = [&S,
C,
this](CodeGenFunction &CGF,
6287 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
6289 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
6290 OutlinedFn, CapturedVars);
6296 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
6297 CGM.getOpenMPRuntime().emitOrderedRegion(*
this,
CodeGen, S.getBeginLoc(), !
C);
6304 "DestType must have scalar evaluation kind.");
6305 assert(!Val.
isAggregate() &&
"Must be a scalar or complex.");
6316 "DestType must have complex evaluation kind.");
6325 ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
6327 assert(Val.
isComplex() &&
"Must be a scalar or complex.");
6332 Val.
getComplexVal().first, SrcElementType, DestElementType, Loc);
6334 Val.
getComplexVal().second, SrcElementType, DestElementType, Loc);
6340 LValue LVal,
RValue RVal) {
6341 if (LVal.isGlobalReg())
6348 llvm::AtomicOrdering AO, LValue LVal,
6350 if (LVal.isGlobalReg())
6353 LVal, Loc, llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO),
6362 *
this, RVal, RValTy, LVal.
getType(), Loc)),
6371 llvm_unreachable(
"Must be a scalar or complex.");
6379 assert(
V->isLValue() &&
"V of 'omp atomic read' is not lvalue");
6380 assert(
X->isLValue() &&
"X of 'omp atomic read' is not lvalue");
6389 case llvm::AtomicOrdering::Acquire:
6390 case llvm::AtomicOrdering::AcquireRelease:
6391 case llvm::AtomicOrdering::SequentiallyConsistent:
6393 llvm::AtomicOrdering::Acquire);
6395 case llvm::AtomicOrdering::Monotonic:
6396 case llvm::AtomicOrdering::Release:
6398 case llvm::AtomicOrdering::NotAtomic:
6399 case llvm::AtomicOrdering::Unordered:
6400 llvm_unreachable(
"Unexpected ordering.");
6407 llvm::AtomicOrdering AO,
const Expr *
X,
6410 assert(
X->isLValue() &&
"X of 'omp atomic write' is not lvalue");
6418 case llvm::AtomicOrdering::Release:
6419 case llvm::AtomicOrdering::AcquireRelease:
6420 case llvm::AtomicOrdering::SequentiallyConsistent:
6422 llvm::AtomicOrdering::Release);
6424 case llvm::AtomicOrdering::Acquire:
6425 case llvm::AtomicOrdering::Monotonic:
6427 case llvm::AtomicOrdering::NotAtomic:
6428 case llvm::AtomicOrdering::Unordered:
6429 llvm_unreachable(
"Unexpected ordering.");
6436 llvm::AtomicOrdering AO,
6442 if (BO == BO_Comma || !
Update.isScalar() || !
X.isSimple() ||
6444 (
Update.getScalarVal()->getType() !=
X.getAddress().getElementType())) ||
6445 !Context.getTargetInfo().hasBuiltinAtomic(
6446 Context.getTypeSize(
X.getType()), Context.toBits(
X.getAlignment())))
6447 return std::make_pair(
false,
RValue::get(
nullptr));
6450 if (
T->isIntegerTy())
6453 if (
T->isFloatingPointTy() && (BO == BO_Add || BO == BO_Sub))
6459 if (!CheckAtomicSupport(
Update.getScalarVal()->getType(), BO) ||
6460 !CheckAtomicSupport(
X.getAddress().getElementType(), BO))
6461 return std::make_pair(
false,
RValue::get(
nullptr));
6463 bool IsInteger =
X.getAddress().getElementType()->isIntegerTy();
6464 llvm::AtomicRMWInst::BinOp RMWOp;
6467 RMWOp = IsInteger ? llvm::AtomicRMWInst::Add : llvm::AtomicRMWInst::FAdd;
6471 return std::make_pair(
false,
RValue::get(
nullptr));
6472 RMWOp = IsInteger ? llvm::AtomicRMWInst::Sub : llvm::AtomicRMWInst::FSub;
6475 RMWOp = llvm::AtomicRMWInst::And;
6478 RMWOp = llvm::AtomicRMWInst::Or;
6481 RMWOp = llvm::AtomicRMWInst::Xor;
6485 RMWOp =
X.getType()->hasSignedIntegerRepresentation()
6487 : llvm::AtomicRMWInst::Max)
6489 : llvm::AtomicRMWInst::UMax);
6492 : llvm::AtomicRMWInst::FMax;
6496 RMWOp =
X.getType()->hasSignedIntegerRepresentation()
6498 : llvm::AtomicRMWInst::Min)
6500 : llvm::AtomicRMWInst::UMin);
6503 : llvm::AtomicRMWInst::FMin;
6506 RMWOp = llvm::AtomicRMWInst::Xchg;
6515 return std::make_pair(
false,
RValue::get(
nullptr));
6534 llvm_unreachable(
"Unsupported atomic update operation");
6536 llvm::Value *UpdateVal =
Update.getScalarVal();
6537 if (
auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
6539 UpdateVal = CGF.
Builder.CreateIntCast(
6540 IC,
X.getAddress().getElementType(),
6541 X.getType()->hasSignedIntegerRepresentation());
6543 UpdateVal = CGF.
Builder.CreateCast(llvm::Instruction::CastOps::UIToFP, IC,
6544 X.getAddress().getElementType());
6546 llvm::AtomicRMWInst *Res =
6563 if (
X.isGlobalReg()) {
6576 llvm::AtomicOrdering AO,
const Expr *
X,
6580 "Update expr in 'atomic update' must be a binary operator.");
6588 assert(
X->isLValue() &&
"X of 'omp atomic update' is not lvalue");
6595 auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](
RValue XRValue) {
6601 XLValue, ExprRValue, BOUE->getOpcode(),
IsXLHSInRHSPart, AO, Loc, Gen);
6608 case llvm::AtomicOrdering::Release:
6609 case llvm::AtomicOrdering::AcquireRelease:
6610 case llvm::AtomicOrdering::SequentiallyConsistent:
6612 llvm::AtomicOrdering::Release);
6614 case llvm::AtomicOrdering::Acquire:
6615 case llvm::AtomicOrdering::Monotonic:
6617 case llvm::AtomicOrdering::NotAtomic:
6618 case llvm::AtomicOrdering::Unordered:
6619 llvm_unreachable(
"Unexpected ordering.");
6637 llvm_unreachable(
"Must be a scalar or complex.");
6641 llvm::AtomicOrdering AO,
6646 assert(
X->isLValue() &&
"X of 'omp atomic capture' is not lvalue");
6647 assert(
V->isLValue() &&
"V of 'omp atomic capture' is not lvalue");
6656 "Update expr in 'atomic capture' must be a binary operator.");
6667 NewVValType = XRValExpr->
getType();
6669 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
6678 XLValue, ExprRValue, BOUE->getOpcode(),
IsXLHSInRHSPart, AO, Loc, Gen);
6684 NewVVal = Res.second;
6695 NewVValType =
X->getType().getNonReferenceType();
6697 X->getType().getNonReferenceType(), Loc);
6698 auto &&Gen = [&NewVVal, ExprRValue](
RValue XRValue) {
6704 XLValue, ExprRValue, BO_Assign,
false, AO,
6725 case llvm::AtomicOrdering::Release:
6727 llvm::AtomicOrdering::Release);
6729 case llvm::AtomicOrdering::Acquire:
6731 llvm::AtomicOrdering::Acquire);
6733 case llvm::AtomicOrdering::AcquireRelease:
6734 case llvm::AtomicOrdering::SequentiallyConsistent:
6736 CGF, {}, Loc, llvm::AtomicOrdering::AcquireRelease);
6738 case llvm::AtomicOrdering::Monotonic:
6740 case llvm::AtomicOrdering::NotAtomic:
6741 case llvm::AtomicOrdering::Unordered:
6742 llvm_unreachable(
"Unexpected ordering.");
6748 CodeGenFunction &CGF, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO,
6752 llvm::OpenMPIRBuilder &OMPBuilder =
6755 OMPAtomicCompareOp Op;
6759 Op = OMPAtomicCompareOp::EQ;
6762 Op = OMPAtomicCompareOp::MIN;
6765 Op = OMPAtomicCompareOp::MAX;
6768 llvm_unreachable(
"unsupported atomic compare binary operator");
6772 Address XAddr = XLVal.getAddress();
6774 auto EmitRValueWithCastIfNeeded = [&CGF, Loc](
const Expr *
X,
const Expr *E) {
6779 if (NewE->
getType() ==
X->getType())
6784 llvm::Value *EVal = EmitRValueWithCastIfNeeded(
X, E);
6785 llvm::Value *DVal = D ? EmitRValueWithCastIfNeeded(
X, D) :
nullptr;
6786 if (
auto *CI = dyn_cast<llvm::ConstantInt>(EVal))
6787 EVal = CGF.
Builder.CreateIntCast(
6788 CI, XLVal.getAddress().getElementType(),
6791 if (
auto *CI = dyn_cast<llvm::ConstantInt>(DVal))
6792 DVal = CGF.
Builder.CreateIntCast(
6793 CI, XLVal.getAddress().getElementType(),
6796 llvm::OpenMPIRBuilder::AtomicOpValue XOpVal{
6798 X->getType()->hasSignedIntegerRepresentation(),
6799 X->getType().isVolatileQualified()};
6800 llvm::OpenMPIRBuilder::AtomicOpValue VOpVal, ROpVal;
6804 VOpVal = {
Addr.emitRawPointer(CGF),
Addr.getElementType(),
6805 V->getType()->hasSignedIntegerRepresentation(),
6806 V->getType().isVolatileQualified()};
6811 ROpVal = {
Addr.emitRawPointer(CGF),
Addr.getElementType(),
6816 if (FailAO == llvm::AtomicOrdering::NotAtomic) {
6819 CGF.
Builder.restoreIP(OMPBuilder.createAtomicCompare(
6820 CGF.
Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr,
6823 CGF.
Builder.restoreIP(OMPBuilder.createAtomicCompare(
6824 CGF.
Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr,
6829 llvm::AtomicOrdering AO,
6850 case OMPC_compare: {
6856 llvm_unreachable(
"Clause is not allowed in 'omp atomic'.");
6861 llvm::AtomicOrdering AO =
CGM.getOpenMPRuntime().getDefaultMemoryOrdering();
6863 llvm::AtomicOrdering FailAO = llvm::AtomicOrdering::NotAtomic;
6864 bool MemOrderingSpecified =
false;
6866 AO = llvm::AtomicOrdering::SequentiallyConsistent;
6867 MemOrderingSpecified =
true;
6869 AO = llvm::AtomicOrdering::AcquireRelease;
6870 MemOrderingSpecified =
true;
6872 AO = llvm::AtomicOrdering::Acquire;
6873 MemOrderingSpecified =
true;
6875 AO = llvm::AtomicOrdering::Release;
6876 MemOrderingSpecified =
true;
6878 AO = llvm::AtomicOrdering::Monotonic;
6879 MemOrderingSpecified =
true;
6881 llvm::SmallSet<OpenMPClauseKind, 2> KindsEncountered;
6890 if (K == OMPC_seq_cst || K == OMPC_acq_rel || K == OMPC_acquire ||
6891 K == OMPC_release || K == OMPC_relaxed || K == OMPC_hint)
6894 KindsEncountered.insert(K);
6899 if (KindsEncountered.contains(OMPC_compare) &&
6900 KindsEncountered.contains(OMPC_capture))
6901 Kind = OMPC_compare;
6902 if (!MemOrderingSpecified) {
6903 llvm::AtomicOrdering DefaultOrder =
6904 CGM.getOpenMPRuntime().getDefaultMemoryOrdering();
6905 if (DefaultOrder == llvm::AtomicOrdering::Monotonic ||
6906 DefaultOrder == llvm::AtomicOrdering::SequentiallyConsistent ||
6907 (DefaultOrder == llvm::AtomicOrdering::AcquireRelease &&
6908 Kind == OMPC_capture)) {
6910 }
else if (DefaultOrder == llvm::AtomicOrdering::AcquireRelease) {
6911 if (Kind == OMPC_unknown || Kind == OMPC_update || Kind == OMPC_write) {
6912 AO = llvm::AtomicOrdering::Release;
6913 }
else if (Kind == OMPC_read) {
6914 assert(Kind == OMPC_read &&
"Unexpected atomic kind.");
6915 AO = llvm::AtomicOrdering::Acquire;
6920 if (KindsEncountered.contains(OMPC_compare) &&
6921 KindsEncountered.contains(OMPC_fail)) {
6922 Kind = OMPC_compare;
6923 const auto *FailClause = S.getSingleClause<
OMPFailClause>();
6926 if (FailParameter == llvm::omp::OMPC_relaxed)
6927 FailAO = llvm::AtomicOrdering::Monotonic;
6928 else if (FailParameter == llvm::omp::OMPC_acquire)
6929 FailAO = llvm::AtomicOrdering::Acquire;
6930 else if (FailParameter == llvm::omp::OMPC_seq_cst)
6931 FailAO = llvm::AtomicOrdering::SequentiallyConsistent;
6938 S.getV(), S.getR(), S.getExpr(), S.getUpdateExpr(),
6939 S.getD(), S.getCondExpr(), S.isXLHSInRHSPart(),
6940 S.isFailOnly(), S.getBeginLoc());
6951 OMPLexicalScope
Scope(CGF, S, OMPD_target);
6954 CGF.
EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
6960 llvm::Function *Fn =
nullptr;
6961 llvm::Constant *FnID =
nullptr;
6963 const Expr *IfCond =
nullptr;
6965 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
6966 if (
C->getNameModifier() == OMPD_unknown ||
6967 C->getNameModifier() == OMPD_target) {
6968 IfCond =
C->getCondition();
6974 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>
Device(
6977 Device.setPointerAndInt(
C->getDevice(),
C->getModifier());
6982 bool IsOffloadEntry =
true;
6986 IsOffloadEntry =
false;
6989 IsOffloadEntry =
false;
6991 if (
CGM.
getLangOpts().OpenMPOffloadMandatory && !IsOffloadEntry) {
6994 "No offloading entry generated while offloading is mandatory.");
6998 assert(CGF.
CurFuncDecl &&
"No parent declaration for target region!");
6999 StringRef ParentName;
7002 if (
const auto *D = dyn_cast<CXXConstructorDecl>(CGF.
CurFuncDecl))
7004 else if (
const auto *D = dyn_cast<CXXDestructorDecl>(CGF.
CurFuncDecl))
7013 OMPLexicalScope
Scope(CGF, S, OMPD_task);
7014 auto &&SizeEmitter =
7017 if (IsOffloadEntry) {
7018 OMPLoopScope(CGF, D);
7020 llvm::Value *NumIterations = CGF.
EmitScalarExpr(D.getNumIterations());
7021 NumIterations = CGF.
Builder.CreateIntCast(NumIterations, CGF.
Int64Ty,
7023 return NumIterations;
7041 CGF.
EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt());
7046 StringRef ParentName,
7052 llvm::Constant *
Addr;
7054 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7056 assert(Fn &&
Addr &&
"Target device function emission failed.");
7070 const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);
7071 llvm::Function *OutlinedFn =
7079 const Expr *NumTeams = NT ? NT->getNumTeams().front() :
nullptr;
7080 const Expr *ThreadLimit = TL ? TL->getThreadLimit().front() :
nullptr;
7086 OMPTeamsScope
Scope(CGF, S);
7102 CGF.
EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt());
7107 [](CodeGenFunction &) {
return nullptr; });
7112 auto *CS = S.getCapturedStmt(OMPD_teams);
7139 llvm::Constant *
Addr;
7141 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7143 assert(Fn &&
Addr &&
"Target device function emission failed.");
7185 llvm::Constant *
Addr;
7187 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7189 assert(Fn &&
Addr &&
"Target device function emission failed.");
7231 llvm::Constant *
Addr;
7233 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7235 assert(Fn &&
Addr &&
"Target device function emission failed.");
7249 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF,
PrePostActionTy &) {
7254 auto &&
CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7266 [](CodeGenFunction &) {
return nullptr; });
7271 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF,
PrePostActionTy &) {
7276 auto &&
CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7288 [](CodeGenFunction &) {
return nullptr; });
7293 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF,
PrePostActionTy &) {
7299 auto &&
CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7311 [](CodeGenFunction &) {
return nullptr; });
7316 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF,
PrePostActionTy &) {
7322 auto &&
CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7329 CGF, OMPD_distribute, CodeGenDistribute,
false);
7335 [](CodeGenFunction &) {
return nullptr; });
7339 llvm::OpenMPIRBuilder &OMPBuilder =
CGM.getOpenMPRuntime().getOMPBuilder();
7340 llvm::Value *
Device =
nullptr;
7341 llvm::Value *NumDependences =
nullptr;
7342 llvm::Value *DependenceList =
nullptr;
7350 if (!
Data.Dependences.empty()) {
7352 std::tie(NumDependences, DependenciesArray) =
7353 CGM.getOpenMPRuntime().emitDependClause(*
this,
Data.Dependences,
7362 "OMPNowaitClause clause is used separately in OMPInteropDirective.");
7365 if (!ItOMPInitClause.empty()) {
7368 llvm::Value *InteropvarPtr =
7370 llvm::omp::OMPInteropType InteropType =
7371 llvm::omp::OMPInteropType::Unknown;
7372 if (
C->getIsTarget()) {
7373 InteropType = llvm::omp::OMPInteropType::Target;
7375 assert(
C->getIsTargetSync() &&
7376 "Expected interop-type target/targetsync");
7377 InteropType = llvm::omp::OMPInteropType::TargetSync;
7379 OMPBuilder.createOMPInteropInit(
Builder, InteropvarPtr, InteropType,
7380 Device, NumDependences, DependenceList,
7381 Data.HasNowaitClause);
7385 if (!ItOMPDestroyClause.empty()) {
7388 llvm::Value *InteropvarPtr =
7390 OMPBuilder.createOMPInteropDestroy(
Builder, InteropvarPtr,
Device,
7391 NumDependences, DependenceList,
7392 Data.HasNowaitClause);
7395 auto ItOMPUseClause = S.getClausesOfKind<
OMPUseClause>();
7396 if (!ItOMPUseClause.empty()) {
7399 llvm::Value *InteropvarPtr =
7401 OMPBuilder.createOMPInteropUse(
Builder, InteropvarPtr,
Device,
7402 NumDependences, DependenceList,
7403 Data.HasNowaitClause);
7425 CGF, OMPD_distribute, CodeGenDistribute,
false);
7444 llvm::Constant *
Addr;
7446 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7448 assert(Fn &&
Addr &&
"Target device function emission failed.");
7477 CGF, OMPD_distribute, CodeGenDistribute,
false);
7496 llvm::Constant *
Addr;
7498 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7500 assert(Fn &&
Addr &&
"Target device function emission failed.");
7513 CGM.getOpenMPRuntime().emitCancellationPointCall(*
this, S.getBeginLoc(),
7518 const Expr *IfCond =
nullptr;
7519 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
7520 if (
C->getNameModifier() == OMPD_unknown ||
7521 C->getNameModifier() == OMPD_cancel) {
7522 IfCond =
C->getCondition();
7526 if (
CGM.getLangOpts().OpenMPIRBuilder) {
7527 llvm::OpenMPIRBuilder &OMPBuilder =
CGM.getOpenMPRuntime().getOMPBuilder();
7533 llvm::Value *IfCondition =
nullptr;
7537 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
7539 return Builder.restoreIP(AfterIP);
7543 CGM.getOpenMPRuntime().emitCancelCall(*
this, S.getBeginLoc(), IfCond,
7549 if (Kind == OMPD_parallel || Kind == OMPD_task ||
7550 Kind == OMPD_target_parallel || Kind == OMPD_taskloop ||
7551 Kind == OMPD_master_taskloop || Kind == OMPD_parallel_master_taskloop)
7553 assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
7554 Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
7555 Kind == OMPD_distribute_parallel_for ||
7556 Kind == OMPD_target_parallel_for ||
7557 Kind == OMPD_teams_distribute_parallel_for ||
7558 Kind == OMPD_target_teams_distribute_parallel_for);
7559 return OMPCancelStack.getExitBlock();
7564 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
7565 CaptureDeviceAddrMap) {
7566 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
7567 for (
const Expr *OrigVarIt :
C.varlist()) {
7569 if (!Processed.insert(OrigVD).second)
7576 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
7581 "Base should be the current struct!");
7582 MatchingVD = ME->getMemberDecl();
7587 auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
7588 if (InitAddrIt == CaptureDeviceAddrMap.end())
7596 Address(InitAddrIt->second, Ty,
7598 assert(IsRegistered &&
"firstprivate var already registered as private");
7606 while (
const auto *OASE = dyn_cast<ArraySectionExpr>(
Base))
7607 Base = OASE->getBase()->IgnoreParenImpCasts();
7608 while (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(
Base))
7609 Base = ASE->getBase()->IgnoreParenImpCasts();
7615 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
7616 CaptureDeviceAddrMap) {
7617 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
7618 for (
const Expr *Ref :
C.varlist()) {
7620 if (!Processed.insert(OrigVD).second)
7626 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
7631 "Base should be the current struct!");
7632 MatchingVD = ME->getMemberDecl();
7637 auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
7638 if (InitAddrIt == CaptureDeviceAddrMap.end())
7644 Address(InitAddrIt->second, Ty,
7657 (void)PrivateScope.
addPrivate(OrigVD, PrivAddr);
7670 bool PrivatizeDevicePointers =
false;
7672 bool &PrivatizeDevicePointers;
7675 explicit DevicePointerPrivActionTy(
bool &PrivatizeDevicePointers)
7676 : PrivatizeDevicePointers(PrivatizeDevicePointers) {}
7677 void Enter(CodeGenFunction &CGF)
override {
7678 PrivatizeDevicePointers =
true;
7681 DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
7684 auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF,
PrePostActionTy &) {
7685 CGF.
EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
7689 auto &&PrivCodeGen = [&](CodeGenFunction &CGF,
PrePostActionTy &Action) {
7691 PrivatizeDevicePointers =
false;
7697 if (PrivatizeDevicePointers) {
7711 std::optional<OpenMPDirectiveKind> CaptureRegion;
7712 if (
CGM.getLangOpts().OMPTargetTriples.empty()) {
7715 for (
const Expr *E :
C->varlist()) {
7717 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
7721 for (
const Expr *E :
C->varlist()) {
7723 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
7727 CaptureRegion = OMPD_unknown;
7730 OMPLexicalScope
Scope(CGF, S, CaptureRegion);
7742 OMPLexicalScope
Scope(CGF, S);
7751 if (
CGM.getLangOpts().OMPTargetTriples.empty()) {
7757 const Expr *IfCond =
nullptr;
7759 IfCond =
C->getCondition();
7770 CGM.getOpenMPRuntime().emitTargetDataCalls(*
this, S, IfCond,
Device, RCG,
7778 if (
CGM.getLangOpts().OMPTargetTriples.empty())
7782 const Expr *IfCond =
nullptr;
7784 IfCond =
C->getCondition();
7791 OMPLexicalScope
Scope(*
this, S, OMPD_task);
7792 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*
this, S, IfCond,
Device);
7799 if (
CGM.getLangOpts().OMPTargetTriples.empty())
7803 const Expr *IfCond =
nullptr;
7805 IfCond =
C->getCondition();
7812 OMPLexicalScope
Scope(*
this, S, OMPD_task);
7813 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*
this, S, IfCond,
Device);
7820 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
7848 llvm::Constant *
Addr;
7850 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7852 assert(Fn &&
Addr &&
"Target device function emission failed.");
7872 CGF, OMPD_target_parallel_for, S.
hasCancel());
7888 llvm::Constant *
Addr;
7890 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7892 assert(Fn &&
Addr &&
"Target device function emission failed.");
7927 llvm::Constant *
Addr;
7929 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7931 assert(Fn &&
Addr &&
"Target device function emission failed.");
7953 const CapturedStmt *CS = S.getCapturedStmt(OMPD_taskloop);
7956 OMPLexicalScope
Scope(*
this, S, OMPD_taskloop,
false);
7961 const Expr *IfCond =
nullptr;
7962 for (
const auto *
C : S.getClausesOfKind<
OMPIfClause>()) {
7963 if (
C->getNameModifier() == OMPD_unknown ||
7964 C->getNameModifier() == OMPD_taskloop) {
7965 IfCond =
C->getCondition();
7978 Data.Schedule.setInt(
false);
7981 (Clause->getModifier() == OMPC_GRAINSIZE_strict) ?
true :
false;
7984 Data.Schedule.setInt(
true);
7987 (Clause->getModifier() == OMPC_NUMTASKS_strict) ?
true :
false;
8001 llvm::BasicBlock *ContBlock =
nullptr;
8002 OMPLoopScope PreInitScope(CGF, S);
8003 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
8007 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock(
"taskloop.if.then");
8008 ContBlock = CGF.createBasicBlock(
"taskloop.if.end");
8009 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
8010 CGF.getProfileCount(&S));
8011 CGF.EmitBlock(ThenBlock);
8012 CGF.incrementProfileCounter(&S);
8015 (void)CGF.EmitOMPLinearClauseInit(S);
8019 enum { LowerBound = 5, UpperBound, Stride, LastIter };
8021 auto *LBP = std::next(I, LowerBound);
8022 auto *UBP = std::next(I, UpperBound);
8023 auto *STP = std::next(I, Stride);
8024 auto *LIP = std::next(I, LastIter);
8032 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
8033 CGF.EmitOMPLinearClause(S, LoopScope);
8034 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
8037 const Expr *IVExpr = S.getIterationVariable();
8039 CGF.EmitVarDecl(*IVDecl);
8040 CGF.EmitIgnoredExpr(S.getInit());
8045 if (
const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
8048 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
8052 OMPLexicalScope
Scope(CGF, S, OMPD_taskloop,
false);
8062 [&S](CodeGenFunction &CGF) {
8063 emitOMPLoopBodyWithStopPoint(CGF, S,
8064 CodeGenFunction::JumpDest());
8066 [](CodeGenFunction &) {});
8071 CGF.EmitBranch(ContBlock);
8072 CGF.EmitBlock(ContBlock,
true);
8075 if (HasLastprivateClause) {
8076 CGF.EmitOMPLastprivateClauseFinal(
8078 CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
8079 CGF.GetAddrOfLocalVar(*LIP),
false,
8080 (*LIP)->getType(), S.getBeginLoc())));
8083 CGF.EmitOMPLinearClauseFinal(S, [LIP, &S](CodeGenFunction &CGF) {
8084 return CGF.
Builder.CreateIsNotNull(
8086 (*LIP)->
getType(), S.getBeginLoc()));
8089 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
8090 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
8092 auto &&
CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,
8094 OMPLoopScope PreInitScope(CGF, S);
8095 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S,
8096 OutlinedFn, SharedsTy,
8097 CapturedStruct, IfCond,
Data);
8099 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
8105 CGM.getOpenMPRuntime().emitTaskgroupRegion(
8107 [&S, &BodyGen, &TaskGen, &
Data](CodeGenFunction &CGF,
8127 OMPLexicalScope
Scope(*
this, S);
8139 OMPLexicalScope
Scope(*
this, S, std::nullopt,
false);
8140 CGM.getOpenMPRuntime().emitMasterRegion(*
this,
CodeGen, S.getBeginLoc());
8151 OMPLexicalScope
Scope(*
this, S, std::nullopt,
false);
8152 CGM.getOpenMPRuntime().emitMaskedRegion(*
this,
CodeGen, S.getBeginLoc());
8163 OMPLexicalScope
Scope(*
this, S);
8164 CGM.getOpenMPRuntime().emitMasterRegion(*
this,
CodeGen, S.getBeginLoc());
8175 OMPLexicalScope
Scope(*
this, S);
8176 CGM.getOpenMPRuntime().emitMaskedRegion(*
this,
CodeGen, S.getBeginLoc());
8182 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8187 OMPLexicalScope
Scope(CGF, S, OMPD_parallel,
false);
8188 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen,
8200 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8205 OMPLexicalScope
Scope(CGF, S, OMPD_parallel,
false);
8206 CGM.getOpenMPRuntime().emitMaskedRegion(CGF, TaskLoopCodeGen,
8218 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8223 OMPLexicalScope
Scope(CGF, S, OMPD_parallel,
false);
8224 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen,
8236 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8241 OMPLexicalScope
Scope(CGF, S, OMPD_parallel,
false);
8242 CGM.getOpenMPRuntime().emitMaskedRegion(CGF, TaskLoopCodeGen,
8256 if (
CGM.getLangOpts().OMPTargetTriples.empty())
8260 const Expr *IfCond =
nullptr;
8262 IfCond =
C->getCondition();
8269 OMPLexicalScope
Scope(*
this, S, OMPD_task);
8270 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*
this, S, IfCond,
Device);
8280 BindKind =
C->getBindKind();
8283 case OMPC_BIND_parallel:
8285 case OMPC_BIND_teams:
8287 case OMPC_BIND_thread:
8298 const auto *ForS = dyn_cast<ForStmt>(CS);
8309 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
8310 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_loop,
CodeGen);
8336 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF,
PrePostActionTy &) {
8341 auto &&
CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
8353 [](CodeGenFunction &) {
return nullptr; });
8358 std::string StatusMsg,
8362 StatusMsg +=
": DEVICE";
8364 StatusMsg +=
": HOST";
8371 llvm::dbgs() << StatusMsg <<
": " <<
FileName <<
": " << LineNo <<
"\n";
8394 CGF, OMPD_distribute, CodeGenDistribute,
false);
8423 CGF, OMPD_distribute, CodeGenDistribute,
false);
8456 llvm::Constant *
Addr;
8458 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
8460 assert(Fn &&
Addr &&
8461 "Target device function emission failed for 'target teams loop'.");
8472 CGF, OMPD_target_parallel_loop,
false);
8488 llvm::Constant *
Addr;
8490 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
8492 assert(Fn &&
Addr &&
"Target device function emission failed.");
8507 if (
const auto *SD = dyn_cast<OMPScanDirective>(&D)) {
8511 if (!D.hasAssociatedStmt() || !D.getAssociatedStmt())
8518 for (
const Expr *Ref :
C->varlist()) {
8522 const auto *VD = dyn_cast<VarDecl>(DRE->getDecl());
8525 if (!CGF.LocalDeclMap.count(VD)) {
8537 if (
const auto *LD = dyn_cast<OMPLoopDirective>(&D)) {
8538 for (
const Expr *E : LD->counters()) {
8546 if (!CGF.LocalDeclMap.count(VD))
8551 if (!
C->getNumForLoops())
8553 for (
unsigned I = LD->getLoopsNumber(),
8554 E =
C->getLoopNumIterations().size();
8556 if (
const auto *VD = dyn_cast<OMPCapturedExprDecl>(
8559 if (!CGF.LocalDeclMap.count(VD))
8566 CGF.
EmitStmt(D.getInnermostCapturedStmt()->getCapturedStmt());
8569 if (D.getDirectiveKind() == OMPD_atomic ||
8570 D.getDirectiveKind() == OMPD_critical ||
8571 D.getDirectiveKind() == OMPD_section ||
8572 D.getDirectiveKind() == OMPD_master ||
8573 D.getDirectiveKind() == OMPD_masked ||
8574 D.getDirectiveKind() == OMPD_unroll ||
8575 D.getDirectiveKind() == OMPD_assume) {
8580 OMPSimdLexicalScope
Scope(*
this, D);
8581 CGM.getOpenMPRuntime().emitInlinedDirective(
8584 : D.getDirectiveKind(),
Defines the clang::ASTContext interface.
static bool isAllocatableDecl(const VarDecl *VD)
static const VarDecl * getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE)
static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S, PrePostActionTy &Action)
static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S, PrePostActionTy &Action)
static const VarDecl * getBaseDecl(const Expr *Ref)
static void emitTargetTeamsGenericLoopRegionAsParallel(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsGenericLoopDirective &S)
static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, const Expr *X, const Expr *V, SourceLocation Loc)
static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, bool IsPostfixUpdate, const Expr *V, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
static void emitScanBasedDirective(CodeGenFunction &CGF, const OMPLoopDirective &S, llvm::function_ref< llvm::Value *(CodeGenFunction &)> NumIteratorsGen, llvm::function_ref< void(CodeGenFunction &)> FirstGen, llvm::function_ref< void(CodeGenFunction &)> SecondGen)
Emits the code for the directive with inscan reductions.
static void emitSimpleAtomicStore(CodeGenFunction &CGF, llvm::AtomicOrdering AO, LValue LVal, RValue RVal)
static bool isSupportedByOpenMPIRBuilder(const OMPTaskgroupDirective &T)
static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, QualType DstType, StringRef Name, LValue AddrLV)
static void emitDistributeParallelForDistributeInnerBoundParams(CodeGenFunction &CGF, const OMPExecutableDirective &S, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars)
static void emitScanBasedDirectiveFinals(CodeGenFunction &CGF, const OMPLoopDirective &S, llvm::function_ref< llvm::Value *(CodeGenFunction &)> NumIteratorsGen)
Copies final inscan reductions values to the original variables.
static void checkForLastprivateConditionalUpdate(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static std::pair< LValue, LValue > emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
The following two functions generate expressions for the loop lower and upper bounds in case of stati...
static void emitTargetParallelForRegion(CodeGenFunction &CGF, const OMPTargetParallelForDirective &S, PrePostActionTy &Action)
static LValue EmitOMPHelperVar(CodeGenFunction &CGF, const DeclRefExpr *Helper)
Emit a helper variable and return corresponding lvalue.
static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
static llvm::Value * convertToScalarValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
static llvm::Function * emitOutlinedOrderedFunction(CodeGenModule &CGM, const CapturedStmt *S, const OMPExecutableDirective &D)
static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
static std::pair< bool, RValue > emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, RValue Update, BinaryOperatorKind BO, llvm::AtomicOrdering AO, bool IsXLHSInRHSPart)
static std::pair< LValue, LValue > emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitTargetTeamsGenericLoopRegionAsDistribute(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsGenericLoopDirective &S)
static void emitTargetParallelRegion(CodeGenFunction &CGF, const OMPTargetParallelDirective &S, PrePostActionTy &Action)
static std::pair< llvm::Value *, llvm::Value * > emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
When dealing with dispatch schedules (e.g.
static void emitMaster(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitRestoreIP(CodeGenFunction &CGF, const T *C, llvm::OpenMPIRBuilder::InsertPointTy AllocaIP, llvm::OpenMPIRBuilder &OMPBuilder)
static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, const RegionCodeGenTy &CodeGen)
static void emitSimdlenSafelenClause(CodeGenFunction &CGF, const OMPExecutableDirective &D)
static void emitAlignedClause(CodeGenFunction &CGF, const OMPExecutableDirective &D)
static bool isSimdSupportedByOpenMPIRBuilder(const OMPLoopDirective &S)
static void emitCommonOMPParallelDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, const CodeGenBoundParametersTy &CodeGenBoundParameters)
static void applyConservativeSimdOrderedDirective(const Stmt &AssociatedStmt, LoopInfoStack &LoopStack)
static bool emitWorksharingDirective(CodeGenFunction &CGF, const OMPLoopDirective &S, bool HasCancel)
static void emitPostUpdateForReductionClause(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc, const unsigned IVSize, const bool IVSigned)
static void emitTargetTeamsLoopCodegenStatus(CodeGenFunction &CGF, std::string StatusMsg, const OMPExecutableDirective &D)
static bool isForSupportedByOpenMPIRBuilder(const OMPLoopDirective &S, bool HasCancel)
static RValue emitSimpleAtomicLoad(CodeGenFunction &CGF, llvm::AtomicOrdering AO, LValue LVal, SourceLocation Loc)
static std::pair< llvm::Value *, llvm::Value * > emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
if the 'for' loop has a dispatch schedule (e.g.
static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO, bool IsPostfixUpdate, const Expr *X, const Expr *V, const Expr *R, const Expr *E, const Expr *UE, const Expr *D, const Expr *CE, bool IsXLHSInRHSPart, bool IsFailOnly, SourceLocation Loc)
static CodeGenFunction::ComplexPairTy convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
static ImplicitParamDecl * createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data, QualType Ty, CapturedDecl *CD, SourceLocation Loc)
static EmittedClosureTy emitCapturedStmtFunc(CodeGenFunction &ParentCGF, const CapturedStmt *S)
Emit a captured statement and return the function as well as its captured closure context.
static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
static void emitOMPDistributeDirective(const OMPLoopDirective &S, CodeGenFunction &CGF, CodeGenModule &CGM)
static void emitOMPCopyinClause(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitTargetTeamsDistributeParallelForRegion(CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S, PrePostActionTy &Action)
static bool hasOrderedDirective(const Stmt *S)
static llvm::CallInst * emitCapturedStmtCall(CodeGenFunction &ParentCGF, EmittedClosureTy Cap, llvm::ArrayRef< llvm::Value * > Args)
Emit a call to a previously captured closure.
static void emitMasked(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop, int MaxLevel, int Level=0)
static void emitOMPForDirective(const OMPLoopDirective &S, CodeGenFunction &CGF, CodeGenModule &CGM, bool HasCancel)
static void emitEmptyBoundParameters(CodeGenFunction &, const OMPExecutableDirective &, llvm::SmallVectorImpl< llvm::Value * > &)
static void emitTargetParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetParallelForSimdDirective &S, PrePostActionTy &Action)
static void emitOMPSimdDirective(const OMPLoopDirective &S, CodeGenFunction &CGF, CodeGenModule &CGM)
static void emitOMPAtomicCompareExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO, const Expr *X, const Expr *V, const Expr *R, const Expr *E, const Expr *D, const Expr *CE, bool IsXBinopExpr, bool IsPostfixUpdate, bool IsFailOnly, SourceLocation Loc)
std::pair< llvm::Function *, llvm::Value * > EmittedClosureTy
static OpenMPDirectiveKind getEffectiveDirectiveKind(const OMPExecutableDirective &S)
static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDirective &S)
static void buildDependences(const OMPExecutableDirective &S, OMPTaskDataTy &Data)
static RValue convertToType(CodeGenFunction &CGF, RValue Value, QualType SourceType, QualType ResType, SourceLocation Loc)
static void emitScanBasedDirectiveDecls(CodeGenFunction &CGF, const OMPLoopDirective &S, llvm::function_ref< llvm::Value *(CodeGenFunction &)> NumIteratorsGen)
Emits internal temp array declarations for the directive with inscan reductions.
static void emitTargetTeamsDistributeParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForSimdDirective &S, PrePostActionTy &Action)
static void emitTargetTeamsDistributeSimdRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeSimdDirective &S)
static llvm::MapVector< llvm::Value *, llvm::Value * > GetAlignedMapping(const OMPLoopDirective &S, CodeGenFunction &CGF)
static llvm::omp::ScheduleKind convertClauseKindToSchedKind(OpenMPScheduleClauseKind ScheduleClauseKind)
static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, const ImplicitParamDecl *PVD, CodeGenFunction::OMPPrivateScope &Privates)
Emit a helper variable and return corresponding lvalue.
static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
static void emitTargetParallelGenericLoopRegion(CodeGenFunction &CGF, const OMPTargetParallelGenericLoopDirective &S, PrePostActionTy &Action)
static QualType getCanonicalParamType(ASTContext &C, QualType T)
static void emitCommonSimdLoop(CodeGenFunction &CGF, const OMPLoopDirective &S, const RegionCodeGenTy &SimdInitGen, const RegionCodeGenTy &BodyCodeGen)
static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, const Twine &Name, llvm::Value *Init=nullptr)
static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, const Expr *X, const Expr *E, SourceLocation Loc)
static llvm::Function * emitOutlinedFunctionPrologue(CodeGenFunction &CGF, FunctionArgList &Args, llvm::MapVector< const Decl *, std::pair< const VarDecl *, Address > > &LocalAddrs, llvm::DenseMap< const Decl *, std::pair< const Expr *, llvm::Value * > > &VLASizes, llvm::Value *&CXXThisValue, const FunctionOptions &FO)
static void emitInnerParallelForWhenCombined(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
static void emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeDirective &S)
This file defines OpenMP nodes for declarative directives.
static const Decl * getCanonicalDecl(const Decl *D)
This file defines OpenMP AST classes for clauses.
Defines some OpenMP-specific enums and functions.
Defines the PrettyStackTraceEntry class, which is used to make crashes give more contextual informati...
Defines the SourceManager interface.
This file defines OpenMP AST classes for executable directives and clauses.
This represents 'pragma omp cancel' directive.
OpenMPDirectiveKind getCancelRegion() const
Get cancellation region for the current cancellation point.
This represents 'pragma omp cancellation point' directive.
OpenMPDirectiveKind getCancelRegion() const
Get cancellation region for the current cancellation point.
This represents 'pragma omp distribute' directive.
This represents 'pragma omp distribute parallel for' composite directive.
This represents 'pragma omp distribute parallel for simd' composite directive.
This represents 'pragma omp distribute simd' composite directive.
This represents 'pragma omp error' directive.
Represents the 'pragma omp fuse' loop transformation directive.
Stmt * getTransformedStmt() const
Gets the associated loops after the transformation.
This represents 'pragma omp loop' directive.
Represents the 'pragma omp interchange' loop transformation directive.
Stmt * getTransformedStmt() const
Gets the associated loops after the transformation.
This represents 'pragma omp interop' directive.
This represents 'pragma omp masked' directive.
This represents 'pragma omp masked taskloop' directive.
This represents 'pragma omp masked taskloop simd' directive.
This represents 'pragma omp master taskloop' directive.
This represents 'pragma omp master taskloop simd' directive.
This represents 'pragma omp parallel masked taskloop' directive.
This represents 'pragma omp parallel masked taskloop simd' directive.
This represents 'pragma omp parallel master taskloop' directive.
This represents 'pragma omp parallel master taskloop simd' directive.
Represents the 'pragma omp reverse' loop transformation directive.
Stmt * getTransformedStmt() const
Gets/sets the associated loops after the transformation, i.e.
This represents 'pragma omp scan' directive.
This represents the 'pragma omp stripe' loop transformation directive.
Stmt * getTransformedStmt() const
Gets/sets the associated loops after striping.
This represents 'pragma omp target data' directive.
This represents 'pragma omp target' directive.
This represents 'pragma omp target enter data' directive.
This represents 'pragma omp target exit data' directive.
This represents 'pragma omp target parallel' directive.
This represents 'pragma omp target parallel for' directive.
bool hasCancel() const
Return true if current directive has inner cancel directive.
This represents 'pragma omp target parallel for simd' directive.
This represents 'pragma omp target parallel loop' directive.
This represents 'pragma omp target simd' directive.
This represents 'pragma omp target teams' directive.
This represents 'pragma omp target teams distribute' combined directive.
This represents 'pragma omp target teams distribute parallel for' combined directive.
This represents 'pragma omp target teams distribute parallel for simd' combined directive.
This represents 'pragma omp target teams distribute simd' combined directive.
This represents 'pragma omp target teams loop' directive.
bool canBeParallelFor() const
Return true if current loop directive's associated loop can be a parallel for.
This represents 'pragma omp target update' directive.
This represents 'pragma omp taskloop' directive.
This represents 'pragma omp taskloop simd' directive.
This represents 'pragma omp teams' directive.
This represents 'pragma omp teams distribute' directive.
This represents 'pragma omp teams distribute parallel for' composite directive.
This represents 'pragma omp teams distribute parallel for simd' composite directive.
This represents 'pragma omp teams distribute simd' combined directive.
This represents 'pragma omp teams loop' directive.
This represents the 'pragma omp tile' loop transformation directive.
Stmt * getTransformedStmt() const
Gets/sets the associated loops after tiling.
This represents the 'pragma omp unroll' loop transformation directive.
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
SourceManager & getSourceManager()
TranslationUnitDecl * getTranslationUnitDecl() const
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
QualType getConstantArrayType(QualType EltTy, const llvm::APInt &ArySize, const Expr *SizeExpr, ArraySizeModifier ASM, unsigned IndexTypeQuals) const
Return the unique reference to the type for a constant array of the specified element type.
QualType getUIntPtrType() const
Return a type compatible with "uintptr_t" (C99 7.18.1.4), as defined by the target.
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth,...
TypeSourceInfo * getTrivialTypeSourceInfo(QualType T, SourceLocation Loc=SourceLocation()) const
Allocate a TypeSourceInfo where all locations have been initialized to a given location,...
unsigned getOpenMPDefaultSimdAlign(QualType T) const
Get default simd alignment of the specified complete type in bits.
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
QualType getFunctionType(QualType ResultTy, ArrayRef< QualType > Args, const FunctionProtoType::ExtProtoInfo &EPI) const
Return a normal function type with a typed argument list.
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
CanQualType getCanonicalTagType(const TagDecl *TD) const
ASTRecordLayout - This class contains layout information for one RecordDecl, which is a struct/union/...
uint64_t getFieldOffset(unsigned FieldNo) const
getFieldOffset - Get the offset of the given field index, in bits.
Represents an array type, per C99 6.7.5.2 - Array Declarators.
Represents an attribute applied to a statement.
ArrayRef< const Attr * > getAttrs() const
static BinaryOperator * Create(const ASTContext &C, Expr *lhs, Expr *rhs, Opcode opc, QualType ResTy, ExprValueKind VK, ExprObjectKind OK, SourceLocation opLoc, FPOptionsOverride FPFeatures)
Represents the body of a CapturedStmt, and serves as its DeclContext.
unsigned getContextParamPosition() const
static CapturedDecl * Create(ASTContext &C, DeclContext *DC, unsigned NumParams)
param_iterator param_end() const
Retrieve an iterator one past the last parameter decl.
param_iterator param_begin() const
Retrieve an iterator pointing to the first parameter decl.
Stmt * getBody() const override
getBody - If this Decl represents a declaration for a body of code, such as a function or method defi...
ImplicitParamDecl * getParam(unsigned i) const
This captures a statement into a function.
SourceLocation getEndLoc() const LLVM_READONLY
CapturedDecl * getCapturedDecl()
Retrieve the outlined function declaration.
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Stmt * getCapturedStmt()
Retrieve the statement being captured.
capture_init_iterator capture_init_begin()
Retrieve the first initialization argument.
SourceLocation getBeginLoc() const LLVM_READONLY
capture_init_iterator capture_init_end()
Retrieve the iterator pointing one past the last initialization argument.
Expr *const * const_capture_init_iterator
Const iterator that walks over the capture initialization arguments.
CharUnits - This is an opaque type for sizes expressed in character units.
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
CharUnits alignTo(const CharUnits &Align) const
alignTo - Returns the next integer (mod 2**64) that is greater than or equal to this quantity and is ...
Like RawAddress, an abstract representation of an aligned address, but the pointer contained in this ...
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
CharUnits getAlignment() const
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
Address withAlignment(CharUnits NewAlignment) const
Return address with different alignment, but same pointer and element type.
llvm::PointerType * getType() const
Return the type of the pointer value.
static AggValueSlot ignored()
ignored - Returns an aggregate value slot indicating that the aggregate value is being ignored.
static ApplyDebugLocation CreateDefaultArtificial(CodeGenFunction &CGF, SourceLocation TemporaryLocation)
Apply TemporaryLocation if it is valid.
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
CGFunctionInfo - Class to encapsulate the information about a function definition.
Manages list of lastprivate conditional decls for the specified directive.
static LastprivateConditionalRAII disable(CodeGenFunction &CGF, const OMPExecutableDirective &S)
Manages list of nontemporal decls for the specified directive.
Struct that keeps all the relevant information that should be kept throughout a 'target data' region.
llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap
Map between the a declaration of a capture and the corresponding new llvm address where the runtime r...
Manages list of nontemporal decls for the specified directive.
virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
Emit task region for the task directive.
virtual llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST)
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars, const Expr *IfCond, llvm::Value *NumThreads, OpenMPNumThreadsClauseModifier NumThreadsModifier=OMPC_NUMTHREADS_unknown, OpenMPSeverityClauseKind Severity=OMPC_SEVERITY_fatal, const Expr *Message=nullptr)
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
virtual void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc, OpenMPNumThreadsClauseModifier Modifier=OMPC_NUMTHREADS_unknown, OpenMPSeverityClauseKind Severity=OMPC_SEVERITY_fatal, SourceLocation SeverityLoc=SourceLocation(), const Expr *Message=nullptr, SourceLocation MessageLoc=SourceLocation())
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32global_tid, kmp_int32 num_threads) ...
virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal)
Get the address of void * type of the privatue copy of the reduction item specified by the SharedLVal...
virtual void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc)
This is used for non static scheduled types and when the ordered clause is present on the loop constr...
virtual void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars)
Emits code for teams call of the OutlinedFn with variables captured in a record which address is stor...
virtual const VarDecl * translateParameter(const FieldDecl *FD, const VarDecl *NativeParam) const
Translates the native parameter of outlined function if this is required for target.
virtual llvm::Function * emitTeamsOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP teams directive D.
virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, ArrayRef< Expr * > NumIterations)
Emit initialization for doacross loop nesting support.
virtual void adjustTargetSpecificDataForLambdas(CodeGenFunction &CGF, const OMPExecutableDirective &D) const
Adjust some parameters for the target-based directives, like addresses of the variables captured by r...
virtual Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const
Gets the address of the native argument basing on the address of the target-specific parameter.
virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc)
Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32global_tid, kmp_int32 num_teams,...
virtual llvm::Value * emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, const OMPTaskDataTy &Data)
Emit a code for initialization of task reduction clause.
virtual void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr * > Vars, SourceLocation Loc, llvm::AtomicOrdering AO)
Emit flush of the variables specified in 'omp flush' directive.
virtual void emitProcBindClause(CodeGenFunction &CGF, llvm::omp::ProcBindKind ProcBind, SourceLocation Loc)
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32global_tid, int proc_bind) to generate...
virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false)
Emit an implicit/explicit barrier for OpenMP threads.
virtual void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values)
virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind)
Call the appropriate runtime routine to notify that we finished all the work with current loop.
void emitIfClause(CodeGenFunction &CGF, const Expr *Cond, const RegionCodeGenTy &ThenGen, const RegionCodeGenTy &ElseGen)
Emits code for OpenMP 'if' clause using specified CodeGen function.
virtual llvm::Function * emitParallelOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP parallel directive D.
virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values)
Call the appropriate runtime routine to initialize it before start of loop.
virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static non-chunked.
virtual void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc)
Emits a master region.
virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Required to resolve existing problems in the runtime.
virtual void checkAndEmitLastprivateConditional(CodeGenFunction &CGF, const Expr *LHS)
Checks if the provided LVal is lastprivate conditional and emits the code to update the value of the ...
llvm::OpenMPIRBuilder & getOMPBuilder()
virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen)
Emit outilined function for 'target' directive.
virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned)
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
virtual void checkAndEmitSharedLastprivateConditional(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::DenseSet< CanonicalDeclPtr< const VarDecl > > &IgnoredDecls)
Checks if the lastprivate conditional was updated in inner region and writes the value.
virtual void emitInlinedDirective(CodeGenFunction &CGF, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool HasCancel=false)
Emit code for the directive that does not require outlining.
virtual bool isStaticChunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static chunked.
virtual void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter)
Emit the target offloading code associated with D.
virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const
Check if the specified ScheduleKind is dynamic.
virtual void emitMaskedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc, const Expr *Filter=nullptr)
Emits a masked region.
virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues)
Call the appropriate runtime routine to initialize it before start of loop.
Address getAllocatedAddress() const
Returns the raw, allocated address, which is not necessarily the address of the object itself.
API for captured statement code generation.
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
RAII for correct setting/restoring of CapturedStmtInfo.
LValue getReferenceLValue(CodeGenFunction &CGF, const Expr *RefExpr) const
void ForceCleanup()
Force the emission of cleanups now, instead of waiting until this object is destroyed.
RAII for preserving necessary info during inlined region body codegen.
Cleanup action for allocate support.
RAII for preserving necessary info during Outlined region body codegen.
Controls insertion of cancellation exit blocks in worksharing constructs.
Save/restore original map of previously emitted local vars in case when we need to duplicate emission...
The class used to assign some variables some temporarily addresses.
bool apply(CodeGenFunction &CGF)
Applies new addresses to the list of the variables.
void restore(CodeGenFunction &CGF)
Restores original addresses of the variables.
bool setVarAddr(CodeGenFunction &CGF, const VarDecl *LocalVD, Address TempAddr)
Sets the address of the variable LocalVD to be TempAddr in function CGF.
The scope used to remap some variables as private in the OpenMP loop body (or other captured region e...
void restoreMap()
Restore all mapped variables w/o clean up.
bool Privatize()
Privatizes local variables previously registered as private.
bool addPrivate(const VarDecl *LocalVD, Address Addr)
Registers LocalVD variable as a private with Addr as the address of the corresponding private variabl...
An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
Manages parent directive for scan directives.
Enters a new scope for capturing cleanups, all of which will be executed once the scope is exited.
void ForceCleanup(std::initializer_list< llvm::Value ** > ValuesToReload={})
Force the emission of cleanups now, instead of waiting until this object is destroyed.
bool requiresCleanups() const
Determine whether this scope requires any cleanups.
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
void EmitOMPParallelMaskedTaskLoopDirective(const OMPParallelMaskedTaskLoopDirective &S)
void EmitOMPParallelMaskedDirective(const OMPParallelMaskedDirective &S)
void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S)
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount, Stmt::Likelihood LH=Stmt::LH_None, const Expr *ConditionalOp=nullptr, const VarDecl *ConditionalDecl=nullptr)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D, bool NoFinals, llvm::Value *IsLastIterCond=nullptr)
Emit final copying of lastprivate values to original variables at the end of the worksharing or simd ...
void processInReduction(const OMPExecutableDirective &S, OMPTaskDataTy &Data, CodeGenFunction &CGF, const CapturedStmt *CS, OMPPrivateScope &Scope)
JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target)
The given basic block lies in the current EH scope, but may be a target of a potentially scope-crossi...
void EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S)
void emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy, SourceLocation Loc)
static void EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelDirective &S)
void EmitOMPCanonicalLoop(const OMPCanonicalLoop *S)
Emit an OMPCanonicalLoop using the OpenMPIRBuilder.
void EmitOMPGenericLoopDirective(const OMPGenericLoopDirective &S)
void EmitOMPScanDirective(const OMPScanDirective &S)
static bool hasScalarEvaluationKind(QualType T)
llvm::function_ref< std::pair< llvm::Value *, llvm::Value * >(CodeGenFunction &, const OMPExecutableDirective &S, Address LB, Address UB)> CodeGenDispatchBoundsTy
LValue InitCapturedStruct(const CapturedStmt &S)
CGCapturedStmtInfo * CapturedStmtInfo
void EmitOMPDistributeDirective(const OMPDistributeDirective &S)
void EmitOMPParallelForDirective(const OMPParallelForDirective &S)
void EmitOMPMasterDirective(const OMPMasterDirective &S)
void EmitOMPParallelMasterTaskLoopSimdDirective(const OMPParallelMasterTaskLoopSimdDirective &S)
void EmitOMPSimdInit(const OMPLoopDirective &D)
Helpers for the OpenMP loop directives.
const OMPExecutableDirective * OMPParentLoopDirectiveForScan
Parent loop-based directive for scan directive.
void EmitOMPFlushDirective(const OMPFlushDirective &S)
static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetDirective &S)
Emit device code for the target directive.
bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S)
void EmitOMPTargetTeamsDistributeParallelForSimdDirective(const OMPTargetTeamsDistributeParallelForSimdDirective &S)
static void EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDirective &S)
Emit device code for the target teams directive.
void EmitOMPReductionClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope, bool ForInscan=false)
Emit initial code for reduction variables.
void EmitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &S)
void EmitAutoVarDecl(const VarDecl &D)
EmitAutoVarDecl - Emit an auto variable declaration.
static void EmitOMPTargetTeamsDistributeDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeDirective &S)
Emit device code for the target teams distribute directive.
void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S)
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
void EmitOMPTargetParallelForDirective(const OMPTargetParallelForDirective &S)
const LangOptions & getLangOpts() const
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var)
EmitAutoVarAlloca - Emit the alloca and debug information for a local variable.
void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO, const llvm::function_ref< RValue(RValue)> &UpdateOp, bool IsVolatile)
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Load a pointer with type PtrTy stored at address Ptr.
void EmitBranchThroughCleanup(JumpDest Dest)
EmitBranchThroughCleanup - Emit a branch from the current insert block through the normal cleanup han...
void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind)
Emit final update of reduction values to original variables at the end of the directive.
void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit)
Helper for the OpenMP loop directives.
void EmitOMPScopeDirective(const OMPScopeDirective &S)
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
llvm::AssertingVH< llvm::Instruction > AllocaInsertPt
AllocaInsertPoint - This is an instruction in the entry block before which we prefer to insert alloca...
void EmitAggregateAssign(LValue Dest, LValue Src, QualType EltTy)
Emit an aggregate assignment.
CodeGenFunction * ParentCGF
JumpDest ReturnBlock
ReturnBlock - Unified return block.
void EmitOMPTargetTeamsDistributeSimdDirective(const OMPTargetTeamsDistributeSimdDirective &S)
const llvm::function_ref< void(CodeGenFunction &, llvm::Function *, const OMPTaskDataTy &)> TaskGenTy
llvm::DebugLoc SourceLocToDebugLoc(SourceLocation Location)
Converts Location to a DebugLoc, if debug information is enabled.
bool EmitOMPCopyinClause(const OMPExecutableDirective &D)
Emit code for copyin clause in D directive.
void EmitOMPLinearClause(const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope)
Emit initial code for linear clauses.
llvm::BasicBlock * OMPBeforeScanBlock
void EmitOMPInterchangeDirective(const OMPInterchangeDirective &S)
void EmitOMPPrivateLoopCounters(const OMPLoopDirective &S, OMPPrivateScope &LoopScope)
Emit initial code for loop counters of loop-based directives.
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value * > &CapturedVars)
void EmitOMPDepobjDirective(const OMPDepobjDirective &S)
void EmitOMPMetaDirective(const OMPMetaDirective &S)
void EmitOMPCriticalDirective(const OMPCriticalDirective &S)
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S)
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
void EmitOMPCancelDirective(const OMPCancelDirective &S)
void EmitOMPBarrierDirective(const OMPBarrierDirective &S)
llvm::Value * EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified complex type to the specified destination type,...
void EmitOMPOrderedDirective(const OMPOrderedDirective &S)
bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB, const CodeGenLoopBoundsTy &CodeGenLoopBounds, const CodeGenDispatchBoundsTy &CGDispatchBounds)
Emit code for the worksharing loop-based directive.
LValue EmitOMPSharedLValue(const Expr *E)
Emits the lvalue for the expression with possibly captured variable.
llvm::CanonicalLoopInfo * EmitOMPCollapsedCanonicalLoopNest(const Stmt *S, int Depth)
Emit the Stmt S and return its topmost canonical loop, if any.
void EmitOMPSectionsDirective(const OMPSectionsDirective &S)
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
void EmitOMPInteropDirective(const OMPInteropDirective &S)
void EmitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &S)
void EmitOMPTargetParallelDirective(const OMPTargetParallelDirective &S)
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
llvm::Value * EvaluateExprAsBool(const Expr *E)
EvaluateExprAsBool - Perform the usual unary conversions on the specified expression and compare the ...
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
void EmitOMPTargetParallelForSimdDirective(const OMPTargetParallelForSimdDirective &S)
void EmitOMPTargetParallelGenericLoopDirective(const OMPTargetParallelGenericLoopDirective &S)
Emit combined directive 'target parallel loop' as if its constituent constructs are 'target',...
void EmitOMPUseDeviceAddrClause(const OMPUseDeviceAddrClause &C, OMPPrivateScope &PrivateScope, const llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap)
void EmitOMPTeamsDistributeParallelForSimdDirective(const OMPTeamsDistributeParallelForSimdDirective &S)
void EmitOMPMaskedDirective(const OMPMaskedDirective &S)
llvm::Value * emitArrayLength(const ArrayType *arrayType, QualType &baseType, Address &addr)
emitArrayLength - Compute the length of an array, even if it's a VLA, and drill down to the base elem...
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
void EmitOMPTeamsDistributeSimdDirective(const OMPTeamsDistributeSimdDirective &S)
RValue EmitAtomicLoad(LValue LV, SourceLocation SL, AggValueSlot Slot=AggValueSlot::ignored())
CGDebugInfo * getDebugInfo()
void EmitOMPDistributeLoop(const OMPLoopDirective &S, const CodeGenLoopTy &CodeGenLoop, Expr *IncExpr)
Emit code for the distribute loop-based directive.
void EmitOMPMasterTaskLoopDirective(const OMPMasterTaskLoopDirective &S)
void EmitOMPReverseDirective(const OMPReverseDirective &S)
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
void EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S)
void EmitOMPTargetTeamsDistributeParallelForDirective(const OMPTargetTeamsDistributeParallelForDirective &S)
void EmitOMPMaskedTaskLoopDirective(const OMPMaskedTaskLoopDirective &S)
llvm::BasicBlock * OMPScanDispatch
llvm::function_ref< std::pair< LValue, LValue >(CodeGenFunction &, const OMPExecutableDirective &S)> CodeGenLoopBoundsTy
void EmitOMPTargetExitDataDirective(const OMPTargetExitDataDirective &S)
void incrementProfileCounter(const Stmt *S, llvm::Value *StepV=nullptr)
Increment the profiler's counter for the given statement by StepV.
void EmitOMPTargetEnterDataDirective(const OMPTargetEnterDataDirective &S)
void EmitOMPMaskedTaskLoopSimdDirective(const OMPMaskedTaskLoopSimdDirective &S)
std::pair< bool, RValue > EmitOMPAtomicSimpleUpdateExpr(LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, llvm::AtomicOrdering AO, SourceLocation Loc, const llvm::function_ref< RValue(RValue)> CommonGen)
Emit atomic update code for constructs: X = X BO E or X = E BO E.
VlaSizePair getVLASize(const VariableArrayType *vla)
Returns an LLVM value that corresponds to the size, in non-variably-sized elements,...
void EmitOMPParallelDirective(const OMPParallelDirective &S)
void EmitOMPTaskDirective(const OMPTaskDirective &S)
void EmitOMPMasterTaskLoopSimdDirective(const OMPMasterTaskLoopSimdDirective &S)
void EmitOMPDistributeParallelForDirective(const OMPDistributeParallelForDirective &S)
void EmitOMPAssumeDirective(const OMPAssumeDirective &S)
int ExpectedOMPLoopDepth
Number of nested loop to be consumed by the last surrounding loop-associated directive.
void EmitOMPPrivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S)
ASTContext & getContext() const
void EmitStopPoint(const Stmt *S)
EmitStopPoint - Emit a debug stoppoint if we are emitting debug info.
void EmitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &S)
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
void EmitOMPTargetTeamsGenericLoopDirective(const OMPTargetTeamsGenericLoopDirective &S)
void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit)
EmitStoreOfComplex - Store a complex number into the specified l-value.
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
void EmitAutoVarCleanups(const AutoVarEmission &emission)
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
SmallVector< llvm::CanonicalLoopInfo *, 4 > OMPLoopNestStack
List of recently emitted OMPCanonicalLoops.
void EmitOMPTeamsDistributeParallelForDirective(const OMPTeamsDistributeParallelForDirective &S)
llvm::AtomicRMWInst * emitAtomicRMWInst(llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val, llvm::AtomicOrdering Order=llvm::AtomicOrdering::SequentiallyConsistent, llvm::SyncScope::ID SSID=llvm::SyncScope::System, const AtomicExpr *AE=nullptr)
Emit an atomicrmw instruction, and applying relevant metadata when applicable.
void EmitOMPFuseDirective(const OMPFuseDirective &S)
void EmitOMPTargetTeamsDistributeDirective(const OMPTargetTeamsDistributeDirective &S)
void EmitOMPUseDevicePtrClause(const OMPUseDevicePtrClause &C, OMPPrivateScope &PrivateScope, const llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap)
RValue EmitAnyExpr(const Expr *E, AggValueSlot aggSlot=AggValueSlot::ignored(), bool ignoreResult=false)
EmitAnyExpr - Emit code to compute the specified expression which can have any type.
void EmitStmt(const Stmt *S, ArrayRef< const Attr * > Attrs={})
EmitStmt - Emit the code for the statement.
llvm::DenseMap< const ValueDecl *, FieldDecl * > LambdaCaptureFields
void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S)
llvm::Type * ConvertTypeForMem(QualType T)
void EmitOMPInnerLoop(const OMPExecutableDirective &S, bool RequiresCleanup, const Expr *LoopCond, const Expr *IncExpr, const llvm::function_ref< void(CodeGenFunction &)> BodyGen, const llvm::function_ref< void(CodeGenFunction &)> PostIncGen)
Emit inner loop of the worksharing/simd construct.
void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S)
static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForDirective &S)
void EmitOMPTargetDirective(const OMPTargetDirective &S)
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
llvm::BasicBlock * OMPScanExitBlock
void EmitOMPTeamsDirective(const OMPTeamsDirective &S)
void EmitSimpleOMPExecutableDirective(const OMPExecutableDirective &D)
Emit simple code for OpenMP directives in Simd-only mode.
void EmitOMPErrorDirective(const OMPErrorDirective &S)
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
void EmitOMPParallelMaskedTaskLoopSimdDirective(const OMPParallelMaskedTaskLoopSimdDirective &S)
void EmitOMPTargetTeamsDirective(const OMPTargetTeamsDirective &S)
void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S)
Address GenerateCapturedStmtArgument(const CapturedStmt &S)
bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Emit initial code for lastprivate variables.
static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForSimdDirective &S)
Emit device code for the target teams distribute parallel for simd directive.
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block,...
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S, const OMPExecutableDirective &D)
void EmitOMPSimdDirective(const OMPSimdDirective &S)
RawAddress CreateMemTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
Address EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo=nullptr, TBAAAccessInfo *PointeeTBAAInfo=nullptr)
void EmitOMPParallelGenericLoopDirective(const OMPLoopDirective &S)
void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S)
void EmitOMPTeamsGenericLoopDirective(const OMPTeamsGenericLoopDirective &S)
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
bool EmitOMPLinearClauseInit(const OMPLoopDirective &D)
Emit initial code for linear variables.
static void EmitOMPTargetParallelGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelGenericLoopDirective &S)
Emit device code for the target parallel loop directive.
void EmitOMPUnrollDirective(const OMPUnrollDirective &S)
void EmitOMPStripeDirective(const OMPStripeDirective &S)
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
void EmitOMPSingleDirective(const OMPSingleDirective &S)
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
llvm::function_ref< void(CodeGenFunction &, SourceLocation, const unsigned, const bool)> CodeGenOrderedTy
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
llvm::Value * EmitFromMemory(llvm::Value *Value, QualType Ty)
EmitFromMemory - Change a scalar value from its memory representation to its value representation.
static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S)
Emit device code for the target simd directive.
llvm::Function * GenerateCapturedStmtFunction(const CapturedStmt &S)
Creates the outlined function for a CapturedStmt.
static void EmitOMPTargetParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForDirective &S)
Emit device code for the target parallel for directive.
uint64_t getProfileCount(const Stmt *S)
Get the profiler's count for the given statement.
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant,...
static void EmitOMPTargetTeamsGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsGenericLoopDirective &S)
Emit device code for the target teams loop directive.
void EmitOMPTileDirective(const OMPTileDirective &S)
void EmitDecl(const Decl &D, bool EvaluateConditionDecl=false)
EmitDecl - Emit a declaration.
void EmitOMPAtomicDirective(const OMPAtomicDirective &S)
llvm::BasicBlock * OMPAfterScanBlock
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
ConstantEmission tryEmitAsConstant(const DeclRefExpr *RefExpr)
Try to emit a reference to the given value without producing it as an l-value.
LValue EmitLValue(const Expr *E, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst)
Store of global named registers are always calls to intrinsics.
void EmitOMPParallelMasterTaskLoopDirective(const OMPParallelMasterTaskLoopDirective &S)
void EmitOMPDistributeParallelForSimdDirective(const OMPDistributeParallelForSimdDirective &S)
void EmitOMPSectionDirective(const OMPSectionDirective &S)
void EnsureInsertPoint()
EnsureInsertPoint - Ensure that an insertion point is defined so that emitted IR has a place to go.
void EmitOMPForSimdDirective(const OMPForSimdDirective &S)
llvm::LLVMContext & getLLVMContext()
void emitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty, SourceLocation Loc, SourceLocation AssumptionLoc, llvm::Value *Alignment, llvm::Value *OffsetValue=nullptr)
static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeSimdDirective &S)
Emit device code for the target teams distribute simd directive.
llvm::function_ref< void(CodeGenFunction &, const OMPLoopDirective &, JumpDest)> CodeGenLoopTy
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
bool isTrivialInitializer(const Expr *Init)
Determine whether the given initializer is trivial in the sense that it requires no code to be genera...
void EmitOMPParallelMasterDirective(const OMPParallelMasterDirective &S)
void EmitOMPTaskBasedDirective(const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion, const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen, OMPTaskDataTy &Data)
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
void EmitOMPForDirective(const OMPForDirective &S)
void EmitOMPLinearClauseFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
Emit final code for linear clauses.
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue, bool capturedByInit)
EmitExprAsInit - Emits the code necessary to initialize a location in memory with the given initializ...
void EmitOMPSimdFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
This class organizes the cross-function state that is used while generating LLVM code.
void SetInternalFunctionAttributes(GlobalDecl GD, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
llvm::Module & getModule() const
DiagnosticsEngine & getDiags() const
const LangOptions & getLangOpts() const
CodeGenTypes & getTypes()
const llvm::DataLayout & getDataLayout() const
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
ASTContext & getContext() const
const CodeGenOptions & getCodeGenOpts() const
StringRef getMangledName(GlobalDecl GD)
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
const CGFunctionInfo & arrangeDeviceKernelCallerDeclaration(QualType resultType, const FunctionArgList &args)
A device kernel caller function is an offload device entry point function with a target device depend...
FunctionArgList - Type for representing both the decl and type of parameters to a function.
LValue - This represents an lvalue references.
llvm::Value * getPointer(CodeGenFunction &CGF) const
Address getAddress() const
void setAddress(Address address)
A stack of loop information corresponding to loop nesting levels.
void setVectorizeWidth(unsigned W)
Set the vectorize width for the next loop pushed.
void setParallel(bool Enable=true)
Set the next pushed loop as parallel.
void push(llvm::BasicBlock *Header, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc)
Begin a new structured loop.
void setVectorizeEnable(bool Enable=true)
Set the next pushed loop 'vectorize.enable'.
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
virtual void Enter(CodeGenFunction &CGF)
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
static RValue get(llvm::Value *V)
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
An abstract representation of an aligned address.
llvm::PointerType * getType() const
Return the type of the pointer value.
llvm::Value * getPointer() const
Class intended to support codegen of all kind of the reduction clauses.
LValue getSharedLValue(unsigned N) const
Returns LValue for the reduction item.
void emitAggregateType(CodeGenFunction &CGF, unsigned N)
Emits the code for the variable-modified type, if required.
const VarDecl * getBaseDecl(unsigned N) const
Returns the base declaration of the reduction item.
void emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N)
Emits lvalue for the shared and original reduction item.
void emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr, llvm::function_ref< bool(CodeGenFunction &)> DefaultInit)
Performs initialization of the private copy for the reduction item.
Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Adjusts PrivatedAddr for using instead of the original variable address in normal operations.
Class provides a way to call simple version of codegen for OpenMP region, or an advanced with possibl...
void setAction(PrePostActionTy &Action) const
Complex values, per C99 6.2.5p11.
CompoundStmt - This represents a group of statements like { stmt stmt }.
ConstStmtVisitor - This class implements a simple visitor for Stmt subclasses.
DeclContext * getParent()
getParent - Returns the containing DeclContext.
A reference to a declared variable, function, enum, etc.
static DeclRefExpr * Create(const ASTContext &Context, NestedNameSpecifierLoc QualifierLoc, SourceLocation TemplateKWLoc, ValueDecl *D, bool RefersToEnclosingVariableOrCapture, SourceLocation NameLoc, QualType T, ExprValueKind VK, NamedDecl *FoundD=nullptr, const TemplateArgumentListInfo *TemplateArgs=nullptr, NonOdrUseReason NOUR=NOUR_None)
DeclStmt - Adaptor class for mixing declarations with statements and expressions.
Decl - This represents one declaration (or definition), e.g.
SourceLocation getBodyRBrace() const
getBodyRBrace - Gets the right brace of the body, if a body exists.
virtual bool hasBody() const
Returns true if this Decl represents a declaration for a body of code, such as a function or method d...
SourceLocation getLocation() const
The name of a declaration.
SourceLocation getBeginLoc() const LLVM_READONLY
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
unsigned getCustomDiagID(Level L, const char(&FormatString)[N])
Return an ID for a diagnostic with the specified format string and level.
This represents one expression.
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer,...
Expr * IgnoreParenImpCasts() LLVM_READONLY
Skip past any parentheses and implicit casts which might surround this expression until reaching a fi...
Expr * IgnoreImplicitAsWritten() LLVM_READONLY
Skip past any implicit AST nodes which might surround this expression until reaching a fixed point.
Expr * IgnoreImpCasts() LLVM_READONLY
Skip past any implicit casts which might surround this expression until reaching a fixed point.
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
Represents difference between two FPOptions values.
Represents a member of a struct/union/class.
Represents a function declaration or definition.
static FunctionDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation NLoc, DeclarationName N, QualType T, TypeSourceInfo *TInfo, StorageClass SC, bool UsesFPIntrin=false, bool isInlineSpecified=false, bool hasWrittenPrototype=true, ConstexprSpecKind ConstexprKind=ConstexprSpecKind::Unspecified, const AssociatedConstraint &TrailingRequiresClause={})
GlobalDecl - represents a global declaration.
One of these records is kept for each identifier that is lexed.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
static ImplicitCastExpr * Create(const ASTContext &Context, QualType T, CastKind Kind, Expr *Operand, const CXXCastPath *BasePath, ExprValueKind Cat, FPOptionsOverride FPO)
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
std::vector< llvm::Triple > OMPTargetTriples
Triples of the OpenMP targets that the host code codegen should take into account in order to generat...
Represents a point when we exit a loop.
IdentifierInfo * getIdentifier() const
Get the identifier that names this declaration, if there is one.
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
A C++ nested-name-specifier augmented with source location information.
This represents 'acq_rel' clause in the 'pragma omp atomic|flush' directives.
This represents 'acquire' clause in the 'pragma omp atomic|flush' directives.
This represents clause 'aligned' in the 'pragma omp ...' directives.
This represents 'bind' clause in the 'pragma omp ...' directives.
static OMPClauseWithPreInit * get(OMPClause *C)
This is a basic class for representing single OpenMP clause.
This represents clause 'copyin' in the 'pragma omp ...' directives.
This represents clause 'copyprivate' in the 'pragma omp ...' directives.
This represents implicit clause 'depend' for the 'pragma omp task' directive.
This represents implicit clause 'depobj' for the 'pragma omp depobj' directive.
This represents 'destroy' clause in the 'pragma omp depobj' directive or the 'pragma omp interop' dir...
This represents 'device' clause in the 'pragma omp ...' directive.
This represents 'dist_schedule' clause in the 'pragma omp ...' directive.
This represents the 'doacross' clause for the 'pragma omp ordered' directive.
This represents 'fail' clause in the 'pragma omp atomic' directive.
This represents 'filter' clause in the 'pragma omp ...' directive.
This represents 'final' clause in the 'pragma omp ...' directive.
This represents clause 'firstprivate' in the 'pragma omp ...' directives.
This represents implicit clause 'flush' for the 'pragma omp flush' directive.
Representation of the 'full' clause of the 'pragma omp unroll' directive.
This represents 'grainsize' clause in the 'pragma omp ...' directive.
This represents 'hint' clause in the 'pragma omp ...' directive.
This represents 'if' clause in the 'pragma omp ...' directive.
This represents clause 'in_reduction' in the 'pragma omp task' directives.
This represents clause 'inclusive' in the 'pragma omp scan' directive.
This represents the 'init' clause in 'pragma omp ...' directives.
This represents clause 'lastprivate' in the 'pragma omp ...' directives.
This represents clause 'linear' in the 'pragma omp ...' directives.
This represents the 'message' clause in the 'pragma omp error' and the 'pragma omp parallel' directiv...
Expr * getMessageString() const
Returns message string of the clause.
This represents 'nogroup' clause in the 'pragma omp ...' directive.
This represents 'nowait' clause in the 'pragma omp ...' directive.
This represents 'num_tasks' clause in the 'pragma omp ...' directive.
This represents 'num_teams' clause in the 'pragma omp ...' directive.
This represents 'num_threads' clause in the 'pragma omp ...' directive.
This represents 'order' clause in the 'pragma omp ...' directive.
This represents 'ordered' clause in the 'pragma omp ...' directive.
Representation of the 'partial' clause of the 'pragma omp unroll' directive.
This represents 'priority' clause in the 'pragma omp ...' directive.
This represents clause 'private' in the 'pragma omp ...' directives.
This represents 'proc_bind' clause in the 'pragma omp ...' directive.
This represents clause 'reduction' in the 'pragma omp ...' directives.
This represents 'relaxed' clause in the 'pragma omp atomic' directives.
This represents 'release' clause in the 'pragma omp atomic|flush' directives.
This represents 'simd' clause in the 'pragma omp ...' directive.
This represents 'safelen' clause in the 'pragma omp ...' directive.
This represents 'schedule' clause in the 'pragma omp ...' directive.
This represents 'seq_cst' clause in the 'pragma omp atomic|flush' directives.
This represents the 'severity' clause in the 'pragma omp error' and the 'pragma omp parallel' directi...
OpenMPSeverityClauseKind getSeverityKind() const
Returns kind of the clause.
This represents 'simdlen' clause in the 'pragma omp ...' directive.
This represents clause 'task_reduction' in the 'pragma omp taskgroup' directives.
This represents 'thread_limit' clause in the 'pragma omp ...' directive.
This represents 'untied' clause in the 'pragma omp ...' directive.
This represents 'update' clause in the 'pragma omp atomic' directive.
This represents the 'use' clause in 'pragma omp ...' directives.
This represents clause 'use_device_addr' in the 'pragma omp ...' directives.
This represents clause 'use_device_ptr' in the 'pragma omp ...' directives.
OpaqueValueExpr - An expression referring to an opaque object of a fixed type and value class.
static ParmVarDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation IdLoc, const IdentifierInfo *Id, QualType T, TypeSourceInfo *TInfo, StorageClass S, Expr *DefArg)
PointerType - C99 6.7.5.1 - Pointer Declarators.
Represents an unpacked "presumed" location which can be presented to the user.
const char * getFilename() const
Return the presumed filename of this location.
unsigned getLine() const
Return the presumed line number of this location.
If a crash happens while one of these objects are live, the message is printed out along with the spe...
A (possibly-)qualified type.
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
Represents a struct/union/class.
field_range fields() const
field_iterator field_begin() const
Base for LValueReferenceType and RValueReferenceType.
Scope - A scope is a transient data structure that is used while parsing the program.
Encodes a location in the source.
A trivial tuple used to represent a source range.
SourceLocation getEnd() const
SourceLocation getBegin() const
Stmt - This represents one statement.
SourceRange getSourceRange() const LLVM_READONLY
SourceLocation tokens are not useful in isolation - they are low level value objects created/interpre...
Stmt * IgnoreContainers(bool IgnoreCaptured=false)
Skip no-op (attributed, compound) container stmts and skip captured stmt at the top,...
SourceLocation getBeginLoc() const LLVM_READONLY
bool isPointerType() const
const T * castAs() const
Member-template castAs<specific type>.
bool isReferenceType() const
bool isLValueReferenceType() const
bool isAnyComplexType() const
bool hasSignedIntegerRepresentation() const
Determine whether this type has an signed integer representation of some sort, e.g....
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type.
static UnaryOperator * Create(const ASTContext &C, Expr *input, Opcode opc, QualType type, ExprValueKind VK, ExprObjectKind OK, SourceLocation l, bool CanOverflow, FPOptionsOverride FPFeatures)
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
Represents a variable declaration or definition.
TLSKind getTLSKind() const
VarDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
@ CInit
C-style initialization with assignment.
bool hasGlobalStorage() const
Returns true for all variables that do not have local storage.
bool isStaticLocal() const
Returns true if a variable with function scope is a static local variable.
const Expr * getInit() const
bool hasLocalStorage() const
Returns true if a variable with function scope is a non-static local variable.
@ TLS_None
Not a TLS variable.
Represents a C array with a specified size that is not an integer-constant-expression.
Expr * getSizeExpr() const
@ Type
The l-value was considered opaque, so the alignment was determined from a type.
@ Decl
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
bool Inc(InterpState &S, CodePtr OpPC, bool CanOverflow)
1) Pops a pointer from the stack 2) Load the value from the pointer 3) Writes the value increased by ...
The JSON file list parser is used to communicate input to InstallAPI.
bool isOpenMPWorksharingDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a worksharing directive.
CanQual< Type > CanQualType
Represents a canonical, potentially-qualified type.
bool needsTaskBasedThreadLimit(OpenMPDirectiveKind DKind)
Checks if the specified target directive, combined or not, needs task based thread_limit.
@ Ctor_Complete
Complete object ctor.
bool isa(CodeGen::Address addr)
if(T->getSizeExpr()) TRY_TO(TraverseStmt(const_cast< Expr * >(T -> getSizeExpr())))
@ OK_Ordinary
An ordinary object is located at an address in memory.
bool isOpenMPDistributeDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a distribute directive.
@ Tile
'tile' clause, allowed on 'loop' and Combined constructs.
OpenMPScheduleClauseModifier
OpenMP modifiers for 'schedule' clause.
@ OMPC_SCHEDULE_MODIFIER_unknown
bool isOpenMPParallelDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a parallel-kind directive.
OpenMPDistScheduleClauseKind
OpenMP attributes for 'dist_schedule' clause.
@ OMPC_DIST_SCHEDULE_unknown
bool isOpenMPTaskingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of tasking directives - task, taskloop,...
bool isOpenMPTargetExecutionDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target code offload directive.
@ Result
The result type of a method or function.
bool isOpenMPTeamsDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a teams-kind directive.
bool isOpenMPGenericLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive constitutes a 'loop' directive in the outermost nest.
OpenMPBindClauseKind
OpenMP bindings for the 'bind' clause.
const FunctionProtoType * T
OpenMPDependClauseKind
OpenMP attributes for 'depend' clause.
bool IsXLHSInRHSPart
True if UE has the first form and false if the second.
bool IsPostfixUpdate
True if original value of 'x' must be stored in 'v', not an updated one.
@ Dtor_Complete
Complete object dtor.
OpenMPSeverityClauseKind
OpenMP attributes for 'severity' clause.
bool isOpenMPLoopBoundSharingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of the composite or combined directives that need loop ...
llvm::omp::Directive OpenMPDirectiveKind
OpenMP directives.
bool isOpenMPSimdDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a simd directive.
@ VK_PRValue
A pr-value expression (in the C++11 taxonomy) produces a temporary value.
@ VK_LValue
An l-value expression is a reference to an object with independent storage.
void getOpenMPCaptureRegions(llvm::SmallVectorImpl< OpenMPDirectiveKind > &CaptureRegions, OpenMPDirectiveKind DKind)
Return the captured regions of an OpenMP directive.
OpenMPNumThreadsClauseModifier
@ OMPC_NUMTHREADS_unknown
bool IsFailOnly
True if 'v' is updated only when the condition is false (compare capture only).
U cast(CodeGen::Address addr)
llvm::omp::Clause OpenMPClauseKind
OpenMP clauses.
@ ThreadPrivateVar
Parameter for Thread private variable.
@ Other
Other implicit parameter.
OpenMPScheduleClauseKind
OpenMP attributes for 'schedule' clause.
bool isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a taskloop directive.
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
llvm::BasicBlock * getBlock() const
static Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc)
Returns address of the threadprivate variable for the current thread.
llvm::OpenMPIRBuilder::InsertPointTy InsertPointTy
static void EmitOMPOutlinedRegionBody(CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Twine RegionName)
Emit the body of an OMP region that will be outlined in OpenMPIRBuilder::finalize().
static Address getAddressOfLocalVariable(CodeGenFunction &CGF, const VarDecl *VD)
Gets the OpenMP-specific address of the local variable /p VD.
static void EmitCaptureStmt(CodeGenFunction &CGF, InsertPointTy CodeGenIP, llvm::BasicBlock &FiniBB, llvm::Function *Fn, ArrayRef< llvm::Value * > Args)
static std::string getNameWithSeparators(ArrayRef< StringRef > Parts, StringRef FirstSeparator=".", StringRef Separator=".")
Get the platform-specific name separator.
static void FinalizeOMPRegion(CodeGenFunction &CGF, InsertPointTy IP)
Emit the Finalization for an OMP region.
static void EmitOMPInlinedRegionBody(CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Twine RegionName)
Emit the body of an OMP region.
unsigned NumberOfTargetItems
Address BasePointersArray
llvm::PointerType * VoidPtrTy
llvm::IntegerType * Int64Ty
llvm::IntegerType * SizeTy
SmallVector< const Expr *, 4 > DepExprs
EvalResult is a struct with detailed info about an evaluated expression.
Extra information about a function prototype.
Scheduling data for loop-based OpenMP directives.
OpenMPScheduleClauseModifier M2
OpenMPScheduleClauseModifier M1
OpenMPScheduleClauseKind Schedule