Thanks to visit codestin.com
Credit goes to clang.llvm.org

clang 22.0.0git
CGStmtOpenMP.cpp
Go to the documentation of this file.
1//===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit OpenMP nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCleanup.h"
14#include "CGDebugInfo.h"
15#include "CGOpenMPRuntime.h"
16#include "CodeGenFunction.h"
17#include "CodeGenModule.h"
18#include "CodeGenPGO.h"
19#include "TargetInfo.h"
21#include "clang/AST/Attr.h"
24#include "clang/AST/Stmt.h"
30#include "llvm/ADT/SmallSet.h"
31#include "llvm/BinaryFormat/Dwarf.h"
32#include "llvm/Frontend/OpenMP/OMPConstants.h"
33#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
34#include "llvm/IR/Constants.h"
35#include "llvm/IR/DebugInfoMetadata.h"
36#include "llvm/IR/Instructions.h"
37#include "llvm/IR/IntrinsicInst.h"
38#include "llvm/IR/Metadata.h"
39#include "llvm/Support/AtomicOrdering.h"
40#include "llvm/Support/Debug.h"
41#include <optional>
42using namespace clang;
43using namespace CodeGen;
44using namespace llvm::omp;
45
46#define TTL_CODEGEN_TYPE "target-teams-loop-codegen"
47
48static const VarDecl *getBaseDecl(const Expr *Ref);
51
52namespace {
53/// Lexical scope for OpenMP executable constructs, that handles correct codegen
54/// for captured expressions.
55class OMPLexicalScope : public CodeGenFunction::LexicalScope {
56 void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
57 for (const auto *C : S.clauses()) {
58 if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
59 if (const auto *PreInit =
60 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
61 for (const auto *I : PreInit->decls()) {
62 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
64 } else {
65 CodeGenFunction::AutoVarEmission Emission =
67 CGF.EmitAutoVarCleanups(Emission);
68 }
69 }
70 }
71 }
72 }
73 }
74 CodeGenFunction::OMPPrivateScope InlinedShareds;
75
76 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
77 return CGF.LambdaCaptureFields.lookup(VD) ||
78 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
79 (isa_and_nonnull<BlockDecl>(CGF.CurCodeDecl) &&
80 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
81 }
82
83public:
84 OMPLexicalScope(
85 CodeGenFunction &CGF, const OMPExecutableDirective &S,
86 const std::optional<OpenMPDirectiveKind> CapturedRegion = std::nullopt,
87 const bool EmitPreInitStmt = true)
88 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
89 InlinedShareds(CGF) {
90 if (EmitPreInitStmt)
91 emitPreInitStmt(CGF, S);
92 if (!CapturedRegion)
93 return;
94 assert(S.hasAssociatedStmt() &&
95 "Expected associated statement for inlined directive.");
96 const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);
97 for (const auto &C : CS->captures()) {
98 if (C.capturesVariable() || C.capturesVariableByCopy()) {
99 auto *VD = C.getCapturedVar();
100 assert(VD == VD->getCanonicalDecl() &&
101 "Canonical decl must be captured.");
102 DeclRefExpr DRE(
103 CGF.getContext(), const_cast<VarDecl *>(VD),
104 isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo &&
105 InlinedShareds.isGlobalVarCaptured(VD)),
106 VD->getType().getNonReferenceType(), VK_LValue, C.getLocation());
107 InlinedShareds.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress());
108 }
109 }
110 (void)InlinedShareds.Privatize();
111 }
112};
113
114/// Lexical scope for OpenMP parallel construct, that handles correct codegen
115/// for captured expressions.
116class OMPParallelScope final : public OMPLexicalScope {
117 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
119 return !(isOpenMPTargetExecutionDirective(EKind) ||
122 }
123
124public:
125 OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
126 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/std::nullopt,
127 EmitPreInitStmt(S)) {}
128};
129
130/// Lexical scope for OpenMP teams construct, that handles correct codegen
131/// for captured expressions.
132class OMPTeamsScope final : public OMPLexicalScope {
133 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
135 return !isOpenMPTargetExecutionDirective(EKind) &&
137 }
138
139public:
140 OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
141 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/std::nullopt,
142 EmitPreInitStmt(S)) {}
143};
144
145/// Private scope for OpenMP loop-based directives, that supports capturing
146/// of used expression from loop statement.
147class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
148 void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopBasedDirective &S) {
149 const Stmt *PreInits;
150 CodeGenFunction::OMPMapVars PreCondVars;
151 if (auto *LD = dyn_cast<OMPLoopDirective>(&S)) {
152 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
153 for (const auto *E : LD->counters()) {
154 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
155 EmittedAsPrivate.insert(VD->getCanonicalDecl());
156 (void)PreCondVars.setVarAddr(
157 CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType()));
158 }
159 // Mark private vars as undefs.
160 for (const auto *C : LD->getClausesOfKind<OMPPrivateClause>()) {
161 for (const Expr *IRef : C->varlist()) {
162 const auto *OrigVD =
163 cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
164 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
165 QualType OrigVDTy = OrigVD->getType().getNonReferenceType();
166 (void)PreCondVars.setVarAddr(
167 CGF, OrigVD,
168 Address(llvm::UndefValue::get(CGF.ConvertTypeForMem(
169 CGF.getContext().getPointerType(OrigVDTy))),
170 CGF.ConvertTypeForMem(OrigVDTy),
171 CGF.getContext().getDeclAlign(OrigVD)));
172 }
173 }
174 }
175 (void)PreCondVars.apply(CGF);
176 // Emit init, __range and __end variables for C++ range loops.
177 (void)OMPLoopBasedDirective::doForAllLoops(
178 LD->getInnermostCapturedStmt()->getCapturedStmt(),
179 /*TryImperfectlyNestedLoops=*/true, LD->getLoopsNumber(),
180 [&CGF](unsigned Cnt, const Stmt *CurStmt) {
181 if (const auto *CXXFor = dyn_cast<CXXForRangeStmt>(CurStmt)) {
182 if (const Stmt *Init = CXXFor->getInit())
183 CGF.EmitStmt(Init);
184 CGF.EmitStmt(CXXFor->getRangeStmt());
185 CGF.EmitStmt(CXXFor->getEndStmt());
186 }
187 return false;
188 });
189 PreInits = LD->getPreInits();
190 } else if (const auto *Tile = dyn_cast<OMPTileDirective>(&S)) {
191 PreInits = Tile->getPreInits();
192 } else if (const auto *Stripe = dyn_cast<OMPStripeDirective>(&S)) {
193 PreInits = Stripe->getPreInits();
194 } else if (const auto *Unroll = dyn_cast<OMPUnrollDirective>(&S)) {
195 PreInits = Unroll->getPreInits();
196 } else if (const auto *Reverse = dyn_cast<OMPReverseDirective>(&S)) {
197 PreInits = Reverse->getPreInits();
198 } else if (const auto *Interchange =
199 dyn_cast<OMPInterchangeDirective>(&S)) {
200 PreInits = Interchange->getPreInits();
201 } else {
202 llvm_unreachable("Unknown loop-based directive kind.");
203 }
204 doEmitPreinits(PreInits);
205 PreCondVars.restore(CGF);
206 }
207
208 void
209 emitPreInitStmt(CodeGenFunction &CGF,
211 const Stmt *PreInits;
212 if (const auto *Fuse = dyn_cast<OMPFuseDirective>(&S)) {
213 PreInits = Fuse->getPreInits();
214 } else {
215 llvm_unreachable(
216 "Unknown canonical loop sequence transform directive kind.");
217 }
218 doEmitPreinits(PreInits);
219 }
220
221 void doEmitPreinits(const Stmt *PreInits) {
222 if (PreInits) {
223 // CompoundStmts and DeclStmts are used as lists of PreInit statements and
224 // declarations. Since declarations must be visible in the the following
225 // that they initialize, unpack the CompoundStmt they are nested in.
226 SmallVector<const Stmt *> PreInitStmts;
227 if (auto *PreInitCompound = dyn_cast<CompoundStmt>(PreInits))
228 llvm::append_range(PreInitStmts, PreInitCompound->body());
229 else
230 PreInitStmts.push_back(PreInits);
231
232 for (const Stmt *S : PreInitStmts) {
233 // EmitStmt skips any OMPCapturedExprDecls, but needs to be emitted
234 // here.
235 if (auto *PreInitDecl = dyn_cast<DeclStmt>(S)) {
236 for (Decl *I : PreInitDecl->decls())
237 CGF.EmitVarDecl(cast<VarDecl>(*I));
238 continue;
239 }
240 CGF.EmitStmt(S);
241 }
242 }
243 }
244
245public:
246 OMPLoopScope(CodeGenFunction &CGF, const OMPLoopBasedDirective &S)
247 : CodeGenFunction::RunCleanupsScope(CGF) {
248 emitPreInitStmt(CGF, S);
249 }
250 OMPLoopScope(CodeGenFunction &CGF,
252 : CodeGenFunction::RunCleanupsScope(CGF) {
253 emitPreInitStmt(CGF, S);
254 }
255};
256
257class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope {
258 CodeGenFunction::OMPPrivateScope InlinedShareds;
259
260 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
261 return CGF.LambdaCaptureFields.lookup(VD) ||
262 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
263 (isa_and_nonnull<BlockDecl>(CGF.CurCodeDecl) &&
264 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
265 }
266
267public:
268 OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
269 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
270 InlinedShareds(CGF) {
271 for (const auto *C : S.clauses()) {
272 if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
273 if (const auto *PreInit =
274 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
275 for (const auto *I : PreInit->decls()) {
276 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
277 CGF.EmitVarDecl(cast<VarDecl>(*I));
278 } else {
279 CodeGenFunction::AutoVarEmission Emission =
280 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
281 CGF.EmitAutoVarCleanups(Emission);
282 }
283 }
284 }
285 } else if (const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(C)) {
286 for (const Expr *E : UDP->varlist()) {
287 const Decl *D = cast<DeclRefExpr>(E)->getDecl();
288 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
289 CGF.EmitVarDecl(*OED);
290 }
291 } else if (const auto *UDP = dyn_cast<OMPUseDeviceAddrClause>(C)) {
292 for (const Expr *E : UDP->varlist()) {
293 const Decl *D = getBaseDecl(E);
294 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
295 CGF.EmitVarDecl(*OED);
296 }
297 }
298 }
300 CGF.EmitOMPPrivateClause(S, InlinedShareds);
301 if (const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
302 if (const Expr *E = TG->getReductionRef())
303 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
304 }
305 // Temp copy arrays for inscan reductions should not be emitted as they are
306 // not used in simd only mode.
307 llvm::DenseSet<CanonicalDeclPtr<const Decl>> CopyArrayTemps;
308 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
309 if (C->getModifier() != OMPC_REDUCTION_inscan)
310 continue;
311 for (const Expr *E : C->copy_array_temps())
312 CopyArrayTemps.insert(cast<DeclRefExpr>(E)->getDecl());
313 }
314 const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
315 while (CS) {
316 for (auto &C : CS->captures()) {
317 if (C.capturesVariable() || C.capturesVariableByCopy()) {
318 auto *VD = C.getCapturedVar();
319 if (CopyArrayTemps.contains(VD))
320 continue;
321 assert(VD == VD->getCanonicalDecl() &&
322 "Canonical decl must be captured.");
323 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
324 isCapturedVar(CGF, VD) ||
325 (CGF.CapturedStmtInfo &&
326 InlinedShareds.isGlobalVarCaptured(VD)),
328 C.getLocation());
329 InlinedShareds.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress());
330 }
331 }
332 CS = dyn_cast<CapturedStmt>(CS->getCapturedStmt());
333 }
334 (void)InlinedShareds.Privatize();
335 }
336};
337
338} // namespace
339
340// The loop directive with a bind clause will be mapped to a different
341// directive with corresponding semantics.
344 OpenMPDirectiveKind Kind = S.getDirectiveKind();
345 if (Kind != OMPD_loop)
346 return Kind;
347
349 if (const auto *C = S.getSingleClause<OMPBindClause>())
350 BindKind = C->getBindKind();
351
352 switch (BindKind) {
353 case OMPC_BIND_parallel:
354 return OMPD_for;
355 case OMPC_BIND_teams:
356 return OMPD_distribute;
357 case OMPC_BIND_thread:
358 return OMPD_simd;
359 default:
360 return OMPD_loop;
361 }
362}
363
365 const OMPExecutableDirective &S,
366 const RegionCodeGenTy &CodeGen);
367
369 if (const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
370 if (const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
371 OrigVD = OrigVD->getCanonicalDecl();
372 bool IsCaptured =
373 LambdaCaptureFields.lookup(OrigVD) ||
374 (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) ||
375 (isa_and_nonnull<BlockDecl>(CurCodeDecl));
376 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), IsCaptured,
377 OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());
378 return EmitLValue(&DRE);
379 }
380 }
381 return EmitLValue(E);
382}
383
386 llvm::Value *Size = nullptr;
387 auto SizeInChars = C.getTypeSizeInChars(Ty);
388 if (SizeInChars.isZero()) {
389 // getTypeSizeInChars() returns 0 for a VLA.
390 while (const VariableArrayType *VAT = C.getAsVariableArrayType(Ty)) {
391 VlaSizePair VlaSize = getVLASize(VAT);
392 Ty = VlaSize.Type;
393 Size =
394 Size ? Builder.CreateNUWMul(Size, VlaSize.NumElts) : VlaSize.NumElts;
395 }
396 SizeInChars = C.getTypeSizeInChars(Ty);
397 if (SizeInChars.isZero())
398 return llvm::ConstantInt::get(SizeTy, /*V=*/0);
399 return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
400 }
401 return CGM.getSize(SizeInChars);
402}
403
405 const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
406 const RecordDecl *RD = S.getCapturedRecordDecl();
407 auto CurField = RD->field_begin();
408 auto CurCap = S.captures().begin();
410 E = S.capture_init_end();
411 I != E; ++I, ++CurField, ++CurCap) {
412 if (CurField->hasCapturedVLAType()) {
413 const VariableArrayType *VAT = CurField->getCapturedVLAType();
414 llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()];
415 CapturedVars.push_back(Val);
416 } else if (CurCap->capturesThis()) {
417 CapturedVars.push_back(CXXThisValue);
418 } else if (CurCap->capturesVariableByCopy()) {
419 llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation());
420
421 // If the field is not a pointer, we need to save the actual value
422 // and load it as a void pointer.
423 if (!CurField->getType()->isAnyPointerType()) {
424 ASTContext &Ctx = getContext();
425 Address DstAddr = CreateMemTemp(
426 Ctx.getUIntPtrType(),
427 Twine(CurCap->getCapturedVar()->getName(), ".casted"));
428 LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());
429
430 llvm::Value *SrcAddrVal = EmitScalarConversion(
431 DstAddr.emitRawPointer(*this),
433 Ctx.getPointerType(CurField->getType()), CurCap->getLocation());
434 LValue SrcLV =
435 MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
436
437 // Store the value using the source type pointer.
439
440 // Load the value using the destination type pointer.
441 CV = EmitLoadOfScalar(DstLV, CurCap->getLocation());
442 }
443 CapturedVars.push_back(CV);
444 } else {
445 assert(CurCap->capturesVariable() && "Expected capture by reference.");
446 CapturedVars.push_back(EmitLValue(*I).getAddress().emitRawPointer(*this));
447 }
448 }
449}
450
452 QualType DstType, StringRef Name,
453 LValue AddrLV) {
454 ASTContext &Ctx = CGF.getContext();
455
456 llvm::Value *CastedPtr = CGF.EmitScalarConversion(
457 AddrLV.getAddress().emitRawPointer(CGF), Ctx.getUIntPtrType(),
458 Ctx.getPointerType(DstType), Loc);
459 // FIXME: should the pointee type (DstType) be passed?
460 Address TmpAddr =
461 CGF.MakeNaturalAlignAddrLValue(CastedPtr, DstType).getAddress();
462 return TmpAddr;
463}
464
466 if (T->isLValueReferenceType())
467 return C.getLValueReferenceType(
468 getCanonicalParamType(C, T.getNonReferenceType()),
469 /*SpelledAsLValue=*/false);
470 if (T->isPointerType())
471 return C.getPointerType(getCanonicalParamType(C, T->getPointeeType()));
472 if (const ArrayType *A = T->getAsArrayTypeUnsafe()) {
473 if (const auto *VLA = dyn_cast<VariableArrayType>(A))
474 return getCanonicalParamType(C, VLA->getElementType());
475 if (!A->isVariablyModifiedType())
476 return C.getCanonicalType(T);
477 }
478 return C.getCanonicalParamType(T);
479}
480
481namespace {
482/// Contains required data for proper outlined function codegen.
483struct FunctionOptions {
484 /// Captured statement for which the function is generated.
485 const CapturedStmt *S = nullptr;
486 /// true if cast to/from UIntPtr is required for variables captured by
487 /// value.
488 const bool UIntPtrCastRequired = true;
489 /// true if only casted arguments must be registered as local args or VLA
490 /// sizes.
491 const bool RegisterCastedArgsOnly = false;
492 /// Name of the generated function.
493 const StringRef FunctionName;
494 /// Location of the non-debug version of the outlined function.
495 SourceLocation Loc;
496 const bool IsDeviceKernel = false;
497 explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,
498 bool RegisterCastedArgsOnly, StringRef FunctionName,
499 SourceLocation Loc, bool IsDeviceKernel)
500 : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
501 RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
502 FunctionName(FunctionName), Loc(Loc), IsDeviceKernel(IsDeviceKernel) {}
503};
504} // namespace
505
506static llvm::Function *emitOutlinedFunctionPrologue(
508 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
509 &LocalAddrs,
510 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
511 &VLASizes,
512 llvm::Value *&CXXThisValue, const FunctionOptions &FO) {
513 const CapturedDecl *CD = FO.S->getCapturedDecl();
514 const RecordDecl *RD = FO.S->getCapturedRecordDecl();
515 assert(CD->hasBody() && "missing CapturedDecl body");
516
517 CXXThisValue = nullptr;
518 // Build the argument list.
519 CodeGenModule &CGM = CGF.CGM;
520 ASTContext &Ctx = CGM.getContext();
521 FunctionArgList TargetArgs;
522 Args.append(CD->param_begin(),
523 std::next(CD->param_begin(), CD->getContextParamPosition()));
524 TargetArgs.append(
525 CD->param_begin(),
526 std::next(CD->param_begin(), CD->getContextParamPosition()));
527 auto I = FO.S->captures().begin();
528 FunctionDecl *DebugFunctionDecl = nullptr;
529 if (!FO.UIntPtrCastRequired) {
531 QualType FunctionTy = Ctx.getFunctionType(Ctx.VoidTy, {}, EPI);
532 DebugFunctionDecl = FunctionDecl::Create(
533 Ctx, Ctx.getTranslationUnitDecl(), FO.S->getBeginLoc(),
534 SourceLocation(), DeclarationName(), FunctionTy,
535 Ctx.getTrivialTypeSourceInfo(FunctionTy), SC_Static,
536 /*UsesFPIntrin=*/false, /*isInlineSpecified=*/false,
537 /*hasWrittenPrototype=*/false);
538 }
539 for (const FieldDecl *FD : RD->fields()) {
540 QualType ArgType = FD->getType();
541 IdentifierInfo *II = nullptr;
542 VarDecl *CapVar = nullptr;
543
544 // If this is a capture by copy and the type is not a pointer, the outlined
545 // function argument type should be uintptr and the value properly casted to
546 // uintptr. This is necessary given that the runtime library is only able to
547 // deal with pointers. We can pass in the same way the VLA type sizes to the
548 // outlined function.
549 if (FO.UIntPtrCastRequired &&
550 ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
551 I->capturesVariableArrayType()))
552 ArgType = Ctx.getUIntPtrType();
553
554 if (I->capturesVariable() || I->capturesVariableByCopy()) {
555 CapVar = I->getCapturedVar();
556 II = CapVar->getIdentifier();
557 } else if (I->capturesThis()) {
558 II = &Ctx.Idents.get("this");
559 } else {
560 assert(I->capturesVariableArrayType());
561 II = &Ctx.Idents.get("vla");
562 }
563 if (ArgType->isVariablyModifiedType())
564 ArgType = getCanonicalParamType(Ctx, ArgType);
565 VarDecl *Arg;
566 if (CapVar && (CapVar->getTLSKind() != clang::VarDecl::TLS_None)) {
567 Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
568 II, ArgType,
570 } else if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
572 Ctx, DebugFunctionDecl,
573 CapVar ? CapVar->getBeginLoc() : FD->getBeginLoc(),
574 CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType,
575 /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr);
576 } else {
577 Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
578 II, ArgType, ImplicitParamKind::Other);
579 }
580 Args.emplace_back(Arg);
581 // Do not cast arguments if we emit function with non-original types.
582 TargetArgs.emplace_back(
583 FO.UIntPtrCastRequired
584 ? Arg
585 : CGM.getOpenMPRuntime().translateParameter(FD, Arg));
586 ++I;
587 }
588 Args.append(std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
589 CD->param_end());
590 TargetArgs.append(
591 std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
592 CD->param_end());
593
594 // Create the function declaration.
595 const CGFunctionInfo &FuncInfo =
596 FO.IsDeviceKernel
598 TargetArgs)
600 TargetArgs);
601 llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
602
603 auto *F =
604 llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
605 FO.FunctionName, &CGM.getModule());
606 CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
607 if (CD->isNothrow())
608 F->setDoesNotThrow();
609 F->setDoesNotRecurse();
610
611 // Always inline the outlined function if optimizations are enabled.
612 if (CGM.getCodeGenOpts().OptimizationLevel != 0) {
613 F->removeFnAttr(llvm::Attribute::NoInline);
614 F->addFnAttr(llvm::Attribute::AlwaysInline);
615 }
616
617 // Generate the function.
618 CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs,
619 FO.UIntPtrCastRequired ? FO.Loc : FO.S->getBeginLoc(),
620 FO.UIntPtrCastRequired ? FO.Loc
621 : CD->getBody()->getBeginLoc());
622 unsigned Cnt = CD->getContextParamPosition();
623 I = FO.S->captures().begin();
624 for (const FieldDecl *FD : RD->fields()) {
625 // Do not map arguments if we emit function with non-original types.
626 Address LocalAddr(Address::invalid());
627 if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
628 LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt],
629 TargetArgs[Cnt]);
630 } else {
631 LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]);
632 }
633 // If we are capturing a pointer by copy we don't need to do anything, just
634 // use the value that we get from the arguments.
635 if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
636 const VarDecl *CurVD = I->getCapturedVar();
637 if (!FO.RegisterCastedArgsOnly)
638 LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
639 ++Cnt;
640 ++I;
641 continue;
642 }
643
644 LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(),
646 if (FD->hasCapturedVLAType()) {
647 if (FO.UIntPtrCastRequired) {
648 ArgLVal = CGF.MakeAddrLValue(
649 castValueFromUintptr(CGF, I->getLocation(), FD->getType(),
650 Args[Cnt]->getName(), ArgLVal),
652 }
653 llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
654 const VariableArrayType *VAT = FD->getCapturedVLAType();
655 VLASizes.try_emplace(Args[Cnt], VAT->getSizeExpr(), ExprArg);
656 } else if (I->capturesVariable()) {
657 const VarDecl *Var = I->getCapturedVar();
658 QualType VarTy = Var->getType();
659 Address ArgAddr = ArgLVal.getAddress();
660 if (ArgLVal.getType()->isLValueReferenceType()) {
661 ArgAddr = CGF.EmitLoadOfReference(ArgLVal);
662 } else if (!VarTy->isVariablyModifiedType() || !VarTy->isPointerType()) {
663 assert(ArgLVal.getType()->isPointerType());
664 ArgAddr = CGF.EmitLoadOfPointer(
665 ArgAddr, ArgLVal.getType()->castAs<PointerType>());
666 }
667 if (!FO.RegisterCastedArgsOnly) {
668 LocalAddrs.insert(
669 {Args[Cnt], {Var, ArgAddr.withAlignment(Ctx.getDeclAlign(Var))}});
670 }
671 } else if (I->capturesVariableByCopy()) {
672 assert(!FD->getType()->isAnyPointerType() &&
673 "Not expecting a captured pointer.");
674 const VarDecl *Var = I->getCapturedVar();
675 LocalAddrs.insert({Args[Cnt],
676 {Var, FO.UIntPtrCastRequired
678 CGF, I->getLocation(), FD->getType(),
679 Args[Cnt]->getName(), ArgLVal)
680 : ArgLVal.getAddress()}});
681 } else {
682 // If 'this' is captured, load it into CXXThisValue.
683 assert(I->capturesThis());
684 CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
685 LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress()}});
686 }
687 ++Cnt;
688 ++I;
689 }
690
691 return F;
692}
693
695 const CapturedStmt &S, const OMPExecutableDirective &D) {
696 SourceLocation Loc = D.getBeginLoc();
697 assert(
699 "CapturedStmtInfo should be set when generating the captured function");
700 const CapturedDecl *CD = S.getCapturedDecl();
701 // Build the argument list.
702 bool NeedWrapperFunction =
703 getDebugInfo() && CGM.getCodeGenOpts().hasReducedDebugInfo();
704 FunctionArgList Args, WrapperArgs;
705 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs,
706 WrapperLocalAddrs;
707 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes,
708 WrapperVLASizes;
709 SmallString<256> Buffer;
710 llvm::raw_svector_ostream Out(Buffer);
711 Out << CapturedStmtInfo->getHelperName();
713 bool IsDeviceKernel = CGM.getOpenMPRuntime().isGPU() &&
715 D.getCapturedStmt(OMPD_target) == &S;
716 CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
717 llvm::Function *WrapperF = nullptr;
718 if (NeedWrapperFunction) {
719 // Emit the final kernel early to allow attributes to be added by the
720 // OpenMPI-IR-Builder.
721 FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,
722 /*RegisterCastedArgsOnly=*/true,
723 CapturedStmtInfo->getHelperName(), Loc,
724 IsDeviceKernel);
726 WrapperF =
727 emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes,
728 WrapperCGF.CXXThisValue, WrapperFO);
729 Out << "_debug__";
730 }
731 FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false,
732 Out.str(), Loc, !NeedWrapperFunction && IsDeviceKernel);
733 llvm::Function *F = emitOutlinedFunctionPrologue(
734 *this, WrapperArgs, WrapperLocalAddrs, WrapperVLASizes, CXXThisValue, FO);
735 CodeGenFunction::OMPPrivateScope LocalScope(*this);
736 for (const auto &LocalAddrPair : WrapperLocalAddrs) {
737 if (LocalAddrPair.second.first) {
738 LocalScope.addPrivate(LocalAddrPair.second.first,
739 LocalAddrPair.second.second);
740 }
741 }
742 (void)LocalScope.Privatize();
743 for (const auto &VLASizePair : WrapperVLASizes)
744 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
745 PGO->assignRegionCounters(GlobalDecl(CD), F);
746 CapturedStmtInfo->EmitBody(*this, CD->getBody());
747 LocalScope.ForceCleanup();
749 if (!NeedWrapperFunction)
750 return F;
751
752 // Reverse the order.
753 WrapperF->removeFromParent();
754 F->getParent()->getFunctionList().insertAfter(F->getIterator(), WrapperF);
755
757 auto *PI = F->arg_begin();
758 for (const auto *Arg : Args) {
759 llvm::Value *CallArg;
760 auto I = LocalAddrs.find(Arg);
761 if (I != LocalAddrs.end()) {
762 LValue LV = WrapperCGF.MakeAddrLValue(
763 I->second.second,
764 I->second.first ? I->second.first->getType() : Arg->getType(),
766 if (LV.getType()->isAnyComplexType())
767 LV.setAddress(LV.getAddress().withElementType(PI->getType()));
768 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
769 } else {
770 auto EI = VLASizes.find(Arg);
771 if (EI != VLASizes.end()) {
772 CallArg = EI->second.second;
773 } else {
774 LValue LV =
775 WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
777 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
778 }
779 }
780 CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType()));
781 ++PI;
782 }
783 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, Loc, F, CallArgs);
784 WrapperCGF.FinishFunction();
785 return WrapperF;
786}
787
788//===----------------------------------------------------------------------===//
789// OpenMP Directive Emission
790//===----------------------------------------------------------------------===//
792 Address DestAddr, Address SrcAddr, QualType OriginalType,
793 const llvm::function_ref<void(Address, Address)> CopyGen) {
794 // Perform element-by-element initialization.
795 QualType ElementTy;
796
797 // Drill down to the base element type on both arrays.
798 const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe();
799 llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
800 SrcAddr = SrcAddr.withElementType(DestAddr.getElementType());
801
802 llvm::Value *SrcBegin = SrcAddr.emitRawPointer(*this);
803 llvm::Value *DestBegin = DestAddr.emitRawPointer(*this);
804 // Cast from pointer to array type to pointer to single element.
805 llvm::Value *DestEnd = Builder.CreateInBoundsGEP(DestAddr.getElementType(),
806 DestBegin, NumElements);
807
808 // The basic structure here is a while-do loop.
809 llvm::BasicBlock *BodyBB = createBasicBlock("omp.arraycpy.body");
810 llvm::BasicBlock *DoneBB = createBasicBlock("omp.arraycpy.done");
811 llvm::Value *IsEmpty =
812 Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
813 Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
814
815 // Enter the loop body, making that address the current address.
816 llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();
817 EmitBlock(BodyBB);
818
819 CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
820
821 llvm::PHINode *SrcElementPHI =
822 Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
823 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
824 Address SrcElementCurrent =
825 Address(SrcElementPHI, SrcAddr.getElementType(),
826 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
827
828 llvm::PHINode *DestElementPHI = Builder.CreatePHI(
829 DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
830 DestElementPHI->addIncoming(DestBegin, EntryBB);
831 Address DestElementCurrent =
832 Address(DestElementPHI, DestAddr.getElementType(),
833 DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
834
835 // Emit copy.
836 CopyGen(DestElementCurrent, SrcElementCurrent);
837
838 // Shift the address forward by one element.
839 llvm::Value *DestElementNext =
840 Builder.CreateConstGEP1_32(DestAddr.getElementType(), DestElementPHI,
841 /*Idx0=*/1, "omp.arraycpy.dest.element");
842 llvm::Value *SrcElementNext =
843 Builder.CreateConstGEP1_32(SrcAddr.getElementType(), SrcElementPHI,
844 /*Idx0=*/1, "omp.arraycpy.src.element");
845 // Check whether we've reached the end.
846 llvm::Value *Done =
847 Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
848 Builder.CreateCondBr(Done, DoneBB, BodyBB);
849 DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
850 SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
851
852 // Done.
853 EmitBlock(DoneBB, /*IsFinished=*/true);
854}
855
857 Address SrcAddr, const VarDecl *DestVD,
858 const VarDecl *SrcVD, const Expr *Copy) {
859 if (OriginalType->isArrayType()) {
860 const auto *BO = dyn_cast<BinaryOperator>(Copy);
861 if (BO && BO->getOpcode() == BO_Assign) {
862 // Perform simple memcpy for simple copying.
863 LValue Dest = MakeAddrLValue(DestAddr, OriginalType);
864 LValue Src = MakeAddrLValue(SrcAddr, OriginalType);
865 EmitAggregateAssign(Dest, Src, OriginalType);
866 } else {
867 // For arrays with complex element types perform element by element
868 // copying.
870 DestAddr, SrcAddr, OriginalType,
871 [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
872 // Working with the single array element, so have to remap
873 // destination and source variables to corresponding array
874 // elements.
876 Remap.addPrivate(DestVD, DestElement);
877 Remap.addPrivate(SrcVD, SrcElement);
878 (void)Remap.Privatize();
880 });
881 }
882 } else {
883 // Remap pseudo source variable to private copy.
885 Remap.addPrivate(SrcVD, SrcAddr);
886 Remap.addPrivate(DestVD, DestAddr);
887 (void)Remap.Privatize();
888 // Emit copying of the whole variable.
890 }
891}
892
894 OMPPrivateScope &PrivateScope) {
895 if (!HaveInsertPoint())
896 return false;
898 bool DeviceConstTarget = getLangOpts().OpenMPIsTargetDevice &&
900 bool FirstprivateIsLastprivate = false;
901 llvm::DenseMap<const VarDecl *, OpenMPLastprivateModifier> Lastprivates;
902 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
903 for (const auto *D : C->varlist())
904 Lastprivates.try_emplace(
906 C->getKind());
907 }
908 llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
910 getOpenMPCaptureRegions(CaptureRegions, EKind);
911 // Force emission of the firstprivate copy if the directive does not emit
912 // outlined function, like omp for, omp simd, omp distribute etc.
913 bool MustEmitFirstprivateCopy =
914 CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
915 for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
916 const auto *IRef = C->varlist_begin();
917 const auto *InitsRef = C->inits().begin();
918 for (const Expr *IInit : C->private_copies()) {
919 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
920 bool ThisFirstprivateIsLastprivate =
921 Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
922 const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD);
923 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
924 if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
925 !FD->getType()->isReferenceType() &&
926 (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
927 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
928 ++IRef;
929 ++InitsRef;
930 continue;
931 }
932 // Do not emit copy for firstprivate constant variables in target regions,
933 // captured by reference.
934 if (DeviceConstTarget && OrigVD->getType().isConstant(getContext()) &&
935 FD && FD->getType()->isReferenceType() &&
936 (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
937 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
938 ++IRef;
939 ++InitsRef;
940 continue;
941 }
942 FirstprivateIsLastprivate =
943 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
944 if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
945 const auto *VDInit =
946 cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
947 bool IsRegistered;
948 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
949 /*RefersToEnclosingVariableOrCapture=*/FD != nullptr,
950 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
951 LValue OriginalLVal;
952 if (!FD) {
953 // Check if the firstprivate variable is just a constant value.
955 if (CE && !CE.isReference()) {
956 // Constant value, no need to create a copy.
957 ++IRef;
958 ++InitsRef;
959 continue;
960 }
961 if (CE && CE.isReference()) {
962 OriginalLVal = CE.getReferenceLValue(*this, &DRE);
963 } else {
964 assert(!CE && "Expected non-constant firstprivate.");
965 OriginalLVal = EmitLValue(&DRE);
966 }
967 } else {
968 OriginalLVal = EmitLValue(&DRE);
969 }
970 QualType Type = VD->getType();
971 if (Type->isArrayType()) {
972 // Emit VarDecl with copy init for arrays.
973 // Get the address of the original variable captured in current
974 // captured region.
975 AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
976 const Expr *Init = VD->getInit();
978 // Perform simple memcpy.
979 LValue Dest = MakeAddrLValue(Emission.getAllocatedAddress(), Type);
980 EmitAggregateAssign(Dest, OriginalLVal, Type);
981 } else {
983 Emission.getAllocatedAddress(), OriginalLVal.getAddress(), Type,
984 [this, VDInit, Init](Address DestElement, Address SrcElement) {
985 // Clean up any temporaries needed by the
986 // initialization.
987 RunCleanupsScope InitScope(*this);
988 // Emit initialization for single element.
989 setAddrOfLocalVar(VDInit, SrcElement);
990 EmitAnyExprToMem(Init, DestElement,
991 Init->getType().getQualifiers(),
992 /*IsInitializer*/ false);
993 LocalDeclMap.erase(VDInit);
994 });
995 }
996 EmitAutoVarCleanups(Emission);
997 IsRegistered =
998 PrivateScope.addPrivate(OrigVD, Emission.getAllocatedAddress());
999 } else {
1000 Address OriginalAddr = OriginalLVal.getAddress();
1001 // Emit private VarDecl with copy init.
1002 // Remap temp VDInit variable to the address of the original
1003 // variable (for proper handling of captured global variables).
1004 setAddrOfLocalVar(VDInit, OriginalAddr);
1005 EmitDecl(*VD);
1006 LocalDeclMap.erase(VDInit);
1007 Address VDAddr = GetAddrOfLocalVar(VD);
1008 if (ThisFirstprivateIsLastprivate &&
1009 Lastprivates[OrigVD->getCanonicalDecl()] ==
1010 OMPC_LASTPRIVATE_conditional) {
1011 // Create/init special variable for lastprivate conditionals.
1012 llvm::Value *V =
1013 EmitLoadOfScalar(MakeAddrLValue(VDAddr, (*IRef)->getType(),
1015 (*IRef)->getExprLoc());
1016 VDAddr = CGM.getOpenMPRuntime().emitLastprivateConditionalInit(
1017 *this, OrigVD);
1018 EmitStoreOfScalar(V, MakeAddrLValue(VDAddr, (*IRef)->getType(),
1020 LocalDeclMap.erase(VD);
1021 setAddrOfLocalVar(VD, VDAddr);
1022 }
1023 IsRegistered = PrivateScope.addPrivate(OrigVD, VDAddr);
1024 }
1025 assert(IsRegistered &&
1026 "firstprivate var already registered as private");
1027 // Silence the warning about unused variable.
1028 (void)IsRegistered;
1029 }
1030 ++IRef;
1031 ++InitsRef;
1032 }
1033 }
1034 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
1035}
1036
1038 const OMPExecutableDirective &D,
1039 CodeGenFunction::OMPPrivateScope &PrivateScope) {
1040 if (!HaveInsertPoint())
1041 return;
1042 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
1043 for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
1044 auto IRef = C->varlist_begin();
1045 for (const Expr *IInit : C->private_copies()) {
1046 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1047 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
1048 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
1049 EmitDecl(*VD);
1050 // Emit private VarDecl with copy init.
1051 bool IsRegistered =
1052 PrivateScope.addPrivate(OrigVD, GetAddrOfLocalVar(VD));
1053 assert(IsRegistered && "private var already registered as private");
1054 // Silence the warning about unused variable.
1055 (void)IsRegistered;
1056 }
1057 ++IRef;
1058 }
1059 }
1060}
1061
1063 if (!HaveInsertPoint())
1064 return false;
1065 // threadprivate_var1 = master_threadprivate_var1;
1066 // operator=(threadprivate_var2, master_threadprivate_var2);
1067 // ...
1068 // __kmpc_barrier(&loc, global_tid);
1069 llvm::DenseSet<const VarDecl *> CopiedVars;
1070 llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
1071 for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
1072 auto IRef = C->varlist_begin();
1073 auto ISrcRef = C->source_exprs().begin();
1074 auto IDestRef = C->destination_exprs().begin();
1075 for (const Expr *AssignOp : C->assignment_ops()) {
1076 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1077 QualType Type = VD->getType();
1078 if (CopiedVars.insert(VD->getCanonicalDecl()).second) {
1079 // Get the address of the master variable. If we are emitting code with
1080 // TLS support, the address is passed from the master as field in the
1081 // captured declaration.
1082 Address MasterAddr = Address::invalid();
1083 if (getLangOpts().OpenMPUseTLS &&
1084 getContext().getTargetInfo().isTLSSupported()) {
1085 assert(CapturedStmtInfo->lookup(VD) &&
1086 "Copyin threadprivates should have been captured!");
1087 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD), true,
1088 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
1089 MasterAddr = EmitLValue(&DRE).getAddress();
1090 LocalDeclMap.erase(VD);
1091 } else {
1092 MasterAddr =
1093 Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD)
1094 : CGM.GetAddrOfGlobal(VD),
1095 CGM.getTypes().ConvertTypeForMem(VD->getType()),
1096 getContext().getDeclAlign(VD));
1097 }
1098 // Get the address of the threadprivate variable.
1099 Address PrivateAddr = EmitLValue(*IRef).getAddress();
1100 if (CopiedVars.size() == 1) {
1101 // At first check if current thread is a master thread. If it is, no
1102 // need to copy data.
1103 CopyBegin = createBasicBlock("copyin.not.master");
1104 CopyEnd = createBasicBlock("copyin.not.master.end");
1105 // TODO: Avoid ptrtoint conversion.
1106 auto *MasterAddrInt = Builder.CreatePtrToInt(
1107 MasterAddr.emitRawPointer(*this), CGM.IntPtrTy);
1108 auto *PrivateAddrInt = Builder.CreatePtrToInt(
1109 PrivateAddr.emitRawPointer(*this), CGM.IntPtrTy);
1110 Builder.CreateCondBr(
1111 Builder.CreateICmpNE(MasterAddrInt, PrivateAddrInt), CopyBegin,
1112 CopyEnd);
1113 EmitBlock(CopyBegin);
1114 }
1115 const auto *SrcVD =
1116 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1117 const auto *DestVD =
1118 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1119 EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
1120 }
1121 ++IRef;
1122 ++ISrcRef;
1123 ++IDestRef;
1124 }
1125 }
1126 if (CopyEnd) {
1127 // Exit out of copying procedure for non-master thread.
1128 EmitBlock(CopyEnd, /*IsFinished=*/true);
1129 return true;
1130 }
1131 return false;
1132}
1133
1135 const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
1136 if (!HaveInsertPoint())
1137 return false;
1138 bool HasAtLeastOneLastprivate = false;
1140 llvm::DenseSet<const VarDecl *> SIMDLCVs;
1141 if (isOpenMPSimdDirective(EKind)) {
1142 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
1143 for (const Expr *C : LoopDirective->counters()) {
1144 SIMDLCVs.insert(
1146 }
1147 }
1148 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1149 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1150 HasAtLeastOneLastprivate = true;
1151 if (isOpenMPTaskLoopDirective(EKind) && !getLangOpts().OpenMPSimd)
1152 break;
1153 const auto *IRef = C->varlist_begin();
1154 const auto *IDestRef = C->destination_exprs().begin();
1155 for (const Expr *IInit : C->private_copies()) {
1156 // Keep the address of the original variable for future update at the end
1157 // of the loop.
1158 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1159 // Taskloops do not require additional initialization, it is done in
1160 // runtime support library.
1161 if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
1162 const auto *DestVD =
1163 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1164 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1165 /*RefersToEnclosingVariableOrCapture=*/
1166 CapturedStmtInfo->lookup(OrigVD) != nullptr,
1167 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
1168 PrivateScope.addPrivate(DestVD, EmitLValue(&DRE).getAddress());
1169 // Check if the variable is also a firstprivate: in this case IInit is
1170 // not generated. Initialization of this variable will happen in codegen
1171 // for 'firstprivate' clause.
1172 if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
1173 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
1174 Address VDAddr = Address::invalid();
1175 if (C->getKind() == OMPC_LASTPRIVATE_conditional) {
1176 VDAddr = CGM.getOpenMPRuntime().emitLastprivateConditionalInit(
1177 *this, OrigVD);
1178 setAddrOfLocalVar(VD, VDAddr);
1179 } else {
1180 // Emit private VarDecl with copy init.
1181 EmitDecl(*VD);
1182 VDAddr = GetAddrOfLocalVar(VD);
1183 }
1184 bool IsRegistered = PrivateScope.addPrivate(OrigVD, VDAddr);
1185 assert(IsRegistered &&
1186 "lastprivate var already registered as private");
1187 (void)IsRegistered;
1188 }
1189 }
1190 ++IRef;
1191 ++IDestRef;
1192 }
1193 }
1194 return HasAtLeastOneLastprivate;
1195}
1196
1198 const OMPExecutableDirective &D, bool NoFinals,
1199 llvm::Value *IsLastIterCond) {
1200 if (!HaveInsertPoint())
1201 return;
1202 // Emit following code:
1203 // if (<IsLastIterCond>) {
1204 // orig_var1 = private_orig_var1;
1205 // ...
1206 // orig_varn = private_orig_varn;
1207 // }
1208 llvm::BasicBlock *ThenBB = nullptr;
1209 llvm::BasicBlock *DoneBB = nullptr;
1210 if (IsLastIterCond) {
1211 // Emit implicit barrier if at least one lastprivate conditional is found
1212 // and this is not a simd mode.
1213 if (!getLangOpts().OpenMPSimd &&
1214 llvm::any_of(D.getClausesOfKind<OMPLastprivateClause>(),
1215 [](const OMPLastprivateClause *C) {
1216 return C->getKind() == OMPC_LASTPRIVATE_conditional;
1217 })) {
1218 CGM.getOpenMPRuntime().emitBarrierCall(*this, D.getBeginLoc(),
1219 OMPD_unknown,
1220 /*EmitChecks=*/false,
1221 /*ForceSimpleCall=*/true);
1222 }
1223 ThenBB = createBasicBlock(".omp.lastprivate.then");
1224 DoneBB = createBasicBlock(".omp.lastprivate.done");
1225 Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
1226 EmitBlock(ThenBB);
1227 }
1228 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1229 llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1230 if (const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
1231 auto IC = LoopDirective->counters().begin();
1232 for (const Expr *F : LoopDirective->finals()) {
1233 const auto *D =
1234 cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
1235 if (NoFinals)
1236 AlreadyEmittedVars.insert(D);
1237 else
1238 LoopCountersAndUpdates[D] = F;
1239 ++IC;
1240 }
1241 }
1242 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1243 auto IRef = C->varlist_begin();
1244 auto ISrcRef = C->source_exprs().begin();
1245 auto IDestRef = C->destination_exprs().begin();
1246 for (const Expr *AssignOp : C->assignment_ops()) {
1247 const auto *PrivateVD =
1248 cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1249 QualType Type = PrivateVD->getType();
1250 const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1251 if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1252 // If lastprivate variable is a loop control variable for loop-based
1253 // directive, update its value before copyin back to original
1254 // variable.
1255 if (const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
1256 EmitIgnoredExpr(FinalExpr);
1257 const auto *SrcVD =
1258 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1259 const auto *DestVD =
1260 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1261 // Get the address of the private variable.
1262 Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
1263 if (const auto *RefTy = PrivateVD->getType()->getAs<ReferenceType>())
1264 PrivateAddr = Address(
1265 Builder.CreateLoad(PrivateAddr),
1266 CGM.getTypes().ConvertTypeForMem(RefTy->getPointeeType()),
1267 CGM.getNaturalTypeAlignment(RefTy->getPointeeType()));
1268 // Store the last value to the private copy in the last iteration.
1269 if (C->getKind() == OMPC_LASTPRIVATE_conditional)
1270 CGM.getOpenMPRuntime().emitLastprivateConditionalFinalUpdate(
1271 *this, MakeAddrLValue(PrivateAddr, (*IRef)->getType()), PrivateVD,
1272 (*IRef)->getExprLoc());
1273 // Get the address of the original variable.
1274 Address OriginalAddr = GetAddrOfLocalVar(DestVD);
1275 EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1276 }
1277 ++IRef;
1278 ++ISrcRef;
1279 ++IDestRef;
1280 }
1281 if (const Expr *PostUpdate = C->getPostUpdateExpr())
1282 EmitIgnoredExpr(PostUpdate);
1283 }
1284 if (IsLastIterCond)
1285 EmitBlock(DoneBB, /*IsFinished=*/true);
1286}
1287
1289 const OMPExecutableDirective &D,
1290 CodeGenFunction::OMPPrivateScope &PrivateScope, bool ForInscan) {
1291 if (!HaveInsertPoint())
1292 return;
1295 SmallVector<const Expr *, 4> ReductionOps;
1301 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1302 if (ForInscan != (C->getModifier() == OMPC_REDUCTION_inscan))
1303 continue;
1304 Shareds.append(C->varlist_begin(), C->varlist_end());
1305 Privates.append(C->privates().begin(), C->privates().end());
1306 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1307 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1308 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1309 if (C->getModifier() == OMPC_REDUCTION_task) {
1310 Data.ReductionVars.append(C->privates().begin(), C->privates().end());
1311 Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
1312 Data.ReductionCopies.append(C->privates().begin(), C->privates().end());
1313 Data.ReductionOps.append(C->reduction_ops().begin(),
1314 C->reduction_ops().end());
1315 TaskLHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1316 TaskRHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1317 }
1318 }
1319 ReductionCodeGen RedCG(Shareds, Shareds, Privates, ReductionOps);
1320 unsigned Count = 0;
1321 auto *ILHS = LHSs.begin();
1322 auto *IRHS = RHSs.begin();
1323 auto *IPriv = Privates.begin();
1324 for (const Expr *IRef : Shareds) {
1325 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1326 // Emit private VarDecl with reduction init.
1327 RedCG.emitSharedOrigLValue(*this, Count);
1328 RedCG.emitAggregateType(*this, Count);
1329 AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD);
1330 RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(),
1331 RedCG.getSharedLValue(Count).getAddress(),
1332 [&Emission](CodeGenFunction &CGF) {
1333 CGF.EmitAutoVarInit(Emission);
1334 return true;
1335 });
1336 EmitAutoVarCleanups(Emission);
1337 Address BaseAddr = RedCG.adjustPrivateAddress(
1338 *this, Count, Emission.getAllocatedAddress());
1339 bool IsRegistered =
1340 PrivateScope.addPrivate(RedCG.getBaseDecl(Count), BaseAddr);
1341 assert(IsRegistered && "private var already registered as private");
1342 // Silence the warning about unused variable.
1343 (void)IsRegistered;
1344
1345 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1346 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1347 QualType Type = PrivateVD->getType();
1348 bool isaOMPArraySectionExpr = isa<ArraySectionExpr>(IRef);
1349 if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
1350 // Store the address of the original variable associated with the LHS
1351 // implicit variable.
1352 PrivateScope.addPrivate(LHSVD, RedCG.getSharedLValue(Count).getAddress());
1353 PrivateScope.addPrivate(RHSVD, GetAddrOfLocalVar(PrivateVD));
1354 } else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
1356 // Store the address of the original variable associated with the LHS
1357 // implicit variable.
1358 PrivateScope.addPrivate(LHSVD, RedCG.getSharedLValue(Count).getAddress());
1359 PrivateScope.addPrivate(RHSVD,
1360 GetAddrOfLocalVar(PrivateVD).withElementType(
1361 ConvertTypeForMem(RHSVD->getType())));
1362 } else {
1363 QualType Type = PrivateVD->getType();
1364 bool IsArray = getContext().getAsArrayType(Type) != nullptr;
1365 Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress();
1366 // Store the address of the original variable associated with the LHS
1367 // implicit variable.
1368 if (IsArray) {
1369 OriginalAddr =
1370 OriginalAddr.withElementType(ConvertTypeForMem(LHSVD->getType()));
1371 }
1372 PrivateScope.addPrivate(LHSVD, OriginalAddr);
1373 PrivateScope.addPrivate(
1374 RHSVD, IsArray ? GetAddrOfLocalVar(PrivateVD).withElementType(
1375 ConvertTypeForMem(RHSVD->getType()))
1376 : GetAddrOfLocalVar(PrivateVD));
1377 }
1378 ++ILHS;
1379 ++IRHS;
1380 ++IPriv;
1381 ++Count;
1382 }
1383 if (!Data.ReductionVars.empty()) {
1385 Data.IsReductionWithTaskMod = true;
1386 Data.IsWorksharingReduction = isOpenMPWorksharingDirective(EKind);
1387 llvm::Value *ReductionDesc = CGM.getOpenMPRuntime().emitTaskReductionInit(
1388 *this, D.getBeginLoc(), TaskLHSs, TaskRHSs, Data);
1389 const Expr *TaskRedRef = nullptr;
1390 switch (EKind) {
1391 case OMPD_parallel:
1392 TaskRedRef = cast<OMPParallelDirective>(D).getTaskReductionRefExpr();
1393 break;
1394 case OMPD_for:
1395 TaskRedRef = cast<OMPForDirective>(D).getTaskReductionRefExpr();
1396 break;
1397 case OMPD_sections:
1398 TaskRedRef = cast<OMPSectionsDirective>(D).getTaskReductionRefExpr();
1399 break;
1400 case OMPD_parallel_for:
1401 TaskRedRef = cast<OMPParallelForDirective>(D).getTaskReductionRefExpr();
1402 break;
1403 case OMPD_parallel_master:
1404 TaskRedRef =
1405 cast<OMPParallelMasterDirective>(D).getTaskReductionRefExpr();
1406 break;
1407 case OMPD_parallel_sections:
1408 TaskRedRef =
1409 cast<OMPParallelSectionsDirective>(D).getTaskReductionRefExpr();
1410 break;
1411 case OMPD_target_parallel:
1412 TaskRedRef =
1413 cast<OMPTargetParallelDirective>(D).getTaskReductionRefExpr();
1414 break;
1415 case OMPD_target_parallel_for:
1416 TaskRedRef =
1417 cast<OMPTargetParallelForDirective>(D).getTaskReductionRefExpr();
1418 break;
1419 case OMPD_distribute_parallel_for:
1420 TaskRedRef =
1421 cast<OMPDistributeParallelForDirective>(D).getTaskReductionRefExpr();
1422 break;
1423 case OMPD_teams_distribute_parallel_for:
1425 .getTaskReductionRefExpr();
1426 break;
1427 case OMPD_target_teams_distribute_parallel_for:
1429 .getTaskReductionRefExpr();
1430 break;
1431 case OMPD_simd:
1432 case OMPD_for_simd:
1433 case OMPD_section:
1434 case OMPD_single:
1435 case OMPD_master:
1436 case OMPD_critical:
1437 case OMPD_parallel_for_simd:
1438 case OMPD_task:
1439 case OMPD_taskyield:
1440 case OMPD_error:
1441 case OMPD_barrier:
1442 case OMPD_taskwait:
1443 case OMPD_taskgroup:
1444 case OMPD_flush:
1445 case OMPD_depobj:
1446 case OMPD_scan:
1447 case OMPD_ordered:
1448 case OMPD_atomic:
1449 case OMPD_teams:
1450 case OMPD_target:
1451 case OMPD_cancellation_point:
1452 case OMPD_cancel:
1453 case OMPD_target_data:
1454 case OMPD_target_enter_data:
1455 case OMPD_target_exit_data:
1456 case OMPD_taskloop:
1457 case OMPD_taskloop_simd:
1458 case OMPD_master_taskloop:
1459 case OMPD_master_taskloop_simd:
1460 case OMPD_parallel_master_taskloop:
1461 case OMPD_parallel_master_taskloop_simd:
1462 case OMPD_distribute:
1463 case OMPD_target_update:
1464 case OMPD_distribute_parallel_for_simd:
1465 case OMPD_distribute_simd:
1466 case OMPD_target_parallel_for_simd:
1467 case OMPD_target_simd:
1468 case OMPD_teams_distribute:
1469 case OMPD_teams_distribute_simd:
1470 case OMPD_teams_distribute_parallel_for_simd:
1471 case OMPD_target_teams:
1472 case OMPD_target_teams_distribute:
1473 case OMPD_target_teams_distribute_parallel_for_simd:
1474 case OMPD_target_teams_distribute_simd:
1475 case OMPD_declare_target:
1476 case OMPD_end_declare_target:
1477 case OMPD_threadprivate:
1478 case OMPD_allocate:
1479 case OMPD_declare_reduction:
1480 case OMPD_declare_mapper:
1481 case OMPD_declare_simd:
1482 case OMPD_requires:
1483 case OMPD_declare_variant:
1484 case OMPD_begin_declare_variant:
1485 case OMPD_end_declare_variant:
1486 case OMPD_unknown:
1487 default:
1488 llvm_unreachable("Unexpected directive with task reductions.");
1489 }
1490
1491 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(TaskRedRef)->getDecl());
1492 EmitVarDecl(*VD);
1493 EmitStoreOfScalar(ReductionDesc, GetAddrOfLocalVar(VD),
1494 /*Volatile=*/false, TaskRedRef->getType());
1495 }
1496}
1497
1499 const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) {
1500 if (!HaveInsertPoint())
1501 return;
1506 llvm::SmallVector<bool, 8> IsPrivateVarReduction;
1507 bool HasAtLeastOneReduction = false;
1508 bool IsReductionWithTaskMod = false;
1509 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1510 // Do not emit for inscan reductions.
1511 if (C->getModifier() == OMPC_REDUCTION_inscan)
1512 continue;
1513 HasAtLeastOneReduction = true;
1514 Privates.append(C->privates().begin(), C->privates().end());
1515 LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1516 RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1517 IsPrivateVarReduction.append(C->private_var_reduction_flags().begin(),
1518 C->private_var_reduction_flags().end());
1519 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1520 IsReductionWithTaskMod =
1521 IsReductionWithTaskMod || C->getModifier() == OMPC_REDUCTION_task;
1522 }
1523 if (HasAtLeastOneReduction) {
1525 if (IsReductionWithTaskMod) {
1526 CGM.getOpenMPRuntime().emitTaskReductionFini(
1527 *this, D.getBeginLoc(), isOpenMPWorksharingDirective(EKind));
1528 }
1529 bool TeamsLoopCanBeParallel = false;
1530 if (auto *TTLD = dyn_cast<OMPTargetTeamsGenericLoopDirective>(&D))
1531 TeamsLoopCanBeParallel = TTLD->canBeParallelFor();
1532 bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
1534 TeamsLoopCanBeParallel || ReductionKind == OMPD_simd;
1535 bool SimpleReduction = ReductionKind == OMPD_simd;
1536 // Emit nowait reduction if nowait clause is present or directive is a
1537 // parallel directive (it always has implicit barrier).
1538 CGM.getOpenMPRuntime().emitReduction(
1539 *this, D.getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
1540 {WithNowait, SimpleReduction, IsPrivateVarReduction, ReductionKind});
1541 }
1542}
1543
1546 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1547 if (!CGF.HaveInsertPoint())
1548 return;
1549 llvm::BasicBlock *DoneBB = nullptr;
1550 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1551 if (const Expr *PostUpdate = C->getPostUpdateExpr()) {
1552 if (!DoneBB) {
1553 if (llvm::Value *Cond = CondGen(CGF)) {
1554 // If the first post-update expression is found, emit conditional
1555 // block if it was requested.
1556 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");
1557 DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done");
1558 CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1559 CGF.EmitBlock(ThenBB);
1560 }
1561 }
1562 CGF.EmitIgnoredExpr(PostUpdate);
1563 }
1564 }
1565 if (DoneBB)
1566 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
1567}
1568
1569namespace {
1570/// Codegen lambda for appending distribute lower and upper bounds to outlined
1571/// parallel function. This is necessary for combined constructs such as
1572/// 'distribute parallel for'
1573typedef llvm::function_ref<void(CodeGenFunction &,
1574 const OMPExecutableDirective &,
1575 llvm::SmallVectorImpl<llvm::Value *> &)>
1576 CodeGenBoundParametersTy;
1577} // anonymous namespace
1578
1579static void
1581 const OMPExecutableDirective &S) {
1582 if (CGF.getLangOpts().OpenMP < 50)
1583 return;
1584 llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> PrivateDecls;
1585 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
1586 for (const Expr *Ref : C->varlist()) {
1587 if (!Ref->getType()->isScalarType())
1588 continue;
1589 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1590 if (!DRE)
1591 continue;
1592 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1594 }
1595 }
1596 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
1597 for (const Expr *Ref : C->varlist()) {
1598 if (!Ref->getType()->isScalarType())
1599 continue;
1600 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1601 if (!DRE)
1602 continue;
1603 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1605 }
1606 }
1607 for (const auto *C : S.getClausesOfKind<OMPLinearClause>()) {
1608 for (const Expr *Ref : C->varlist()) {
1609 if (!Ref->getType()->isScalarType())
1610 continue;
1611 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1612 if (!DRE)
1613 continue;
1614 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1616 }
1617 }
1618 // Privates should ne analyzed since they are not captured at all.
1619 // Task reductions may be skipped - tasks are ignored.
1620 // Firstprivates do not return value but may be passed by reference - no need
1621 // to check for updated lastprivate conditional.
1622 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
1623 for (const Expr *Ref : C->varlist()) {
1624 if (!Ref->getType()->isScalarType())
1625 continue;
1626 const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
1627 if (!DRE)
1628 continue;
1629 PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
1630 }
1631 }
1633 CGF, S, PrivateDecls);
1634}
1635
1638 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1639 const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1640 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1641 llvm::Value *NumThreads = nullptr;
1643 // OpenMP 6.0, 10.4: "If no severity clause is specified then the effect is as
1644 // if sev-level is fatal."
1645 OpenMPSeverityClauseKind Severity = OMPC_SEVERITY_fatal;
1646 clang::Expr *Message = nullptr;
1647 SourceLocation SeverityLoc = SourceLocation();
1648 SourceLocation MessageLoc = SourceLocation();
1649
1650 llvm::Function *OutlinedFn =
1652 CGF, S, *CS->getCapturedDecl()->param_begin(), InnermostKind,
1653 CodeGen);
1654
1655 if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
1656 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
1657 NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
1658 /*IgnoreResultAssign=*/true);
1659 Modifier = NumThreadsClause->getModifier();
1660 if (const auto *MessageClause = S.getSingleClause<OMPMessageClause>()) {
1661 Message = MessageClause->getMessageString();
1662 MessageLoc = MessageClause->getBeginLoc();
1663 }
1664 if (const auto *SeverityClause = S.getSingleClause<OMPSeverityClause>()) {
1665 Severity = SeverityClause->getSeverityKind();
1666 SeverityLoc = SeverityClause->getBeginLoc();
1667 }
1669 CGF, NumThreads, NumThreadsClause->getBeginLoc(), Modifier, Severity,
1670 SeverityLoc, Message, MessageLoc);
1671 }
1672 if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
1673 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
1675 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
1676 }
1677 const Expr *IfCond = nullptr;
1678 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
1679 if (C->getNameModifier() == OMPD_unknown ||
1680 C->getNameModifier() == OMPD_parallel) {
1681 IfCond = C->getCondition();
1682 break;
1683 }
1684 }
1685
1686 OMPParallelScope Scope(CGF, S);
1688 // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk
1689 // lower and upper bounds with the pragma 'for' chunking mechanism.
1690 // The following lambda takes care of appending the lower and upper bound
1691 // parameters when necessary
1692 CodeGenBoundParameters(CGF, S, CapturedVars);
1693 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
1694 CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn,
1695 CapturedVars, IfCond, NumThreads,
1696 Modifier, Severity, Message);
1697}
1698
1699static bool isAllocatableDecl(const VarDecl *VD) {
1700 const VarDecl *CVD = VD->getCanonicalDecl();
1701 if (!CVD->hasAttr<OMPAllocateDeclAttr>())
1702 return false;
1703 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
1704 // Use the default allocation.
1705 return !((AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc ||
1706 AA->getAllocatorType() == OMPAllocateDeclAttr::OMPNullMemAlloc) &&
1707 !AA->getAllocator());
1708}
1709
1713
1715 const OMPExecutableDirective &S) {
1716 bool Copyins = CGF.EmitOMPCopyinClause(S);
1717 if (Copyins) {
1718 // Emit implicit barrier to synchronize threads and avoid data races on
1719 // propagation master's thread values of threadprivate variables to local
1720 // instances of that variables of all other implicit threads.
1722 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
1723 /*ForceSimpleCall=*/true);
1724 }
1725}
1726
1728 CodeGenFunction &CGF, const VarDecl *VD) {
1729 CodeGenModule &CGM = CGF.CGM;
1730 auto &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1731
1732 if (!VD)
1733 return Address::invalid();
1734 const VarDecl *CVD = VD->getCanonicalDecl();
1735 if (!isAllocatableDecl(CVD))
1736 return Address::invalid();
1737 llvm::Value *Size;
1738 CharUnits Align = CGM.getContext().getDeclAlign(CVD);
1739 if (CVD->getType()->isVariablyModifiedType()) {
1740 Size = CGF.getTypeSize(CVD->getType());
1741 // Align the size: ((size + align - 1) / align) * align
1742 Size = CGF.Builder.CreateNUWAdd(
1743 Size, CGM.getSize(Align - CharUnits::fromQuantity(1)));
1744 Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align));
1745 Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align));
1746 } else {
1747 CharUnits Sz = CGM.getContext().getTypeSizeInChars(CVD->getType());
1748 Size = CGM.getSize(Sz.alignTo(Align));
1749 }
1750
1751 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
1752 assert(AA->getAllocator() &&
1753 "Expected allocator expression for non-default allocator.");
1754 llvm::Value *Allocator = CGF.EmitScalarExpr(AA->getAllocator());
1755 // According to the standard, the original allocator type is a enum (integer).
1756 // Convert to pointer type, if required.
1757 if (Allocator->getType()->isIntegerTy())
1758 Allocator = CGF.Builder.CreateIntToPtr(Allocator, CGM.VoidPtrTy);
1759 else if (Allocator->getType()->isPointerTy())
1760 Allocator = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Allocator,
1761 CGM.VoidPtrTy);
1762
1763 llvm::Value *Addr = OMPBuilder.createOMPAlloc(
1764 CGF.Builder, Size, Allocator,
1765 getNameWithSeparators({CVD->getName(), ".void.addr"}, ".", "."));
1766 llvm::CallInst *FreeCI =
1767 OMPBuilder.createOMPFree(CGF.Builder, Addr, Allocator);
1768
1769 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(NormalAndEHCleanup, FreeCI);
1771 Addr,
1772 CGF.ConvertTypeForMem(CGM.getContext().getPointerType(CVD->getType())),
1773 getNameWithSeparators({CVD->getName(), ".addr"}, ".", "."));
1774 return Address(Addr, CGF.ConvertTypeForMem(CVD->getType()), Align);
1775}
1776
1778 CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr,
1779 SourceLocation Loc) {
1780 CodeGenModule &CGM = CGF.CGM;
1781 if (CGM.getLangOpts().OpenMPUseTLS &&
1782 CGM.getContext().getTargetInfo().isTLSSupported())
1783 return VDAddr;
1784
1785 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1786
1787 llvm::Type *VarTy = VDAddr.getElementType();
1788 llvm::Value *Data =
1789 CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.Int8PtrTy);
1790 llvm::ConstantInt *Size = CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy));
1791 std::string Suffix = getNameWithSeparators({"cache", ""});
1792 llvm::Twine CacheName = Twine(CGM.getMangledName(VD)).concat(Suffix);
1793
1794 llvm::CallInst *ThreadPrivateCacheCall =
1795 OMPBuilder.createCachedThreadPrivate(CGF.Builder, Data, Size, CacheName);
1796
1797 return Address(ThreadPrivateCacheCall, CGM.Int8Ty, VDAddr.getAlignment());
1798}
1799
1801 ArrayRef<StringRef> Parts, StringRef FirstSeparator, StringRef Separator) {
1802 SmallString<128> Buffer;
1803 llvm::raw_svector_ostream OS(Buffer);
1804 StringRef Sep = FirstSeparator;
1805 for (StringRef Part : Parts) {
1806 OS << Sep << Part;
1807 Sep = Separator;
1808 }
1809 return OS.str().str();
1810}
1811
1813 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP,
1814 InsertPointTy CodeGenIP, Twine RegionName) {
1816 Builder.restoreIP(CodeGenIP);
1817 llvm::BasicBlock *FiniBB = splitBBWithSuffix(Builder, /*CreateBranch=*/false,
1818 "." + RegionName + ".after");
1819
1820 {
1821 OMPBuilderCBHelpers::InlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB);
1822 CGF.EmitStmt(RegionBodyStmt);
1823 }
1824
1825 if (Builder.saveIP().isSet())
1826 Builder.CreateBr(FiniBB);
1827}
1828
1830 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP,
1831 InsertPointTy CodeGenIP, Twine RegionName) {
1833 Builder.restoreIP(CodeGenIP);
1834 llvm::BasicBlock *FiniBB = splitBBWithSuffix(Builder, /*CreateBranch=*/false,
1835 "." + RegionName + ".after");
1836
1837 {
1838 OMPBuilderCBHelpers::OutlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB);
1839 CGF.EmitStmt(RegionBodyStmt);
1840 }
1841
1842 if (Builder.saveIP().isSet())
1843 Builder.CreateBr(FiniBB);
1844}
1845
1846void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {
1847 if (CGM.getLangOpts().OpenMPIRBuilder) {
1848 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1849 // Check if we have any if clause associated with the directive.
1850 llvm::Value *IfCond = nullptr;
1851 if (const auto *C = S.getSingleClause<OMPIfClause>())
1852 IfCond = EmitScalarExpr(C->getCondition(),
1853 /*IgnoreResultAssign=*/true);
1854
1855 llvm::Value *NumThreads = nullptr;
1856 if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>())
1857 NumThreads = EmitScalarExpr(NumThreadsClause->getNumThreads(),
1858 /*IgnoreResultAssign=*/true);
1859
1860 ProcBindKind ProcBind = OMP_PROC_BIND_default;
1861 if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>())
1862 ProcBind = ProcBindClause->getProcBindKind();
1863
1864 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1865
1866 // The cleanup callback that finalizes all variables at the given location,
1867 // thus calls destructors etc.
1868 auto FiniCB = [this](InsertPointTy IP) {
1870 return llvm::Error::success();
1871 };
1872
1873 // Privatization callback that performs appropriate action for
1874 // shared/private/firstprivate/lastprivate/copyin/... variables.
1875 //
1876 // TODO: This defaults to shared right now.
1877 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
1878 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
1879 // The next line is appropriate only for variables (Val) with the
1880 // data-sharing attribute "shared".
1881 ReplVal = &Val;
1882
1883 return CodeGenIP;
1884 };
1885
1886 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1887 const Stmt *ParallelRegionBodyStmt = CS->getCapturedStmt();
1888
1889 auto BodyGenCB = [&, this](InsertPointTy AllocaIP,
1890 InsertPointTy CodeGenIP) {
1892 *this, ParallelRegionBodyStmt, AllocaIP, CodeGenIP, "parallel");
1893 return llvm::Error::success();
1894 };
1895
1896 CGCapturedStmtInfo CGSI(*CS, CR_OpenMP);
1897 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI);
1898 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
1899 AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());
1900 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
1901 OMPBuilder.createParallel(Builder, AllocaIP, BodyGenCB, PrivCB, FiniCB,
1902 IfCond, NumThreads, ProcBind, S.hasCancel()));
1903 Builder.restoreIP(AfterIP);
1904 return;
1905 }
1906
1907 // Emit parallel region as a standalone region.
1908 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1909 Action.Enter(CGF);
1910 OMPPrivateScope PrivateScope(CGF);
1911 emitOMPCopyinClause(CGF, S);
1912 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
1913 CGF.EmitOMPPrivateClause(S, PrivateScope);
1914 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
1915 (void)PrivateScope.Privatize();
1916 CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
1917 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
1918 };
1919 {
1920 auto LPCRegion =
1922 emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen,
1925 [](CodeGenFunction &) { return nullptr; });
1926 }
1927 // Check for outer lastprivate conditional update.
1929}
1930
1934
1935namespace {
1936/// RAII to handle scopes for loop transformation directives.
1937class OMPTransformDirectiveScopeRAII {
1938 OMPLoopScope *Scope = nullptr;
1940 CodeGenFunction::CGCapturedStmtRAII *CapInfoRAII = nullptr;
1941
1942 OMPTransformDirectiveScopeRAII(const OMPTransformDirectiveScopeRAII &) =
1943 delete;
1944 OMPTransformDirectiveScopeRAII &
1945 operator=(const OMPTransformDirectiveScopeRAII &) = delete;
1946
1947public:
1948 OMPTransformDirectiveScopeRAII(CodeGenFunction &CGF, const Stmt *S) {
1949 if (const auto *Dir = dyn_cast<OMPLoopBasedDirective>(S)) {
1950 Scope = new OMPLoopScope(CGF, *Dir);
1952 CapInfoRAII = new CodeGenFunction::CGCapturedStmtRAII(CGF, CGSI);
1953 }
1954 if (const auto *Dir =
1955 dyn_cast<OMPCanonicalLoopSequenceTransformationDirective>(S)) {
1956 // For simplicity we reuse the loop scope similarly to what we do with
1957 // OMPCanonicalLoopNestTransformationDirective do by being a subclass
1958 // of OMPLoopBasedDirective.
1959 Scope = new OMPLoopScope(CGF, *Dir);
1961 CapInfoRAII = new CodeGenFunction::CGCapturedStmtRAII(CGF, CGSI);
1962 }
1963 }
1964 ~OMPTransformDirectiveScopeRAII() {
1965 if (!Scope)
1966 return;
1967 delete CapInfoRAII;
1968 delete CGSI;
1969 delete Scope;
1970 }
1971};
1972} // namespace
1973
1974static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop,
1975 int MaxLevel, int Level = 0) {
1976 assert(Level < MaxLevel && "Too deep lookup during loop body codegen.");
1977 const Stmt *SimplifiedS = S->IgnoreContainers();
1978 if (const auto *CS = dyn_cast<CompoundStmt>(SimplifiedS)) {
1979 PrettyStackTraceLoc CrashInfo(
1980 CGF.getContext().getSourceManager(), CS->getLBracLoc(),
1981 "LLVM IR generation of compound statement ('{}')");
1982
1983 // Keep track of the current cleanup stack depth, including debug scopes.
1985 for (const Stmt *CurStmt : CS->body())
1986 emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level);
1987 return;
1988 }
1989 if (SimplifiedS == NextLoop) {
1990 if (auto *Dir = dyn_cast<OMPLoopTransformationDirective>(SimplifiedS))
1991 SimplifiedS = Dir->getTransformedStmt();
1992 if (const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(SimplifiedS))
1993 SimplifiedS = CanonLoop->getLoopStmt();
1994 if (const auto *For = dyn_cast<ForStmt>(SimplifiedS)) {
1995 S = For->getBody();
1996 } else {
1997 assert(isa<CXXForRangeStmt>(SimplifiedS) &&
1998 "Expected canonical for loop or range-based for loop.");
1999 const auto *CXXFor = cast<CXXForRangeStmt>(SimplifiedS);
2000 CGF.EmitStmt(CXXFor->getLoopVarStmt());
2001 S = CXXFor->getBody();
2002 }
2003 if (Level + 1 < MaxLevel) {
2004 NextLoop = OMPLoopDirective::tryToFindNextInnerLoop(
2005 S, /*TryImperfectlyNestedLoops=*/true);
2006 emitBody(CGF, S, NextLoop, MaxLevel, Level + 1);
2007 return;
2008 }
2009 }
2010 CGF.EmitStmt(S);
2011}
2012
2015 RunCleanupsScope BodyScope(*this);
2016 // Update counters values on current iteration.
2017 for (const Expr *UE : D.updates())
2018 EmitIgnoredExpr(UE);
2019 // Update the linear variables.
2020 // In distribute directives only loop counters may be marked as linear, no
2021 // need to generate the code for them.
2023 if (!isOpenMPDistributeDirective(EKind)) {
2024 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2025 for (const Expr *UE : C->updates())
2026 EmitIgnoredExpr(UE);
2027 }
2028 }
2029
2030 // On a continue in the body, jump to the end.
2031 JumpDest Continue = getJumpDestInCurrentScope("omp.body.continue");
2032 BreakContinueStack.push_back(BreakContinue(D, LoopExit, Continue));
2033 for (const Expr *E : D.finals_conditions()) {
2034 if (!E)
2035 continue;
2036 // Check that loop counter in non-rectangular nest fits into the iteration
2037 // space.
2038 llvm::BasicBlock *NextBB = createBasicBlock("omp.body.next");
2039 EmitBranchOnBoolExpr(E, NextBB, Continue.getBlock(),
2040 getProfileCount(D.getBody()));
2041 EmitBlock(NextBB);
2042 }
2043
2044 OMPPrivateScope InscanScope(*this);
2045 EmitOMPReductionClauseInit(D, InscanScope, /*ForInscan=*/true);
2046 bool IsInscanRegion = InscanScope.Privatize();
2047 if (IsInscanRegion) {
2048 // Need to remember the block before and after scan directive
2049 // to dispatch them correctly depending on the clause used in
2050 // this directive, inclusive or exclusive. For inclusive scan the natural
2051 // order of the blocks is used, for exclusive clause the blocks must be
2052 // executed in reverse order.
2053 OMPBeforeScanBlock = createBasicBlock("omp.before.scan.bb");
2054 OMPAfterScanBlock = createBasicBlock("omp.after.scan.bb");
2055 // No need to allocate inscan exit block, in simd mode it is selected in the
2056 // codegen for the scan directive.
2057 if (EKind != OMPD_simd && !getLangOpts().OpenMPSimd)
2058 OMPScanExitBlock = createBasicBlock("omp.exit.inscan.bb");
2059 OMPScanDispatch = createBasicBlock("omp.inscan.dispatch");
2062 }
2063
2064 // Emit loop variables for C++ range loops.
2065 const Stmt *Body =
2066 D.getInnermostCapturedStmt()->getCapturedStmt()->IgnoreContainers();
2067 // Emit loop body.
2068 emitBody(*this, Body,
2069 OMPLoopBasedDirective::tryToFindNextInnerLoop(
2070 Body, /*TryImperfectlyNestedLoops=*/true),
2071 D.getLoopsNumber());
2072
2073 // Jump to the dispatcher at the end of the loop body.
2074 if (IsInscanRegion)
2076
2077 // The end (updates/cleanups).
2078 EmitBlock(Continue.getBlock());
2079 BreakContinueStack.pop_back();
2080}
2081
2082using EmittedClosureTy = std::pair<llvm::Function *, llvm::Value *>;
2083
2084/// Emit a captured statement and return the function as well as its captured
2085/// closure context.
2087 const CapturedStmt *S) {
2088 LValue CapStruct = ParentCGF.InitCapturedStruct(*S);
2089 CodeGenFunction CGF(ParentCGF.CGM, /*suppressNewContext=*/true);
2090 std::unique_ptr<CodeGenFunction::CGCapturedStmtInfo> CSI =
2091 std::make_unique<CodeGenFunction::CGCapturedStmtInfo>(*S);
2092 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, CSI.get());
2093 llvm::Function *F = CGF.GenerateCapturedStmtFunction(*S);
2094
2095 return {F, CapStruct.getPointer(ParentCGF)};
2096}
2097
2098/// Emit a call to a previously captured closure.
2099static llvm::CallInst *
2102 // Append the closure context to the argument.
2103 SmallVector<llvm::Value *> EffectiveArgs;
2104 EffectiveArgs.reserve(Args.size() + 1);
2105 llvm::append_range(EffectiveArgs, Args);
2106 EffectiveArgs.push_back(Cap.second);
2107
2108 return ParentCGF.Builder.CreateCall(Cap.first, EffectiveArgs);
2109}
2110
2111llvm::CanonicalLoopInfo *
2113 assert(Depth == 1 && "Nested loops with OpenMPIRBuilder not yet implemented");
2114
2115 // The caller is processing the loop-associated directive processing the \p
2116 // Depth loops nested in \p S. Put the previous pending loop-associated
2117 // directive to the stack. If the current loop-associated directive is a loop
2118 // transformation directive, it will push its generated loops onto the stack
2119 // such that together with the loops left here they form the combined loop
2120 // nest for the parent loop-associated directive.
2121 int ParentExpectedOMPLoopDepth = ExpectedOMPLoopDepth;
2122 ExpectedOMPLoopDepth = Depth;
2123
2124 EmitStmt(S);
2125 assert(OMPLoopNestStack.size() >= (size_t)Depth && "Found too few loops");
2126
2127 // The last added loop is the outermost one.
2128 llvm::CanonicalLoopInfo *Result = OMPLoopNestStack.back();
2129
2130 // Pop the \p Depth loops requested by the call from that stack and restore
2131 // the previous context.
2132 OMPLoopNestStack.pop_back_n(Depth);
2133 ExpectedOMPLoopDepth = ParentExpectedOMPLoopDepth;
2134
2135 return Result;
2136}
2137
2138void CodeGenFunction::EmitOMPCanonicalLoop(const OMPCanonicalLoop *S) {
2139 const Stmt *SyntacticalLoop = S->getLoopStmt();
2140 if (!getLangOpts().OpenMPIRBuilder) {
2141 // Ignore if OpenMPIRBuilder is not enabled.
2142 EmitStmt(SyntacticalLoop);
2143 return;
2144 }
2145
2146 LexicalScope ForScope(*this, S->getSourceRange());
2147
2148 // Emit init statements. The Distance/LoopVar funcs may reference variable
2149 // declarations they contain.
2150 const Stmt *BodyStmt;
2151 if (const auto *For = dyn_cast<ForStmt>(SyntacticalLoop)) {
2152 if (const Stmt *InitStmt = For->getInit())
2153 EmitStmt(InitStmt);
2154 BodyStmt = For->getBody();
2155 } else if (const auto *RangeFor =
2156 dyn_cast<CXXForRangeStmt>(SyntacticalLoop)) {
2157 if (const DeclStmt *RangeStmt = RangeFor->getRangeStmt())
2158 EmitStmt(RangeStmt);
2159 if (const DeclStmt *BeginStmt = RangeFor->getBeginStmt())
2160 EmitStmt(BeginStmt);
2161 if (const DeclStmt *EndStmt = RangeFor->getEndStmt())
2162 EmitStmt(EndStmt);
2163 if (const DeclStmt *LoopVarStmt = RangeFor->getLoopVarStmt())
2164 EmitStmt(LoopVarStmt);
2165 BodyStmt = RangeFor->getBody();
2166 } else
2167 llvm_unreachable("Expected for-stmt or range-based for-stmt");
2168
2169 // Emit closure for later use. By-value captures will be captured here.
2170 const CapturedStmt *DistanceFunc = S->getDistanceFunc();
2171 EmittedClosureTy DistanceClosure = emitCapturedStmtFunc(*this, DistanceFunc);
2172 const CapturedStmt *LoopVarFunc = S->getLoopVarFunc();
2173 EmittedClosureTy LoopVarClosure = emitCapturedStmtFunc(*this, LoopVarFunc);
2174
2175 // Call the distance function to get the number of iterations of the loop to
2176 // come.
2177 QualType LogicalTy = DistanceFunc->getCapturedDecl()
2178 ->getParam(0)
2179 ->getType()
2181 RawAddress CountAddr = CreateMemTemp(LogicalTy, ".count.addr");
2182 emitCapturedStmtCall(*this, DistanceClosure, {CountAddr.getPointer()});
2183 llvm::Value *DistVal = Builder.CreateLoad(CountAddr, ".count");
2184
2185 // Emit the loop structure.
2186 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
2187 auto BodyGen = [&, this](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP,
2188 llvm::Value *IndVar) {
2189 Builder.restoreIP(CodeGenIP);
2190
2191 // Emit the loop body: Convert the logical iteration number to the loop
2192 // variable and emit the body.
2193 const DeclRefExpr *LoopVarRef = S->getLoopVarRef();
2194 LValue LCVal = EmitLValue(LoopVarRef);
2195 Address LoopVarAddress = LCVal.getAddress();
2196 emitCapturedStmtCall(*this, LoopVarClosure,
2197 {LoopVarAddress.emitRawPointer(*this), IndVar});
2198
2199 RunCleanupsScope BodyScope(*this);
2200 EmitStmt(BodyStmt);
2201 return llvm::Error::success();
2202 };
2203
2204 llvm::CanonicalLoopInfo *CL =
2205 cantFail(OMPBuilder.createCanonicalLoop(Builder, BodyGen, DistVal));
2206
2207 // Finish up the loop.
2208 Builder.restoreIP(CL->getAfterIP());
2209 ForScope.ForceCleanup();
2210
2211 // Remember the CanonicalLoopInfo for parent AST nodes consuming it.
2212 OMPLoopNestStack.push_back(CL);
2213}
2214
2216 const OMPExecutableDirective &S, bool RequiresCleanup, const Expr *LoopCond,
2217 const Expr *IncExpr,
2218 const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
2219 const llvm::function_ref<void(CodeGenFunction &)> PostIncGen) {
2220 auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
2221
2222 // Start the loop with a block that tests the condition.
2223 auto CondBlock = createBasicBlock("omp.inner.for.cond");
2224 EmitBlock(CondBlock);
2225 const SourceRange R = S.getSourceRange();
2226
2227 // If attributes are attached, push to the basic block with them.
2228 const auto &OMPED = cast<OMPExecutableDirective>(S);
2229 const CapturedStmt *ICS = OMPED.getInnermostCapturedStmt();
2230 const Stmt *SS = ICS->getCapturedStmt();
2231 const AttributedStmt *AS = dyn_cast_or_null<AttributedStmt>(SS);
2232 OMPLoopNestStack.clear();
2233 if (AS)
2234 LoopStack.push(CondBlock, CGM.getContext(), CGM.getCodeGenOpts(),
2237 else
2238 LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
2240
2241 // If there are any cleanups between here and the loop-exit scope,
2242 // create a block to stage a loop exit along.
2243 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
2244 if (RequiresCleanup)
2245 ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup");
2246
2247 llvm::BasicBlock *LoopBody = createBasicBlock("omp.inner.for.body");
2248
2249 // Emit condition.
2250 EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
2251 if (ExitBlock != LoopExit.getBlock()) {
2252 EmitBlock(ExitBlock);
2254 }
2255
2256 EmitBlock(LoopBody);
2258
2259 // Create a block for the increment.
2260 JumpDest Continue = getJumpDestInCurrentScope("omp.inner.for.inc");
2261 BreakContinueStack.push_back(BreakContinue(S, LoopExit, Continue));
2262
2263 BodyGen(*this);
2264
2265 // Emit "IV = IV + 1" and a back-edge to the condition block.
2266 EmitBlock(Continue.getBlock());
2267 EmitIgnoredExpr(IncExpr);
2268 PostIncGen(*this);
2269 BreakContinueStack.pop_back();
2270 EmitBranch(CondBlock);
2271 LoopStack.pop();
2272 // Emit the fall-through block.
2273 EmitBlock(LoopExit.getBlock());
2274}
2275
2277 if (!HaveInsertPoint())
2278 return false;
2279 // Emit inits for the linear variables.
2280 bool HasLinears = false;
2281 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2282 for (const Expr *Init : C->inits()) {
2283 HasLinears = true;
2284 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
2285 if (const auto *Ref =
2286 dyn_cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())) {
2287 AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
2288 const auto *OrigVD = cast<VarDecl>(Ref->getDecl());
2289 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
2290 CapturedStmtInfo->lookup(OrigVD) != nullptr,
2291 VD->getInit()->getType(), VK_LValue,
2292 VD->getInit()->getExprLoc());
2294 &DRE, VD,
2295 MakeAddrLValue(Emission.getAllocatedAddress(), VD->getType()),
2296 /*capturedByInit=*/false);
2297 EmitAutoVarCleanups(Emission);
2298 } else {
2299 EmitVarDecl(*VD);
2300 }
2301 }
2302 // Emit the linear steps for the linear clauses.
2303 // If a step is not constant, it is pre-calculated before the loop.
2304 if (const auto *CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
2305 if (const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
2306 EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
2307 // Emit calculation of the linear step.
2308 EmitIgnoredExpr(CS);
2309 }
2310 }
2311 return HasLinears;
2312}
2313
2315 const OMPLoopDirective &D,
2316 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
2317 if (!HaveInsertPoint())
2318 return;
2319 llvm::BasicBlock *DoneBB = nullptr;
2320 // Emit the final values of the linear variables.
2321 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2322 auto IC = C->varlist_begin();
2323 for (const Expr *F : C->finals()) {
2324 if (!DoneBB) {
2325 if (llvm::Value *Cond = CondGen(*this)) {
2326 // If the first post-update expression is found, emit conditional
2327 // block if it was requested.
2328 llvm::BasicBlock *ThenBB = createBasicBlock(".omp.linear.pu");
2329 DoneBB = createBasicBlock(".omp.linear.pu.done");
2330 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
2331 EmitBlock(ThenBB);
2332 }
2333 }
2334 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
2335 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
2336 CapturedStmtInfo->lookup(OrigVD) != nullptr,
2337 (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
2338 Address OrigAddr = EmitLValue(&DRE).getAddress();
2339 CodeGenFunction::OMPPrivateScope VarScope(*this);
2340 VarScope.addPrivate(OrigVD, OrigAddr);
2341 (void)VarScope.Privatize();
2342 EmitIgnoredExpr(F);
2343 ++IC;
2344 }
2345 if (const Expr *PostUpdate = C->getPostUpdateExpr())
2346 EmitIgnoredExpr(PostUpdate);
2347 }
2348 if (DoneBB)
2349 EmitBlock(DoneBB, /*IsFinished=*/true);
2350}
2351
2353 const OMPExecutableDirective &D) {
2354 if (!CGF.HaveInsertPoint())
2355 return;
2356 for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
2357 llvm::APInt ClauseAlignment(64, 0);
2358 if (const Expr *AlignmentExpr = Clause->getAlignment()) {
2359 auto *AlignmentCI =
2360 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
2361 ClauseAlignment = AlignmentCI->getValue();
2362 }
2363 for (const Expr *E : Clause->varlist()) {
2364 llvm::APInt Alignment(ClauseAlignment);
2365 if (Alignment == 0) {
2366 // OpenMP [2.8.1, Description]
2367 // If no optional parameter is specified, implementation-defined default
2368 // alignments for SIMD instructions on the target platforms are assumed.
2369 Alignment =
2370 CGF.getContext()
2372 E->getType()->getPointeeType()))
2373 .getQuantity();
2374 }
2375 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
2376 "alignment is not power of 2");
2377 if (Alignment != 0) {
2378 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
2380 PtrValue, E, /*No second loc needed*/ SourceLocation(),
2381 llvm::ConstantInt::get(CGF.getLLVMContext(), Alignment));
2382 }
2383 }
2384 }
2385}
2386
2389 if (!HaveInsertPoint())
2390 return;
2391 auto I = S.private_counters().begin();
2392 for (const Expr *E : S.counters()) {
2393 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2394 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
2395 // Emit var without initialization.
2396 AutoVarEmission VarEmission = EmitAutoVarAlloca(*PrivateVD);
2397 EmitAutoVarCleanups(VarEmission);
2398 LocalDeclMap.erase(PrivateVD);
2399 (void)LoopScope.addPrivate(VD, VarEmission.getAllocatedAddress());
2400 if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
2401 VD->hasGlobalStorage()) {
2402 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD),
2403 LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
2404 E->getType(), VK_LValue, E->getExprLoc());
2405 (void)LoopScope.addPrivate(PrivateVD, EmitLValue(&DRE).getAddress());
2406 } else {
2407 (void)LoopScope.addPrivate(PrivateVD, VarEmission.getAllocatedAddress());
2408 }
2409 ++I;
2410 }
2411 // Privatize extra loop counters used in loops for ordered(n) clauses.
2412 for (const auto *C : S.getClausesOfKind<OMPOrderedClause>()) {
2413 if (!C->getNumForLoops())
2414 continue;
2415 for (unsigned I = S.getLoopsNumber(), E = C->getLoopNumIterations().size();
2416 I < E; ++I) {
2417 const auto *DRE = cast<DeclRefExpr>(C->getLoopCounter(I));
2418 const auto *VD = cast<VarDecl>(DRE->getDecl());
2419 // Override only those variables that can be captured to avoid re-emission
2420 // of the variables declared within the loops.
2421 if (DRE->refersToEnclosingVariableOrCapture()) {
2422 (void)LoopScope.addPrivate(
2423 VD, CreateMemTemp(DRE->getType(), VD->getName()));
2424 }
2425 }
2426 }
2427}
2428
2430 const Expr *Cond, llvm::BasicBlock *TrueBlock,
2431 llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
2432 if (!CGF.HaveInsertPoint())
2433 return;
2434 {
2435 CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
2436 CGF.EmitOMPPrivateLoopCounters(S, PreCondScope);
2437 (void)PreCondScope.Privatize();
2438 // Get initial values of real counters.
2439 for (const Expr *I : S.inits()) {
2440 CGF.EmitIgnoredExpr(I);
2441 }
2442 }
2443 // Create temp loop control variables with their init values to support
2444 // non-rectangular loops.
2445 CodeGenFunction::OMPMapVars PreCondVars;
2446 for (const Expr *E : S.dependent_counters()) {
2447 if (!E)
2448 continue;
2449 assert(!E->getType().getNonReferenceType()->isRecordType() &&
2450 "dependent counter must not be an iterator.");
2451 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2452 Address CounterAddr =
2454 (void)PreCondVars.setVarAddr(CGF, VD, CounterAddr);
2455 }
2456 (void)PreCondVars.apply(CGF);
2457 for (const Expr *E : S.dependent_inits()) {
2458 if (!E)
2459 continue;
2460 CGF.EmitIgnoredExpr(E);
2461 }
2462 // Check that loop is executed at least one time.
2463 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
2464 PreCondVars.restore(CGF);
2465}
2466
2468 const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
2469 if (!HaveInsertPoint())
2470 return;
2471 llvm::DenseSet<const VarDecl *> SIMDLCVs;
2473 if (isOpenMPSimdDirective(EKind)) {
2474 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
2475 for (const Expr *C : LoopDirective->counters()) {
2476 SIMDLCVs.insert(
2478 }
2479 }
2480 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2481 auto CurPrivate = C->privates().begin();
2482 for (const Expr *E : C->varlist()) {
2483 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2484 const auto *PrivateVD =
2485 cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
2486 if (!SIMDLCVs.count(VD->getCanonicalDecl())) {
2487 // Emit private VarDecl with copy init.
2488 EmitVarDecl(*PrivateVD);
2489 bool IsRegistered =
2490 PrivateScope.addPrivate(VD, GetAddrOfLocalVar(PrivateVD));
2491 assert(IsRegistered && "linear var already registered as private");
2492 // Silence the warning about unused variable.
2493 (void)IsRegistered;
2494 } else {
2495 EmitVarDecl(*PrivateVD);
2496 }
2497 ++CurPrivate;
2498 }
2499 }
2500}
2501
2503 const OMPExecutableDirective &D) {
2504 if (!CGF.HaveInsertPoint())
2505 return;
2506 if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
2507 RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
2508 /*ignoreResult=*/true);
2509 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
2510 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
2511 // In presence of finite 'safelen', it may be unsafe to mark all
2512 // the memory instructions parallel, because loop-carried
2513 // dependences of 'safelen' iterations are possible.
2514 CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>());
2515 } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
2516 RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
2517 /*ignoreResult=*/true);
2518 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
2519 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
2520 // In presence of finite 'safelen', it may be unsafe to mark all
2521 // the memory instructions parallel, because loop-carried
2522 // dependences of 'safelen' iterations are possible.
2523 CGF.LoopStack.setParallel(/*Enable=*/false);
2524 }
2525}
2526
2527// Check for the presence of an `OMPOrderedDirective`,
2528// i.e., `ordered` in `#pragma omp ordered simd`.
2529//
2530// Consider the following source code:
2531// ```
2532// __attribute__((noinline)) void omp_simd_loop(float X[ARRAY_SIZE][ARRAY_SIZE])
2533// {
2534// for (int r = 1; r < ARRAY_SIZE; ++r) {
2535// for (int c = 1; c < ARRAY_SIZE; ++c) {
2536// #pragma omp simd
2537// for (int k = 2; k < ARRAY_SIZE; ++k) {
2538// #pragma omp ordered simd
2539// X[r][k] = X[r][k - 2] + sinf((float)(r / c));
2540// }
2541// }
2542// }
2543// }
2544// ```
2545//
2546// Suppose we are in `CodeGenFunction::EmitOMPSimdInit(const OMPLoopDirective
2547// &D)`. By examining `D.dump()` we have the following AST containing
2548// `OMPOrderedDirective`:
2549//
2550// ```
2551// OMPSimdDirective 0x1c32950
2552// `-CapturedStmt 0x1c32028
2553// |-CapturedDecl 0x1c310e8
2554// | |-ForStmt 0x1c31e30
2555// | | |-DeclStmt 0x1c31298
2556// | | | `-VarDecl 0x1c31208 used k 'int' cinit
2557// | | | `-IntegerLiteral 0x1c31278 'int' 2
2558// | | |-<<<NULL>>>
2559// | | |-BinaryOperator 0x1c31308 'int' '<'
2560// | | | |-ImplicitCastExpr 0x1c312f0 'int' <LValueToRValue>
2561// | | | | `-DeclRefExpr 0x1c312b0 'int' lvalue Var 0x1c31208 'k' 'int'
2562// | | | `-IntegerLiteral 0x1c312d0 'int' 256
2563// | | |-UnaryOperator 0x1c31348 'int' prefix '++'
2564// | | | `-DeclRefExpr 0x1c31328 'int' lvalue Var 0x1c31208 'k' 'int'
2565// | | `-CompoundStmt 0x1c31e18
2566// | | `-OMPOrderedDirective 0x1c31dd8
2567// | | |-OMPSimdClause 0x1c31380
2568// | | `-CapturedStmt 0x1c31cd0
2569// ```
2570//
2571// Note the presence of `OMPOrderedDirective` above:
2572// It's (transitively) nested in a `CapturedStmt` representing the pragma
2573// annotated compound statement. Thus, we need to consider this nesting and
2574// include checking the `getCapturedStmt` in this case.
2575static bool hasOrderedDirective(const Stmt *S) {
2577 return true;
2578
2579 if (const auto *CS = dyn_cast<CapturedStmt>(S))
2581
2582 for (const Stmt *Child : S->children()) {
2583 if (Child && hasOrderedDirective(Child))
2584 return true;
2585 }
2586
2587 return false;
2588}
2589
2590static void applyConservativeSimdOrderedDirective(const Stmt &AssociatedStmt,
2592 // Check for the presence of an `OMPOrderedDirective`
2593 // i.e., `ordered` in `#pragma omp ordered simd`
2594 bool HasOrderedDirective = hasOrderedDirective(&AssociatedStmt);
2595 // If present then conservatively disable loop vectorization
2596 // analogously to how `emitSimdlenSafelenClause` does.
2597 if (HasOrderedDirective)
2598 LoopStack.setParallel(/*Enable=*/false);
2599}
2600
2602 // Walk clauses and process safelen/lastprivate.
2603 LoopStack.setParallel(/*Enable=*/true);
2604 LoopStack.setVectorizeEnable();
2605 const Stmt *AssociatedStmt = D.getAssociatedStmt();
2607 emitSimdlenSafelenClause(*this, D);
2608 if (const auto *C = D.getSingleClause<OMPOrderClause>())
2609 if (C->getKind() == OMPC_ORDER_concurrent)
2610 LoopStack.setParallel(/*Enable=*/true);
2612 if ((EKind == OMPD_simd ||
2613 (getLangOpts().OpenMPSimd && isOpenMPSimdDirective(EKind))) &&
2614 llvm::any_of(D.getClausesOfKind<OMPReductionClause>(),
2615 [](const OMPReductionClause *C) {
2616 return C->getModifier() == OMPC_REDUCTION_inscan;
2617 }))
2618 // Disable parallel access in case of prefix sum.
2619 LoopStack.setParallel(/*Enable=*/false);
2620}
2621
2623 const OMPLoopDirective &D,
2624 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
2625 if (!HaveInsertPoint())
2626 return;
2627 llvm::BasicBlock *DoneBB = nullptr;
2628 auto IC = D.counters().begin();
2629 auto IPC = D.private_counters().begin();
2630 for (const Expr *F : D.finals()) {
2631 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
2632 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
2633 const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
2634 if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
2635 OrigVD->hasGlobalStorage() || CED) {
2636 if (!DoneBB) {
2637 if (llvm::Value *Cond = CondGen(*this)) {
2638 // If the first post-update expression is found, emit conditional
2639 // block if it was requested.
2640 llvm::BasicBlock *ThenBB = createBasicBlock(".omp.final.then");
2641 DoneBB = createBasicBlock(".omp.final.done");
2642 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
2643 EmitBlock(ThenBB);
2644 }
2645 }
2646 Address OrigAddr = Address::invalid();
2647 if (CED) {
2648 OrigAddr = EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress();
2649 } else {
2650 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(PrivateVD),
2651 /*RefersToEnclosingVariableOrCapture=*/false,
2652 (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());
2653 OrigAddr = EmitLValue(&DRE).getAddress();
2654 }
2655 OMPPrivateScope VarScope(*this);
2656 VarScope.addPrivate(OrigVD, OrigAddr);
2657 (void)VarScope.Privatize();
2658 EmitIgnoredExpr(F);
2659 }
2660 ++IC;
2661 ++IPC;
2662 }
2663 if (DoneBB)
2664 EmitBlock(DoneBB, /*IsFinished=*/true);
2665}
2666
2673
2674/// Emit a helper variable and return corresponding lvalue.
2676 const DeclRefExpr *Helper) {
2677 auto VDecl = cast<VarDecl>(Helper->getDecl());
2678 CGF.EmitVarDecl(*VDecl);
2679 return CGF.EmitLValue(Helper);
2680}
2681
2683 const RegionCodeGenTy &SimdInitGen,
2684 const RegionCodeGenTy &BodyCodeGen) {
2685 auto &&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](CodeGenFunction &CGF,
2686 PrePostActionTy &) {
2687 CGOpenMPRuntime::NontemporalDeclsRAII NontemporalsRegion(CGF.CGM, S);
2689 SimdInitGen(CGF);
2690
2691 BodyCodeGen(CGF);
2692 };
2693 auto &&ElseGen = [&BodyCodeGen](CodeGenFunction &CGF, PrePostActionTy &) {
2695 CGF.LoopStack.setVectorizeEnable(/*Enable=*/false);
2696
2697 BodyCodeGen(CGF);
2698 };
2699 const Expr *IfCond = nullptr;
2701 if (isOpenMPSimdDirective(EKind)) {
2702 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
2703 if (CGF.getLangOpts().OpenMP >= 50 &&
2704 (C->getNameModifier() == OMPD_unknown ||
2705 C->getNameModifier() == OMPD_simd)) {
2706 IfCond = C->getCondition();
2707 break;
2708 }
2709 }
2710 }
2711 if (IfCond) {
2712 CGF.CGM.getOpenMPRuntime().emitIfClause(CGF, IfCond, ThenGen, ElseGen);
2713 } else {
2714 RegionCodeGenTy ThenRCG(ThenGen);
2715 ThenRCG(CGF);
2716 }
2717}
2718
2720 PrePostActionTy &Action) {
2721 Action.Enter(CGF);
2722 OMPLoopScope PreInitScope(CGF, S);
2723 // if (PreCond) {
2724 // for (IV in 0..LastIteration) BODY;
2725 // <Final counter/linear vars updates>;
2726 // }
2727
2728 // The presence of lower/upper bound variable depends on the actual directive
2729 // kind in the AST node. The variables must be emitted because some of the
2730 // expressions associated with the loop will use them.
2731 OpenMPDirectiveKind DKind = S.getDirectiveKind();
2732 if (isOpenMPDistributeDirective(DKind) ||
2735 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()));
2736 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()));
2737 }
2738
2740 // Emit: if (PreCond) - begin.
2741 // If the condition constant folds and can be elided, avoid emitting the
2742 // whole loop.
2743 bool CondConstant;
2744 llvm::BasicBlock *ContBlock = nullptr;
2745 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
2746 if (!CondConstant)
2747 return;
2748 } else {
2749 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then");
2750 ContBlock = CGF.createBasicBlock("simd.if.end");
2751 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
2752 CGF.getProfileCount(&S));
2753 CGF.EmitBlock(ThenBlock);
2755 }
2756
2757 // Emit the loop iteration variable.
2758 const Expr *IVExpr = S.getIterationVariable();
2759 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
2760 CGF.EmitVarDecl(*IVDecl);
2761 CGF.EmitIgnoredExpr(S.getInit());
2762
2763 // Emit the iterations count variable.
2764 // If it is not a variable, Sema decided to calculate iterations count on
2765 // each iteration (e.g., it is foldable into a constant).
2766 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
2767 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2768 // Emit calculation of the iterations count.
2769 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
2770 }
2771
2772 emitAlignedClause(CGF, S);
2773 (void)CGF.EmitOMPLinearClauseInit(S);
2774 {
2775 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
2776 CGF.EmitOMPPrivateClause(S, LoopScope);
2777 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
2778 CGF.EmitOMPLinearClause(S, LoopScope);
2779 CGF.EmitOMPReductionClauseInit(S, LoopScope);
2781 CGF, S, CGF.EmitLValue(S.getIterationVariable()));
2782 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
2783 (void)LoopScope.Privatize();
2786
2788 CGF, S,
2789 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2790 CGF.EmitOMPSimdInit(S);
2791 },
2792 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
2793 CGF.EmitOMPInnerLoop(
2794 S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(),
2795 [&S](CodeGenFunction &CGF) {
2796 emitOMPLoopBodyWithStopPoint(CGF, S,
2797 CodeGenFunction::JumpDest());
2798 },
2799 [](CodeGenFunction &) {});
2800 });
2801 CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; });
2802 // Emit final copy of the lastprivate variables at the end of loops.
2803 if (HasLastprivateClause)
2804 CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true);
2805 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd);
2807 [](CodeGenFunction &) { return nullptr; });
2808 LoopScope.restoreMap();
2809 CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; });
2810 }
2811 // Emit: if (PreCond) - end.
2812 if (ContBlock) {
2813 CGF.EmitBranch(ContBlock);
2814 CGF.EmitBlock(ContBlock, true);
2815 }
2816}
2817
2818// Pass OMPLoopDirective (instead of OMPSimdDirective) to make this function
2819// available for "loop bind(thread)", which maps to "simd".
2821 // Check for unsupported clauses
2822 for (OMPClause *C : S.clauses()) {
2823 // Currently only order, simdlen and safelen clauses are supported
2826 return false;
2827 }
2828
2829 // Check if we have a statement with the ordered directive.
2830 // Visit the statement hierarchy to find a compound statement
2831 // with a ordered directive in it.
2832 if (const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(S.getRawStmt())) {
2833 if (const Stmt *SyntacticalLoop = CanonLoop->getLoopStmt()) {
2834 for (const Stmt *SubStmt : SyntacticalLoop->children()) {
2835 if (!SubStmt)
2836 continue;
2837 if (const CompoundStmt *CS = dyn_cast<CompoundStmt>(SubStmt)) {
2838 for (const Stmt *CSSubStmt : CS->children()) {
2839 if (!CSSubStmt)
2840 continue;
2841 if (isa<OMPOrderedDirective>(CSSubStmt)) {
2842 return false;
2843 }
2844 }
2845 }
2846 }
2847 }
2848 }
2849 return true;
2850}
2851
2852static llvm::MapVector<llvm::Value *, llvm::Value *>
2854 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars;
2855 for (const auto *Clause : S.getClausesOfKind<OMPAlignedClause>()) {
2856 llvm::APInt ClauseAlignment(64, 0);
2857 if (const Expr *AlignmentExpr = Clause->getAlignment()) {
2858 auto *AlignmentCI =
2859 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
2860 ClauseAlignment = AlignmentCI->getValue();
2861 }
2862 for (const Expr *E : Clause->varlist()) {
2863 llvm::APInt Alignment(ClauseAlignment);
2864 if (Alignment == 0) {
2865 // OpenMP [2.8.1, Description]
2866 // If no optional parameter is specified, implementation-defined default
2867 // alignments for SIMD instructions on the target platforms are assumed.
2868 Alignment =
2869 CGF.getContext()
2871 E->getType()->getPointeeType()))
2872 .getQuantity();
2873 }
2874 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
2875 "alignment is not power of 2");
2876 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
2877 AlignedVars[PtrValue] = CGF.Builder.getInt64(Alignment.getSExtValue());
2878 }
2879 }
2880 return AlignedVars;
2881}
2882
2883// Pass OMPLoopDirective (instead of OMPSimdDirective) to make this function
2884// available for "loop bind(thread)", which maps to "simd".
2887 bool UseOMPIRBuilder =
2888 CGM.getLangOpts().OpenMPIRBuilder && isSimdSupportedByOpenMPIRBuilder(S);
2889 if (UseOMPIRBuilder) {
2890 auto &&CodeGenIRBuilder = [&S, &CGM, UseOMPIRBuilder](CodeGenFunction &CGF,
2891 PrePostActionTy &) {
2892 // Use the OpenMPIRBuilder if enabled.
2893 if (UseOMPIRBuilder) {
2894 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars =
2895 GetAlignedMapping(S, CGF);
2896 // Emit the associated statement and get its loop representation.
2897 const Stmt *Inner = S.getRawStmt();
2898 llvm::CanonicalLoopInfo *CLI =
2899 CGF.EmitOMPCollapsedCanonicalLoopNest(Inner, 1);
2900
2901 llvm::OpenMPIRBuilder &OMPBuilder =
2903 // Add SIMD specific metadata
2904 llvm::ConstantInt *Simdlen = nullptr;
2905 if (const auto *C = S.getSingleClause<OMPSimdlenClause>()) {
2906 RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
2907 /*ignoreResult=*/true);
2908 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
2909 Simdlen = Val;
2910 }
2911 llvm::ConstantInt *Safelen = nullptr;
2912 if (const auto *C = S.getSingleClause<OMPSafelenClause>()) {
2913 RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
2914 /*ignoreResult=*/true);
2915 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
2916 Safelen = Val;
2917 }
2918 llvm::omp::OrderKind Order = llvm::omp::OrderKind::OMP_ORDER_unknown;
2919 if (const auto *C = S.getSingleClause<OMPOrderClause>()) {
2920 if (C->getKind() == OpenMPOrderClauseKind::OMPC_ORDER_concurrent) {
2921 Order = llvm::omp::OrderKind::OMP_ORDER_concurrent;
2922 }
2923 }
2924 // Add simd metadata to the collapsed loop. Do not generate
2925 // another loop for if clause. Support for if clause is done earlier.
2926 OMPBuilder.applySimd(CLI, AlignedVars,
2927 /*IfCond*/ nullptr, Order, Simdlen, Safelen);
2928 return;
2929 }
2930 };
2931 {
2932 auto LPCRegion =
2934 OMPLexicalScope Scope(CGF, S, OMPD_unknown);
2935 CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
2936 CodeGenIRBuilder);
2937 }
2938 return;
2939 }
2940
2942 CGF.OMPFirstScanLoop = true;
2943 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2944 emitOMPSimdRegion(CGF, S, Action);
2945 };
2946 {
2947 auto LPCRegion =
2949 OMPLexicalScope Scope(CGF, S, OMPD_unknown);
2951 }
2952 // Check for outer lastprivate conditional update.
2954}
2955
2956void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) {
2957 emitOMPSimdDirective(S, *this, CGM);
2958}
2959
2961 // Emit the de-sugared statement.
2962 OMPTransformDirectiveScopeRAII TileScope(*this, &S);
2964}
2965
2967 // Emit the de-sugared statement.
2968 OMPTransformDirectiveScopeRAII StripeScope(*this, &S);
2970}
2971
2973 // Emit the de-sugared statement.
2974 OMPTransformDirectiveScopeRAII ReverseScope(*this, &S);
2976}
2977
2979 const OMPInterchangeDirective &S) {
2980 // Emit the de-sugared statement.
2981 OMPTransformDirectiveScopeRAII InterchangeScope(*this, &S);
2983}
2984
2986 // Emit the de-sugared statement
2987 OMPTransformDirectiveScopeRAII FuseScope(*this, &S);
2989}
2990
2992 bool UseOMPIRBuilder = CGM.getLangOpts().OpenMPIRBuilder;
2993
2994 if (UseOMPIRBuilder) {
2995 auto DL = SourceLocToDebugLoc(S.getBeginLoc());
2996 const Stmt *Inner = S.getRawStmt();
2997
2998 // Consume nested loop. Clear the entire remaining loop stack because a
2999 // fully unrolled loop is non-transformable. For partial unrolling the
3000 // generated outer loop is pushed back to the stack.
3001 llvm::CanonicalLoopInfo *CLI = EmitOMPCollapsedCanonicalLoopNest(Inner, 1);
3002 OMPLoopNestStack.clear();
3003
3004 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
3005
3006 bool NeedsUnrolledCLI = ExpectedOMPLoopDepth >= 1;
3007 llvm::CanonicalLoopInfo *UnrolledCLI = nullptr;
3008
3009 if (S.hasClausesOfKind<OMPFullClause>()) {
3010 assert(ExpectedOMPLoopDepth == 0);
3011 OMPBuilder.unrollLoopFull(DL, CLI);
3012 } else if (auto *PartialClause = S.getSingleClause<OMPPartialClause>()) {
3013 uint64_t Factor = 0;
3014 if (Expr *FactorExpr = PartialClause->getFactor()) {
3015 Factor = FactorExpr->EvaluateKnownConstInt(getContext()).getZExtValue();
3016 assert(Factor >= 1 && "Only positive factors are valid");
3017 }
3018 OMPBuilder.unrollLoopPartial(DL, CLI, Factor,
3019 NeedsUnrolledCLI ? &UnrolledCLI : nullptr);
3020 } else {
3021 OMPBuilder.unrollLoopHeuristic(DL, CLI);
3022 }
3023
3024 assert((!NeedsUnrolledCLI || UnrolledCLI) &&
3025 "NeedsUnrolledCLI implies UnrolledCLI to be set");
3026 if (UnrolledCLI)
3027 OMPLoopNestStack.push_back(UnrolledCLI);
3028
3029 return;
3030 }
3031
3032 // This function is only called if the unrolled loop is not consumed by any
3033 // other loop-associated construct. Such a loop-associated construct will have
3034 // used the transformed AST.
3035
3036 // Set the unroll metadata for the next emitted loop.
3037 LoopStack.setUnrollState(LoopAttributes::Enable);
3038
3039 if (S.hasClausesOfKind<OMPFullClause>()) {
3040 LoopStack.setUnrollState(LoopAttributes::Full);
3041 } else if (auto *PartialClause = S.getSingleClause<OMPPartialClause>()) {
3042 if (Expr *FactorExpr = PartialClause->getFactor()) {
3043 uint64_t Factor =
3044 FactorExpr->EvaluateKnownConstInt(getContext()).getZExtValue();
3045 assert(Factor >= 1 && "Only positive factors are valid");
3046 LoopStack.setUnrollCount(Factor);
3047 }
3048 }
3049
3050 EmitStmt(S.getAssociatedStmt());
3051}
3052
3053void CodeGenFunction::EmitOMPOuterLoop(
3054 bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
3056 const CodeGenFunction::OMPLoopArguments &LoopArgs,
3057 const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,
3058 const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) {
3060
3061 const Expr *IVExpr = S.getIterationVariable();
3062 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3063 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3064
3065 JumpDest LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");
3066
3067 // Start the loop with a block that tests the condition.
3068 llvm::BasicBlock *CondBlock = createBasicBlock("omp.dispatch.cond");
3069 EmitBlock(CondBlock);
3070 const SourceRange R = S.getSourceRange();
3071 OMPLoopNestStack.clear();
3074
3075 llvm::Value *BoolCondVal = nullptr;
3076 if (!DynamicOrOrdered) {
3077 // UB = min(UB, GlobalUB) or
3078 // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.
3079 // 'distribute parallel for')
3080 EmitIgnoredExpr(LoopArgs.EUB);
3081 // IV = LB
3082 EmitIgnoredExpr(LoopArgs.Init);
3083 // IV < UB
3084 BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);
3085 } else {
3086 BoolCondVal =
3087 RT.emitForNext(*this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,
3088 LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
3089 }
3090
3091 // If there are any cleanups between here and the loop-exit scope,
3092 // create a block to stage a loop exit along.
3093 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
3094 if (LoopScope.requiresCleanups())
3095 ExitBlock = createBasicBlock("omp.dispatch.cleanup");
3096
3097 llvm::BasicBlock *LoopBody = createBasicBlock("omp.dispatch.body");
3098 Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
3099 if (ExitBlock != LoopExit.getBlock()) {
3100 EmitBlock(ExitBlock);
3102 }
3103 EmitBlock(LoopBody);
3104
3105 // Emit "IV = LB" (in case of static schedule, we have already calculated new
3106 // LB for loop condition and emitted it above).
3107 if (DynamicOrOrdered)
3108 EmitIgnoredExpr(LoopArgs.Init);
3109
3110 // Create a block for the increment.
3111 JumpDest Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
3112 BreakContinueStack.push_back(BreakContinue(S, LoopExit, Continue));
3113
3116 *this, S,
3117 [&S, IsMonotonic, EKind](CodeGenFunction &CGF, PrePostActionTy &) {
3118 // Generate !llvm.loop.parallel metadata for loads and stores for loops
3119 // with dynamic/guided scheduling and without ordered clause.
3120 if (!isOpenMPSimdDirective(EKind)) {
3121 CGF.LoopStack.setParallel(!IsMonotonic);
3122 if (const auto *C = S.getSingleClause<OMPOrderClause>())
3123 if (C->getKind() == OMPC_ORDER_concurrent)
3124 CGF.LoopStack.setParallel(/*Enable=*/true);
3125 } else {
3126 CGF.EmitOMPSimdInit(S);
3127 }
3128 },
3129 [&S, &LoopArgs, LoopExit, &CodeGenLoop, IVSize, IVSigned, &CodeGenOrdered,
3130 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
3131 SourceLocation Loc = S.getBeginLoc();
3132 // when 'distribute' is not combined with a 'for':
3133 // while (idx <= UB) { BODY; ++idx; }
3134 // when 'distribute' is combined with a 'for'
3135 // (e.g. 'distribute parallel for')
3136 // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
3137 CGF.EmitOMPInnerLoop(
3138 S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
3139 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
3140 CodeGenLoop(CGF, S, LoopExit);
3141 },
3142 [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
3143 CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
3144 });
3145 });
3146
3147 EmitBlock(Continue.getBlock());
3148 BreakContinueStack.pop_back();
3149 if (!DynamicOrOrdered) {
3150 // Emit "LB = LB + Stride", "UB = UB + Stride".
3151 EmitIgnoredExpr(LoopArgs.NextLB);
3152 EmitIgnoredExpr(LoopArgs.NextUB);
3153 }
3154
3155 EmitBranch(CondBlock);
3156 OMPLoopNestStack.clear();
3157 LoopStack.pop();
3158 // Emit the fall-through block.
3159 EmitBlock(LoopExit.getBlock());
3160
3161 // Tell the runtime we are done.
3162 auto &&CodeGen = [DynamicOrOrdered, &S, &LoopArgs](CodeGenFunction &CGF) {
3163 if (!DynamicOrOrdered)
3164 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
3165 LoopArgs.DKind);
3166 };
3167 OMPCancelStack.emitExit(*this, EKind, CodeGen);
3168}
3169
3170void CodeGenFunction::EmitOMPForOuterLoop(
3171 const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,
3172 const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
3173 const OMPLoopArguments &LoopArgs,
3174 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
3175 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
3176
3177 // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
3178 const bool DynamicOrOrdered = Ordered || RT.isDynamic(ScheduleKind.Schedule);
3179
3180 assert((Ordered || !RT.isStaticNonchunked(ScheduleKind.Schedule,
3181 LoopArgs.Chunk != nullptr)) &&
3182 "static non-chunked schedule does not need outer loop");
3183
3184 // Emit outer loop.
3185 //
3186 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
3187 // When schedule(dynamic,chunk_size) is specified, the iterations are
3188 // distributed to threads in the team in chunks as the threads request them.
3189 // Each thread executes a chunk of iterations, then requests another chunk,
3190 // until no chunks remain to be distributed. Each chunk contains chunk_size
3191 // iterations, except for the last chunk to be distributed, which may have
3192 // fewer iterations. When no chunk_size is specified, it defaults to 1.
3193 //
3194 // When schedule(guided,chunk_size) is specified, the iterations are assigned
3195 // to threads in the team in chunks as the executing threads request them.
3196 // Each thread executes a chunk of iterations, then requests another chunk,
3197 // until no chunks remain to be assigned. For a chunk_size of 1, the size of
3198 // each chunk is proportional to the number of unassigned iterations divided
3199 // by the number of threads in the team, decreasing to 1. For a chunk_size
3200 // with value k (greater than 1), the size of each chunk is determined in the
3201 // same way, with the restriction that the chunks do not contain fewer than k
3202 // iterations (except for the last chunk to be assigned, which may have fewer
3203 // than k iterations).
3204 //
3205 // When schedule(auto) is specified, the decision regarding scheduling is
3206 // delegated to the compiler and/or runtime system. The programmer gives the
3207 // implementation the freedom to choose any possible mapping of iterations to
3208 // threads in the team.
3209 //
3210 // When schedule(runtime) is specified, the decision regarding scheduling is
3211 // deferred until run time, and the schedule and chunk size are taken from the
3212 // run-sched-var ICV. If the ICV is set to auto, the schedule is
3213 // implementation defined
3214 //
3215 // __kmpc_dispatch_init();
3216 // while(__kmpc_dispatch_next(&LB, &UB)) {
3217 // idx = LB;
3218 // while (idx <= UB) { BODY; ++idx;
3219 // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
3220 // } // inner loop
3221 // }
3222 // __kmpc_dispatch_deinit();
3223 //
3224 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
3225 // When schedule(static, chunk_size) is specified, iterations are divided into
3226 // chunks of size chunk_size, and the chunks are assigned to the threads in
3227 // the team in a round-robin fashion in the order of the thread number.
3228 //
3229 // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
3230 // while (idx <= UB) { BODY; ++idx; } // inner loop
3231 // LB = LB + ST;
3232 // UB = UB + ST;
3233 // }
3234 //
3235
3236 const Expr *IVExpr = S.getIterationVariable();
3237 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3238 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3239
3240 if (DynamicOrOrdered) {
3241 const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
3242 CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);
3243 llvm::Value *LBVal = DispatchBounds.first;
3244 llvm::Value *UBVal = DispatchBounds.second;
3245 CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
3246 LoopArgs.Chunk};
3247 RT.emitForDispatchInit(*this, S.getBeginLoc(), ScheduleKind, IVSize,
3248 IVSigned, Ordered, DipatchRTInputValues);
3249 } else {
3250 CGOpenMPRuntime::StaticRTInput StaticInit(
3251 IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
3252 LoopArgs.ST, LoopArgs.Chunk);
3254 RT.emitForStaticInit(*this, S.getBeginLoc(), EKind, ScheduleKind,
3255 StaticInit);
3256 }
3257
3258 auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
3259 const unsigned IVSize,
3260 const bool IVSigned) {
3261 if (Ordered) {
3262 CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,
3263 IVSigned);
3264 }
3265 };
3266
3267 OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
3268 LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
3269 OuterLoopArgs.IncExpr = S.getInc();
3270 OuterLoopArgs.Init = S.getInit();
3271 OuterLoopArgs.Cond = S.getCond();
3272 OuterLoopArgs.NextLB = S.getNextLowerBound();
3273 OuterLoopArgs.NextUB = S.getNextUpperBound();
3274 OuterLoopArgs.DKind = LoopArgs.DKind;
3275 EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
3276 emitOMPLoopBodyWithStopPoint, CodeGenOrdered);
3277 if (DynamicOrOrdered) {
3278 RT.emitForDispatchDeinit(*this, S.getBeginLoc());
3279 }
3280}
3281
3283 const unsigned IVSize, const bool IVSigned) {}
3284
3285void CodeGenFunction::EmitOMPDistributeOuterLoop(
3286 OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S,
3287 OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,
3288 const CodeGenLoopTy &CodeGenLoopContent) {
3289
3290 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
3291
3292 // Emit outer loop.
3293 // Same behavior as a OMPForOuterLoop, except that schedule cannot be
3294 // dynamic
3295 //
3296
3297 const Expr *IVExpr = S.getIterationVariable();
3298 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3299 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3301
3302 CGOpenMPRuntime::StaticRTInput StaticInit(
3303 IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB,
3304 LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
3305 RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind, StaticInit);
3306
3307 // for combined 'distribute' and 'for' the increment expression of distribute
3308 // is stored in DistInc. For 'distribute' alone, it is in Inc.
3309 Expr *IncExpr;
3311 IncExpr = S.getDistInc();
3312 else
3313 IncExpr = S.getInc();
3314
3315 // this routine is shared by 'omp distribute parallel for' and
3316 // 'omp distribute': select the right EUB expression depending on the
3317 // directive
3318 OMPLoopArguments OuterLoopArgs;
3319 OuterLoopArgs.LB = LoopArgs.LB;
3320 OuterLoopArgs.UB = LoopArgs.UB;
3321 OuterLoopArgs.ST = LoopArgs.ST;
3322 OuterLoopArgs.IL = LoopArgs.IL;
3323 OuterLoopArgs.Chunk = LoopArgs.Chunk;
3324 OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(EKind)
3325 ? S.getCombinedEnsureUpperBound()
3326 : S.getEnsureUpperBound();
3327 OuterLoopArgs.IncExpr = IncExpr;
3328 OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(EKind)
3329 ? S.getCombinedInit()
3330 : S.getInit();
3331 OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(EKind)
3332 ? S.getCombinedCond()
3333 : S.getCond();
3334 OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(EKind)
3335 ? S.getCombinedNextLowerBound()
3336 : S.getNextLowerBound();
3337 OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(EKind)
3338 ? S.getCombinedNextUpperBound()
3339 : S.getNextUpperBound();
3340 OuterLoopArgs.DKind = OMPD_distribute;
3341
3342 EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S,
3343 LoopScope, OuterLoopArgs, CodeGenLoopContent,
3345}
3346
3347static std::pair<LValue, LValue>
3349 const OMPExecutableDirective &S) {
3351 LValue LB =
3352 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
3353 LValue UB =
3354 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
3355
3356 // When composing 'distribute' with 'for' (e.g. as in 'distribute
3357 // parallel for') we need to use the 'distribute'
3358 // chunk lower and upper bounds rather than the whole loop iteration
3359 // space. These are parameters to the outlined function for 'parallel'
3360 // and we copy the bounds of the previous schedule into the
3361 // the current ones.
3362 LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable());
3363 LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable());
3364 llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(
3365 PrevLB, LS.getPrevLowerBoundVariable()->getExprLoc());
3366 PrevLBVal = CGF.EmitScalarConversion(
3367 PrevLBVal, LS.getPrevLowerBoundVariable()->getType(),
3368 LS.getIterationVariable()->getType(),
3369 LS.getPrevLowerBoundVariable()->getExprLoc());
3370 llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(
3371 PrevUB, LS.getPrevUpperBoundVariable()->getExprLoc());
3372 PrevUBVal = CGF.EmitScalarConversion(
3373 PrevUBVal, LS.getPrevUpperBoundVariable()->getType(),
3374 LS.getIterationVariable()->getType(),
3375 LS.getPrevUpperBoundVariable()->getExprLoc());
3376
3377 CGF.EmitStoreOfScalar(PrevLBVal, LB);
3378 CGF.EmitStoreOfScalar(PrevUBVal, UB);
3379
3380 return {LB, UB};
3381}
3382
3383/// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then
3384/// we need to use the LB and UB expressions generated by the worksharing
3385/// code generation support, whereas in non combined situations we would
3386/// just emit 0 and the LastIteration expression
3387/// This function is necessary due to the difference of the LB and UB
3388/// types for the RT emission routines for 'for_static_init' and
3389/// 'for_dispatch_init'
3390static std::pair<llvm::Value *, llvm::Value *>
3392 const OMPExecutableDirective &S,
3393 Address LB, Address UB) {
3395 const Expr *IVExpr = LS.getIterationVariable();
3396 // when implementing a dynamic schedule for a 'for' combined with a
3397 // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop
3398 // is not normalized as each team only executes its own assigned
3399 // distribute chunk
3400 QualType IteratorTy = IVExpr->getType();
3401 llvm::Value *LBVal =
3402 CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
3403 llvm::Value *UBVal =
3404 CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
3405 return {LBVal, UBVal};
3406}
3407
3411 const auto &Dir = cast<OMPLoopDirective>(S);
3412 LValue LB =
3413 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
3414 llvm::Value *LBCast = CGF.Builder.CreateIntCast(
3415 CGF.Builder.CreateLoad(LB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
3416 CapturedVars.push_back(LBCast);
3417 LValue UB =
3418 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
3419
3420 llvm::Value *UBCast = CGF.Builder.CreateIntCast(
3421 CGF.Builder.CreateLoad(UB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
3422 CapturedVars.push_back(UBCast);
3423}
3424
3425static void
3427 const OMPLoopDirective &S,
3430 auto &&CGInlinedWorksharingLoop = [&S, EKind](CodeGenFunction &CGF,
3431 PrePostActionTy &Action) {
3432 Action.Enter(CGF);
3433 bool HasCancel = false;
3434 if (!isOpenMPSimdDirective(EKind)) {
3435 if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
3436 HasCancel = D->hasCancel();
3437 else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
3438 HasCancel = D->hasCancel();
3439 else if (const auto *D =
3440 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
3441 HasCancel = D->hasCancel();
3442 }
3443 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);
3444 CGF.EmitOMPWorksharingLoop(S, S.getPrevEnsureUpperBound(),
3447 };
3448
3450 CGF, S, isOpenMPSimdDirective(EKind) ? OMPD_for_simd : OMPD_for,
3451 CGInlinedWorksharingLoop,
3453}
3454
3457 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3459 S.getDistInc());
3460 };
3461 OMPLexicalScope Scope(*this, S, OMPD_parallel);
3462 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
3463}
3464
3467 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3469 S.getDistInc());
3470 };
3471 OMPLexicalScope Scope(*this, S, OMPD_parallel);
3472 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
3473}
3474
3476 const OMPDistributeSimdDirective &S) {
3477 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3479 };
3480 OMPLexicalScope Scope(*this, S, OMPD_unknown);
3481 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
3482}
3483
3485 CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) {
3486 // Emit SPMD target parallel for region as a standalone region.
3487 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3488 emitOMPSimdRegion(CGF, S, Action);
3489 };
3490 llvm::Function *Fn;
3491 llvm::Constant *Addr;
3492 // Emit target region as a standalone region.
3493 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
3494 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
3495 assert(Fn && Addr && "Target device function emission failed.");
3496}
3497
3499 const OMPTargetSimdDirective &S) {
3500 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3501 emitOMPSimdRegion(CGF, S, Action);
3502 };
3504}
3505
3506namespace {
3507struct ScheduleKindModifiersTy {
3511 ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,
3514 : Kind(Kind), M1(M1), M2(M2) {}
3515};
3516} // namespace
3517
3519 const OMPLoopDirective &S, Expr *EUB,
3520 const CodeGenLoopBoundsTy &CodeGenLoopBounds,
3521 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
3522 // Emit the loop iteration variable.
3523 const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
3524 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
3525 EmitVarDecl(*IVDecl);
3526
3527 // Emit the iterations count variable.
3528 // If it is not a variable, Sema decided to calculate iterations count on each
3529 // iteration (e.g., it is foldable into a constant).
3530 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
3531 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
3532 // Emit calculation of the iterations count.
3533 EmitIgnoredExpr(S.getCalcLastIteration());
3534 }
3535
3536 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
3537
3538 bool HasLastprivateClause;
3539 // Check pre-condition.
3540 {
3541 OMPLoopScope PreInitScope(*this, S);
3542 // Skip the entire loop if we don't meet the precondition.
3543 // If the condition constant folds and can be elided, avoid emitting the
3544 // whole loop.
3545 bool CondConstant;
3546 llvm::BasicBlock *ContBlock = nullptr;
3547 if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
3548 if (!CondConstant)
3549 return false;
3550 } else {
3551 llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
3552 ContBlock = createBasicBlock("omp.precond.end");
3553 emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
3554 getProfileCount(&S));
3555 EmitBlock(ThenBlock);
3557 }
3558
3559 RunCleanupsScope DoacrossCleanupScope(*this);
3560 bool Ordered = false;
3561 if (const auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
3562 if (OrderedClause->getNumForLoops())
3563 RT.emitDoacrossInit(*this, S, OrderedClause->getLoopNumIterations());
3564 else
3565 Ordered = true;
3566 }
3567
3568 emitAlignedClause(*this, S);
3569 bool HasLinears = EmitOMPLinearClauseInit(S);
3570 // Emit helper vars inits.
3571
3572 std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);
3573 LValue LB = Bounds.first;
3574 LValue UB = Bounds.second;
3575 LValue ST =
3576 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
3577 LValue IL =
3578 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
3579
3580 // Emit 'then' code.
3581 {
3583 OMPPrivateScope LoopScope(*this);
3584 if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) {
3585 // Emit implicit barrier to synchronize threads and avoid data races on
3586 // initialization of firstprivate variables and post-update of
3587 // lastprivate variables.
3588 CGM.getOpenMPRuntime().emitBarrierCall(
3589 *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
3590 /*ForceSimpleCall=*/true);
3591 }
3592 EmitOMPPrivateClause(S, LoopScope);
3594 *this, S, EmitLValue(S.getIterationVariable()));
3595 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
3596 EmitOMPReductionClauseInit(S, LoopScope);
3597 EmitOMPPrivateLoopCounters(S, LoopScope);
3598 EmitOMPLinearClause(S, LoopScope);
3599 (void)LoopScope.Privatize();
3601 CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);
3602
3603 // Detect the loop schedule kind and chunk.
3604 const Expr *ChunkExpr = nullptr;
3605 OpenMPScheduleTy ScheduleKind;
3606 if (const auto *C = S.getSingleClause<OMPScheduleClause>()) {
3607 ScheduleKind.Schedule = C->getScheduleKind();
3608 ScheduleKind.M1 = C->getFirstScheduleModifier();
3609 ScheduleKind.M2 = C->getSecondScheduleModifier();
3610 ChunkExpr = C->getChunkSize();
3611 } else {
3612 // Default behaviour for schedule clause.
3613 CGM.getOpenMPRuntime().getDefaultScheduleAndChunk(
3614 *this, S, ScheduleKind.Schedule, ChunkExpr);
3615 }
3616 bool HasChunkSizeOne = false;
3617 llvm::Value *Chunk = nullptr;
3618 if (ChunkExpr) {
3619 Chunk = EmitScalarExpr(ChunkExpr);
3620 Chunk = EmitScalarConversion(Chunk, ChunkExpr->getType(),
3621 S.getIterationVariable()->getType(),
3622 S.getBeginLoc());
3624 if (ChunkExpr->EvaluateAsInt(Result, getContext())) {
3625 llvm::APSInt EvaluatedChunk = Result.Val.getInt();
3626 HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);
3627 }
3628 }
3629 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3630 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3631 // OpenMP 4.5, 2.7.1 Loop Construct, Description.
3632 // If the static schedule kind is specified or if the ordered clause is
3633 // specified, and if no monotonic modifier is specified, the effect will
3634 // be as if the monotonic modifier was specified.
3635 bool StaticChunkedOne =
3636 RT.isStaticChunked(ScheduleKind.Schedule,
3637 /* Chunked */ Chunk != nullptr) &&
3638 HasChunkSizeOne && isOpenMPLoopBoundSharingDirective(EKind);
3639 bool IsMonotonic =
3640 Ordered ||
3641 (ScheduleKind.Schedule == OMPC_SCHEDULE_static &&
3642 !(ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_nonmonotonic ||
3643 ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_nonmonotonic)) ||
3644 ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
3645 ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
3646 if ((RT.isStaticNonchunked(ScheduleKind.Schedule,
3647 /* Chunked */ Chunk != nullptr) ||
3648 StaticChunkedOne) &&
3649 !Ordered) {
3653 *this, S,
3654 [&S, EKind](CodeGenFunction &CGF, PrePostActionTy &) {
3655 if (isOpenMPSimdDirective(EKind)) {
3656 CGF.EmitOMPSimdInit(S);
3657 } else if (const auto *C = S.getSingleClause<OMPOrderClause>()) {
3658 if (C->getKind() == OMPC_ORDER_concurrent)
3659 CGF.LoopStack.setParallel(/*Enable=*/true);
3660 }
3661 },
3662 [IVSize, IVSigned, Ordered, IL, LB, UB, ST, StaticChunkedOne, Chunk,
3663 &S, ScheduleKind, LoopExit, EKind,
3664 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
3665 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
3666 // When no chunk_size is specified, the iteration space is divided
3667 // into chunks that are approximately equal in size, and at most
3668 // one chunk is distributed to each thread. Note that the size of
3669 // the chunks is unspecified in this case.
3671 IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(),
3672 UB.getAddress(), ST.getAddress(),
3673 StaticChunkedOne ? Chunk : nullptr);
3675 CGF, S.getBeginLoc(), EKind, ScheduleKind, StaticInit);
3676 // UB = min(UB, GlobalUB);
3677 if (!StaticChunkedOne)
3678 CGF.EmitIgnoredExpr(S.getEnsureUpperBound());
3679 // IV = LB;
3680 CGF.EmitIgnoredExpr(S.getInit());
3681 // For unchunked static schedule generate:
3682 //
3683 // while (idx <= UB) {
3684 // BODY;
3685 // ++idx;
3686 // }
3687 //
3688 // For static schedule with chunk one:
3689 //
3690 // while (IV <= PrevUB) {
3691 // BODY;
3692 // IV += ST;
3693 // }
3694 CGF.EmitOMPInnerLoop(
3695 S, LoopScope.requiresCleanups(),
3696 StaticChunkedOne ? S.getCombinedParForInDistCond()
3697 : S.getCond(),
3698 StaticChunkedOne ? S.getDistInc() : S.getInc(),
3699 [&S, LoopExit](CodeGenFunction &CGF) {
3700 emitOMPLoopBodyWithStopPoint(CGF, S, LoopExit);
3701 },
3702 [](CodeGenFunction &) {});
3703 });
3704 EmitBlock(LoopExit.getBlock());
3705 // Tell the runtime we are done.
3706 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
3707 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
3708 OMPD_for);
3709 };
3710 OMPCancelStack.emitExit(*this, EKind, CodeGen);
3711 } else {
3712 // Emit the outer loop, which requests its work chunk [LB..UB] from
3713 // runtime and runs the inner loop to process it.
3714 OMPLoopArguments LoopArguments(LB.getAddress(), UB.getAddress(),
3715 ST.getAddress(), IL.getAddress(), Chunk,
3716 EUB);
3717 LoopArguments.DKind = OMPD_for;
3718 EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
3719 LoopArguments, CGDispatchBounds);
3720 }
3721 if (isOpenMPSimdDirective(EKind)) {
3722 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
3723 return CGF.Builder.CreateIsNotNull(
3724 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3725 });
3726 }
3728 S, /*ReductionKind=*/isOpenMPSimdDirective(EKind)
3729 ? /*Parallel and Simd*/ OMPD_parallel_for_simd
3730 : /*Parallel only*/ OMPD_parallel);
3731 // Emit post-update of the reduction variables if IsLastIter != 0.
3733 *this, S, [IL, &S](CodeGenFunction &CGF) {
3734 return CGF.Builder.CreateIsNotNull(
3735 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3736 });
3737 // Emit final copy of the lastprivate variables if IsLastIter != 0.
3738 if (HasLastprivateClause)
3740 S, isOpenMPSimdDirective(EKind),
3741 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
3742 LoopScope.restoreMap();
3743 EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) {
3744 return CGF.Builder.CreateIsNotNull(
3745 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3746 });
3747 }
3748 DoacrossCleanupScope.ForceCleanup();
3749 // We're now done with the loop, so jump to the continuation block.
3750 if (ContBlock) {
3751 EmitBranch(ContBlock);
3752 EmitBlock(ContBlock, /*IsFinished=*/true);
3753 }
3754 }
3755 return HasLastprivateClause;
3756}
3757
3758/// The following two functions generate expressions for the loop lower
3759/// and upper bounds in case of static and dynamic (dispatch) schedule
3760/// of the associated 'for' or 'distribute' loop.
3761static std::pair<LValue, LValue>
3763 const auto &LS = cast<OMPLoopDirective>(S);
3764 LValue LB =
3765 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
3766 LValue UB =
3767 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
3768 return {LB, UB};
3769}
3770
3771/// When dealing with dispatch schedules (e.g. dynamic, guided) we do not
3772/// consider the lower and upper bound expressions generated by the
3773/// worksharing loop support, but we use 0 and the iteration space size as
3774/// constants
3775static std::pair<llvm::Value *, llvm::Value *>
3777 Address LB, Address UB) {
3778 const auto &LS = cast<OMPLoopDirective>(S);
3779 const Expr *IVExpr = LS.getIterationVariable();
3780 const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType());
3781 llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0);
3782 llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration());
3783 return {LBVal, UBVal};
3784}
3785
3786/// Emits internal temp array declarations for the directive with inscan
3787/// reductions.
3788/// The code is the following:
3789/// \code
3790/// size num_iters = <num_iters>;
3791/// <type> buffer[num_iters];
3792/// \endcode
3794 CodeGenFunction &CGF, const OMPLoopDirective &S,
3795 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen) {
3796 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
3797 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false);
3800 SmallVector<const Expr *, 4> ReductionOps;
3801 SmallVector<const Expr *, 4> CopyArrayTemps;
3802 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
3803 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
3804 "Only inscan reductions are expected.");
3805 Shareds.append(C->varlist_begin(), C->varlist_end());
3806 Privates.append(C->privates().begin(), C->privates().end());
3807 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
3808 CopyArrayTemps.append(C->copy_array_temps().begin(),
3809 C->copy_array_temps().end());
3810 }
3811 {
3812 // Emit buffers for each reduction variables.
3813 // ReductionCodeGen is required to emit correctly the code for array
3814 // reductions.
3815 ReductionCodeGen RedCG(Shareds, Shareds, Privates, ReductionOps);
3816 unsigned Count = 0;
3817 auto *ITA = CopyArrayTemps.begin();
3818 for (const Expr *IRef : Privates) {
3819 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
3820 // Emit variably modified arrays, used for arrays/array sections
3821 // reductions.
3822 if (PrivateVD->getType()->isVariablyModifiedType()) {
3823 RedCG.emitSharedOrigLValue(CGF, Count);
3824 RedCG.emitAggregateType(CGF, Count);
3825 }
3827 CGF,
3829 cast<VariableArrayType>((*ITA)->getType()->getAsArrayTypeUnsafe())
3830 ->getSizeExpr()),
3831 RValue::get(OMPScanNumIterations));
3832 // Emit temp buffer.
3833 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(*ITA)->getDecl()));
3834 ++ITA;
3835 ++Count;
3836 }
3837 }
3838}
3839
3840/// Copies final inscan reductions values to the original variables.
3841/// The code is the following:
3842/// \code
3843/// <orig_var> = buffer[num_iters-1];
3844/// \endcode
3846 CodeGenFunction &CGF, const OMPLoopDirective &S,
3847 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen) {
3848 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
3849 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false);
3855 SmallVector<const Expr *, 4> CopyArrayElems;
3856 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
3857 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
3858 "Only inscan reductions are expected.");
3859 Shareds.append(C->varlist_begin(), C->varlist_end());
3860 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
3861 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
3862 Privates.append(C->privates().begin(), C->privates().end());
3863 CopyOps.append(C->copy_ops().begin(), C->copy_ops().end());
3864 CopyArrayElems.append(C->copy_array_elems().begin(),
3865 C->copy_array_elems().end());
3866 }
3867 // Create temp var and copy LHS value to this temp value.
3868 // LHS = TMP[LastIter];
3869 llvm::Value *OMPLast = CGF.Builder.CreateNSWSub(
3870 OMPScanNumIterations,
3871 llvm::ConstantInt::get(CGF.SizeTy, 1, /*isSigned=*/false));
3872 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
3873 const Expr *PrivateExpr = Privates[I];
3874 const Expr *OrigExpr = Shareds[I];
3875 const Expr *CopyArrayElem = CopyArrayElems[I];
3877 CGF,
3879 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3880 RValue::get(OMPLast));
3881 LValue DestLVal = CGF.EmitLValue(OrigExpr);
3882 LValue SrcLVal = CGF.EmitLValue(CopyArrayElem);
3883 CGF.EmitOMPCopy(
3884 PrivateExpr->getType(), DestLVal.getAddress(), SrcLVal.getAddress(),
3885 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
3886 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()), CopyOps[I]);
3887 }
3888}
3889
3890/// Emits the code for the directive with inscan reductions.
3891/// The code is the following:
3892/// \code
3893/// #pragma omp ...
3894/// for (i: 0..<num_iters>) {
3895/// <input phase>;
3896/// buffer[i] = red;
3897/// }
3898/// #pragma omp master // in parallel region
3899/// for (int k = 0; k != ceil(log2(num_iters)); ++k)
3900/// for (size cnt = last_iter; cnt >= pow(2, k); --k)
3901/// buffer[i] op= buffer[i-pow(2,k)];
3902/// #pragma omp barrier // in parallel region
3903/// #pragma omp ...
3904/// for (0..<num_iters>) {
3905/// red = InclusiveScan ? buffer[i] : buffer[i-1];
3906/// <scan phase>;
3907/// }
3908/// \endcode
3910 CodeGenFunction &CGF, const OMPLoopDirective &S,
3911 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen,
3912 llvm::function_ref<void(CodeGenFunction &)> FirstGen,
3913 llvm::function_ref<void(CodeGenFunction &)> SecondGen) {
3914 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
3915 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false);
3917 SmallVector<const Expr *, 4> ReductionOps;
3920 SmallVector<const Expr *, 4> CopyArrayElems;
3921 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
3922 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
3923 "Only inscan reductions are expected.");
3924 Privates.append(C->privates().begin(), C->privates().end());
3925 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
3926 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
3927 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
3928 CopyArrayElems.append(C->copy_array_elems().begin(),
3929 C->copy_array_elems().end());
3930 }
3932 {
3933 // Emit loop with input phase:
3934 // #pragma omp ...
3935 // for (i: 0..<num_iters>) {
3936 // <input phase>;
3937 // buffer[i] = red;
3938 // }
3939 CGF.OMPFirstScanLoop = true;
3941 FirstGen(CGF);
3942 }
3943 // #pragma omp barrier // in parallel region
3944 auto &&CodeGen = [&S, OMPScanNumIterations, &LHSs, &RHSs, &CopyArrayElems,
3945 &ReductionOps,
3946 &Privates](CodeGenFunction &CGF, PrePostActionTy &Action) {
3947 Action.Enter(CGF);
3948 // Emit prefix reduction:
3949 // #pragma omp master // in parallel region
3950 // for (int k = 0; k <= ceil(log2(n)); ++k)
3951 llvm::BasicBlock *InputBB = CGF.Builder.GetInsertBlock();
3952 llvm::BasicBlock *LoopBB = CGF.createBasicBlock("omp.outer.log.scan.body");
3953 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("omp.outer.log.scan.exit");
3954 llvm::Function *F =
3955 CGF.CGM.getIntrinsic(llvm::Intrinsic::log2, CGF.DoubleTy);
3956 llvm::Value *Arg =
3957 CGF.Builder.CreateUIToFP(OMPScanNumIterations, CGF.DoubleTy);
3958 llvm::Value *LogVal = CGF.EmitNounwindRuntimeCall(F, Arg);
3959 F = CGF.CGM.getIntrinsic(llvm::Intrinsic::ceil, CGF.DoubleTy);
3960 LogVal = CGF.EmitNounwindRuntimeCall(F, LogVal);
3961 LogVal = CGF.Builder.CreateFPToUI(LogVal, CGF.IntTy);
3962 llvm::Value *NMin1 = CGF.Builder.CreateNUWSub(
3963 OMPScanNumIterations, llvm::ConstantInt::get(CGF.SizeTy, 1));
3964 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getBeginLoc());
3965 CGF.EmitBlock(LoopBB);
3966 auto *Counter = CGF.Builder.CreatePHI(CGF.IntTy, 2);
3967 // size pow2k = 1;
3968 auto *Pow2K = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
3969 Counter->addIncoming(llvm::ConstantInt::get(CGF.IntTy, 0), InputBB);
3970 Pow2K->addIncoming(llvm::ConstantInt::get(CGF.SizeTy, 1), InputBB);
3971 // for (size i = n - 1; i >= 2 ^ k; --i)
3972 // tmp[i] op= tmp[i-pow2k];
3973 llvm::BasicBlock *InnerLoopBB =
3974 CGF.createBasicBlock("omp.inner.log.scan.body");
3975 llvm::BasicBlock *InnerExitBB =
3976 CGF.createBasicBlock("omp.inner.log.scan.exit");
3977 llvm::Value *CmpI = CGF.Builder.CreateICmpUGE(NMin1, Pow2K);
3978 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
3979 CGF.EmitBlock(InnerLoopBB);
3980 auto *IVal = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
3981 IVal->addIncoming(NMin1, LoopBB);
3982 {
3983 CodeGenFunction::OMPPrivateScope PrivScope(CGF);
3984 auto *ILHS = LHSs.begin();
3985 auto *IRHS = RHSs.begin();
3986 for (const Expr *CopyArrayElem : CopyArrayElems) {
3987 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
3988 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
3989 Address LHSAddr = Address::invalid();
3990 {
3992 CGF,
3994 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
3995 RValue::get(IVal));
3996 LHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress();
3997 }
3998 PrivScope.addPrivate(LHSVD, LHSAddr);
3999 Address RHSAddr = Address::invalid();
4000 {
4001 llvm::Value *OffsetIVal = CGF.Builder.CreateNUWSub(IVal, Pow2K);
4003 CGF,
4005 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
4006 RValue::get(OffsetIVal));
4007 RHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress();
4008 }
4009 PrivScope.addPrivate(RHSVD, RHSAddr);
4010 ++ILHS;
4011 ++IRHS;
4012 }
4013 PrivScope.Privatize();
4014 CGF.CGM.getOpenMPRuntime().emitReduction(
4015 CGF, S.getEndLoc(), Privates, LHSs, RHSs, ReductionOps,
4016 {/*WithNowait=*/true, /*SimpleReduction=*/true,
4017 /*IsPrivateVarReduction*/ {}, OMPD_unknown});
4018 }
4019 llvm::Value *NextIVal =
4020 CGF.Builder.CreateNUWSub(IVal, llvm::ConstantInt::get(CGF.SizeTy, 1));
4021 IVal->addIncoming(NextIVal, CGF.Builder.GetInsertBlock());
4022 CmpI = CGF.Builder.CreateICmpUGE(NextIVal, Pow2K);
4023 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
4024 CGF.EmitBlock(InnerExitBB);
4025 llvm::Value *Next =
4026 CGF.Builder.CreateNUWAdd(Counter, llvm::ConstantInt::get(CGF.IntTy, 1));
4027 Counter->addIncoming(Next, CGF.Builder.GetInsertBlock());
4028 // pow2k <<= 1;
4029 llvm::Value *NextPow2K =
4030 CGF.Builder.CreateShl(Pow2K, 1, "", /*HasNUW=*/true);
4031 Pow2K->addIncoming(NextPow2K, CGF.Builder.GetInsertBlock());
4032 llvm::Value *Cmp = CGF.Builder.CreateICmpNE(Next, LogVal);
4033 CGF.Builder.CreateCondBr(Cmp, LoopBB, ExitBB);
4034 auto DL1 = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getEndLoc());
4035 CGF.EmitBlock(ExitBB);
4036 };
4038 if (isOpenMPParallelDirective(EKind)) {
4039 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc());
4041 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
4042 /*ForceSimpleCall=*/true);
4043 } else {
4044 RegionCodeGenTy RCG(CodeGen);
4045 RCG(CGF);
4046 }
4047
4048 CGF.OMPFirstScanLoop = false;
4049 SecondGen(CGF);
4050}
4051
4053 const OMPLoopDirective &S,
4054 bool HasCancel) {
4055 bool HasLastprivates;
4057 if (llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),
4058 [](const OMPReductionClause *C) {
4059 return C->getModifier() == OMPC_REDUCTION_inscan;
4060 })) {
4061 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
4063 OMPLoopScope LoopScope(CGF, S);
4064 return CGF.EmitScalarExpr(S.getNumIterations());
4065 };
4066 const auto &&FirstGen = [&S, HasCancel, EKind](CodeGenFunction &CGF) {
4067 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);
4068 (void)CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
4071 // Emit an implicit barrier at the end.
4072 CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getBeginLoc(),
4073 OMPD_for);
4074 };
4075 const auto &&SecondGen = [&S, HasCancel, EKind,
4076 &HasLastprivates](CodeGenFunction &CGF) {
4077 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);
4078 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
4081 };
4082 if (!isOpenMPParallelDirective(EKind))
4083 emitScanBasedDirectiveDecls(CGF, S, NumIteratorsGen);
4084 emitScanBasedDirective(CGF, S, NumIteratorsGen, FirstGen, SecondGen);
4085 if (!isOpenMPParallelDirective(EKind))
4086 emitScanBasedDirectiveFinals(CGF, S, NumIteratorsGen);
4087 } else {
4088 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);
4089 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
4092 }
4093 return HasLastprivates;
4094}
4095
4096// Pass OMPLoopDirective (instead of OMPForDirective) to make this check
4097// available for "loop bind(parallel)", which maps to "for".
4099 bool HasCancel) {
4100 if (HasCancel)
4101 return false;
4102 for (OMPClause *C : S.clauses()) {
4104 continue;
4105
4106 if (auto *SC = dyn_cast<OMPScheduleClause>(C)) {
4107 if (SC->getFirstScheduleModifier() != OMPC_SCHEDULE_MODIFIER_unknown)
4108 return false;
4109 if (SC->getSecondScheduleModifier() != OMPC_SCHEDULE_MODIFIER_unknown)
4110 return false;
4111 switch (SC->getScheduleKind()) {
4112 case OMPC_SCHEDULE_auto:
4113 case OMPC_SCHEDULE_dynamic:
4114 case OMPC_SCHEDULE_runtime:
4115 case OMPC_SCHEDULE_guided:
4116 case OMPC_SCHEDULE_static:
4117 continue;
4119 return false;
4120 }
4121 }
4122
4123 return false;
4124 }
4125
4126 return true;
4127}
4128
4129static llvm::omp::ScheduleKind
4131 switch (ScheduleClauseKind) {
4133 return llvm::omp::OMP_SCHEDULE_Default;
4134 case OMPC_SCHEDULE_auto:
4135 return llvm::omp::OMP_SCHEDULE_Auto;
4136 case OMPC_SCHEDULE_dynamic:
4137 return llvm::omp::OMP_SCHEDULE_Dynamic;
4138 case OMPC_SCHEDULE_guided:
4139 return llvm::omp::OMP_SCHEDULE_Guided;
4140 case OMPC_SCHEDULE_runtime:
4141 return llvm::omp::OMP_SCHEDULE_Runtime;
4142 case OMPC_SCHEDULE_static:
4143 return llvm::omp::OMP_SCHEDULE_Static;
4144 }
4145 llvm_unreachable("Unhandled schedule kind");
4146}
4147
4148// Pass OMPLoopDirective (instead of OMPForDirective) to make this function
4149// available for "loop bind(parallel)", which maps to "for".
4151 CodeGenModule &CGM, bool HasCancel) {
4152 bool HasLastprivates = false;
4153 bool UseOMPIRBuilder = CGM.getLangOpts().OpenMPIRBuilder &&
4154 isForSupportedByOpenMPIRBuilder(S, HasCancel);
4155 auto &&CodeGen = [&S, &CGM, HasCancel, &HasLastprivates,
4156 UseOMPIRBuilder](CodeGenFunction &CGF, PrePostActionTy &) {
4157 // Use the OpenMPIRBuilder if enabled.
4158 if (UseOMPIRBuilder) {
4159 bool NeedsBarrier = !S.getSingleClause<OMPNowaitClause>();
4160
4161 llvm::omp::ScheduleKind SchedKind = llvm::omp::OMP_SCHEDULE_Default;
4162 llvm::Value *ChunkSize = nullptr;
4163 if (auto *SchedClause = S.getSingleClause<OMPScheduleClause>()) {
4164 SchedKind =
4165 convertClauseKindToSchedKind(SchedClause->getScheduleKind());
4166 if (const Expr *ChunkSizeExpr = SchedClause->getChunkSize())
4167 ChunkSize = CGF.EmitScalarExpr(ChunkSizeExpr);
4168 }
4169
4170 // Emit the associated statement and get its loop representation.
4171 const Stmt *Inner = S.getRawStmt();
4172 llvm::CanonicalLoopInfo *CLI =
4174
4175 llvm::OpenMPIRBuilder &OMPBuilder =
4177 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
4178 CGF.AllocaInsertPt->getParent(), CGF.AllocaInsertPt->getIterator());
4179 cantFail(OMPBuilder.applyWorkshareLoop(
4180 CGF.Builder.getCurrentDebugLocation(), CLI, AllocaIP, NeedsBarrier,
4181 SchedKind, ChunkSize, /*HasSimdModifier=*/false,
4182 /*HasMonotonicModifier=*/false, /*HasNonmonotonicModifier=*/false,
4183 /*HasOrderedClause=*/false));
4184 return;
4185 }
4186
4187 HasLastprivates = emitWorksharingDirective(CGF, S, HasCancel);
4188 };
4189 {
4190 auto LPCRegion =
4192 OMPLexicalScope Scope(CGF, S, OMPD_unknown);
4194 HasCancel);
4195 }
4196
4197 if (!UseOMPIRBuilder) {
4198 // Emit an implicit barrier at the end.
4199 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
4200 CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getBeginLoc(), OMPD_for);
4201 }
4202 // Check for outer lastprivate conditional update.
4204}
4205
4206void CodeGenFunction::EmitOMPForDirective(const OMPForDirective &S) {
4207 return emitOMPForDirective(S, *this, CGM, S.hasCancel());
4208}
4209
4210void CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &S) {
4211 bool HasLastprivates = false;
4212 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
4213 PrePostActionTy &) {
4214 HasLastprivates = emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
4215 };
4216 {
4217 auto LPCRegion =
4219 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4220 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
4221 }
4222
4223 // Emit an implicit barrier at the end.
4224 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
4225 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
4226 // Check for outer lastprivate conditional update.
4228}
4229
4231 const Twine &Name,
4232 llvm::Value *Init = nullptr) {
4233 LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
4234 if (Init)
4235 CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true);
4236 return LVal;
4237}
4238
4239void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
4240 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
4241 const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);
4242 bool HasLastprivates = false;
4244 auto &&CodeGen = [&S, CapturedStmt, CS, EKind,
4245 &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) {
4246 const ASTContext &C = CGF.getContext();
4247 QualType KmpInt32Ty =
4248 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
4249 // Emit helper vars inits.
4250 LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
4251 CGF.Builder.getInt32(0));
4252 llvm::ConstantInt *GlobalUBVal = CS != nullptr
4253 ? CGF.Builder.getInt32(CS->size() - 1)
4254 : CGF.Builder.getInt32(0);
4255 LValue UB =
4256 createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
4257 LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
4258 CGF.Builder.getInt32(1));
4259 LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
4260 CGF.Builder.getInt32(0));
4261 // Loop counter.
4262 LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
4263 OpaqueValueExpr IVRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
4264 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
4265 OpaqueValueExpr UBRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
4266 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
4267 // Generate condition for loop.
4268 BinaryOperator *Cond = BinaryOperator::Create(
4269 C, &IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_PRValue, OK_Ordinary,
4270 S.getBeginLoc(), FPOptionsOverride());
4271 // Increment for loop counter.
4272 UnaryOperator *Inc = UnaryOperator::Create(
4273 C, &IVRefExpr, UO_PreInc, KmpInt32Ty, VK_PRValue, OK_Ordinary,
4274 S.getBeginLoc(), true, FPOptionsOverride());
4275 auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
4276 // Iterate through all sections and emit a switch construct:
4277 // switch (IV) {
4278 // case 0:
4279 // <SectionStmt[0]>;
4280 // break;
4281 // ...
4282 // case <NumSection> - 1:
4283 // <SectionStmt[<NumSection> - 1]>;
4284 // break;
4285 // }
4286 // .omp.sections.exit:
4287 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
4288 llvm::SwitchInst *SwitchStmt =
4289 CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()),
4290 ExitBB, CS == nullptr ? 1 : CS->size());
4291 if (CS) {
4292 unsigned CaseNumber = 0;
4293 for (const Stmt *SubStmt : CS->children()) {
4294 auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
4295 CGF.EmitBlock(CaseBB);
4296 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
4297 CGF.EmitStmt(SubStmt);
4298 CGF.EmitBranch(ExitBB);
4299 ++CaseNumber;
4300 }
4301 } else {
4302 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case");
4303 CGF.EmitBlock(CaseBB);
4304 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
4305 CGF.EmitStmt(CapturedStmt);
4306 CGF.EmitBranch(ExitBB);
4307 }
4308 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
4309 };
4310
4311 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
4312 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
4313 // Emit implicit barrier to synchronize threads and avoid data races on
4314 // initialization of firstprivate variables and post-update of lastprivate
4315 // variables.
4316 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
4317 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
4318 /*ForceSimpleCall=*/true);
4319 }
4320 CGF.EmitOMPPrivateClause(S, LoopScope);
4321 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(CGF, S, IV);
4322 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4323 CGF.EmitOMPReductionClauseInit(S, LoopScope);
4324 (void)LoopScope.Privatize();
4326 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4327
4328 // Emit static non-chunked loop.
4329 OpenMPScheduleTy ScheduleKind;
4330 ScheduleKind.Schedule = OMPC_SCHEDULE_static;
4331 CGOpenMPRuntime::StaticRTInput StaticInit(
4332 /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(),
4333 LB.getAddress(), UB.getAddress(), ST.getAddress());
4334 CGF.CGM.getOpenMPRuntime().emitForStaticInit(CGF, S.getBeginLoc(), EKind,
4335 ScheduleKind, StaticInit);
4336 // UB = min(UB, GlobalUB);
4337 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc());
4338 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
4339 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
4340 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
4341 // IV = LB;
4342 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV);
4343 // while (idx <= UB) { BODY; ++idx; }
4344 CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, Cond, Inc, BodyGen,
4345 [](CodeGenFunction &) {});
4346 // Tell the runtime we are done.
4347 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
4348 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
4349 OMPD_sections);
4350 };
4351 CGF.OMPCancelStack.emitExit(CGF, EKind, CodeGen);
4352 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4353 // Emit post-update of the reduction variables if IsLastIter != 0.
4354 emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) {
4355 return CGF.Builder.CreateIsNotNull(
4356 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
4357 });
4358
4359 // Emit final copy of the lastprivate variables if IsLastIter != 0.
4360 if (HasLastprivates)
4362 S, /*NoFinals=*/false,
4363 CGF.Builder.CreateIsNotNull(
4364 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())));
4365 };
4366
4367 bool HasCancel = false;
4368 if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
4369 HasCancel = OSD->hasCancel();
4370 else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
4371 HasCancel = OPSD->hasCancel();
4372 OMPCancelStackRAII CancelRegion(*this, EKind, HasCancel);
4373 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,
4374 HasCancel);
4375 // Emit barrier for lastprivates only if 'sections' directive has 'nowait'
4376 // clause. Otherwise the barrier will be generated by the codegen for the
4377 // directive.
4378 if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
4379 // Emit implicit barrier to synchronize threads and avoid data races on
4380 // initialization of firstprivate variables.
4381 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
4382 OMPD_unknown);
4383 }
4384}
4385
4386void CodeGenFunction::EmitOMPScopeDirective(const OMPScopeDirective &S) {
4387 {
4388 // Emit code for 'scope' region
4389 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4390 Action.Enter(CGF);
4391 OMPPrivateScope PrivateScope(CGF);
4392 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4393 CGF.EmitOMPPrivateClause(S, PrivateScope);
4394 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4395 (void)PrivateScope.Privatize();
4396 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
4397 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4398 };
4399 auto LPCRegion =
4401 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4402 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_scope, CodeGen);
4403 }
4404 // Emit an implicit barrier at the end.
4405 if (!S.getSingleClause<OMPNowaitClause>()) {
4406 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_scope);
4407 }
4408 // Check for outer lastprivate conditional update.
4410}
4411
4412void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) {
4413 if (CGM.getLangOpts().OpenMPIRBuilder) {
4414 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4415 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4416 using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
4417
4418 auto FiniCB = [](InsertPointTy IP) {
4419 // Don't FinalizeOMPRegion because this is done inside of OMPIRBuilder for
4420 // sections.
4421 return llvm::Error::success();
4422 };
4423
4424 const CapturedStmt *ICS = S.getInnermostCapturedStmt();
4425 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
4426 const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);
4428 if (CS) {
4429 for (const Stmt *SubStmt : CS->children()) {
4430 auto SectionCB = [this, SubStmt](InsertPointTy AllocaIP,
4431 InsertPointTy CodeGenIP) {
4433 *this, SubStmt, AllocaIP, CodeGenIP, "section");
4434 return llvm::Error::success();
4435 };
4436 SectionCBVector.push_back(SectionCB);
4437 }
4438 } else {
4439 auto SectionCB = [this, CapturedStmt](InsertPointTy AllocaIP,
4440 InsertPointTy CodeGenIP) {
4442 *this, CapturedStmt, AllocaIP, CodeGenIP, "section");
4443 return llvm::Error::success();
4444 };
4445 SectionCBVector.push_back(SectionCB);
4446 }
4447
4448 // Privatization callback that performs appropriate action for
4449 // shared/private/firstprivate/lastprivate/copyin/... variables.
4450 //
4451 // TODO: This defaults to shared right now.
4452 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
4453 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
4454 // The next line is appropriate only for variables (Val) with the
4455 // data-sharing attribute "shared".
4456 ReplVal = &Val;
4457
4458 return CodeGenIP;
4459 };
4460
4461 CGCapturedStmtInfo CGSI(*ICS, CR_OpenMP);
4462 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI);
4463 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
4464 AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());
4465 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
4466 cantFail(OMPBuilder.createSections(
4467 Builder, AllocaIP, SectionCBVector, PrivCB, FiniCB, S.hasCancel(),
4468 S.getSingleClause<OMPNowaitClause>()));
4469 Builder.restoreIP(AfterIP);
4470 return;
4471 }
4472 {
4473 auto LPCRegion =
4475 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4476 EmitSections(S);
4477 }
4478 // Emit an implicit barrier at the end.
4479 if (!S.getSingleClause<OMPNowaitClause>()) {
4480 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
4481 OMPD_sections);
4482 }
4483 // Check for outer lastprivate conditional update.
4485}
4486
4487void CodeGenFunction::EmitOMPSectionDirective(const OMPSectionDirective &S) {
4488 if (CGM.getLangOpts().OpenMPIRBuilder) {
4489 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4490 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4491
4492 const Stmt *SectionRegionBodyStmt = S.getAssociatedStmt();
4493 auto FiniCB = [this](InsertPointTy IP) {
4495 return llvm::Error::success();
4496 };
4497
4498 auto BodyGenCB = [SectionRegionBodyStmt, this](InsertPointTy AllocaIP,
4499 InsertPointTy CodeGenIP) {
4501 *this, SectionRegionBodyStmt, AllocaIP, CodeGenIP, "section");
4502 return llvm::Error::success();
4503 };
4504
4505 LexicalScope Scope(*this, S.getSourceRange());
4506 EmitStopPoint(&S);
4507 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
4508 cantFail(OMPBuilder.createSection(Builder, BodyGenCB, FiniCB));
4509 Builder.restoreIP(AfterIP);
4510
4511 return;
4512 }
4513 LexicalScope Scope(*this, S.getSourceRange());
4514 EmitStopPoint(&S);
4515 EmitStmt(S.getAssociatedStmt());
4516}
4517
4518void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &S) {
4519 llvm::SmallVector<const Expr *, 8> CopyprivateVars;
4523 // Check if there are any 'copyprivate' clauses associated with this
4524 // 'single' construct.
4525 // Build a list of copyprivate variables along with helper expressions
4526 // (<source>, <destination>, <destination>=<source> expressions)
4527 for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
4528 CopyprivateVars.append(C->varlist_begin(), C->varlist_end());
4529 DestExprs.append(C->destination_exprs().begin(),
4530 C->destination_exprs().end());
4531 SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
4532 AssignmentOps.append(C->assignment_ops().begin(),
4533 C->assignment_ops().end());
4534 }
4535 // Emit code for 'single' region along with 'copyprivate' clauses
4536 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4537 Action.Enter(CGF);
4541 (void)SingleScope.Privatize();
4542 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
4543 };
4544 {
4545 auto LPCRegion =
4547 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4548 CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getBeginLoc(),
4549 CopyprivateVars, DestExprs,
4550 SrcExprs, AssignmentOps);
4551 }
4552 // Emit an implicit barrier at the end (to avoid data race on firstprivate
4553 // init or if no 'nowait' clause was specified and no 'copyprivate' clause).
4554 if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {
4555 CGM.getOpenMPRuntime().emitBarrierCall(
4556 *this, S.getBeginLoc(),
4557 S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
4558 }
4559 // Check for outer lastprivate conditional update.
4561}
4562
4564 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4565 Action.Enter(CGF);
4566 CGF.EmitStmt(S.getRawStmt());
4567 };
4568 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc());
4569}
4570
4571void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) {
4572 if (CGM.getLangOpts().OpenMPIRBuilder) {
4573 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4574 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4575
4576 const Stmt *MasterRegionBodyStmt = S.getAssociatedStmt();
4577
4578 auto FiniCB = [this](InsertPointTy IP) {
4580 return llvm::Error::success();
4581 };
4582
4583 auto BodyGenCB = [MasterRegionBodyStmt, this](InsertPointTy AllocaIP,
4584 InsertPointTy CodeGenIP) {
4586 *this, MasterRegionBodyStmt, AllocaIP, CodeGenIP, "master");
4587 return llvm::Error::success();
4588 };
4589
4590 LexicalScope Scope(*this, S.getSourceRange());
4591 EmitStopPoint(&S);
4592 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
4593 cantFail(OMPBuilder.createMaster(Builder, BodyGenCB, FiniCB));
4594 Builder.restoreIP(AfterIP);
4595
4596 return;
4597 }
4598 LexicalScope Scope(*this, S.getSourceRange());
4599 EmitStopPoint(&S);
4600 emitMaster(*this, S);
4601}
4602
4604 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4605 Action.Enter(CGF);
4606 CGF.EmitStmt(S.getRawStmt());
4607 };
4608 Expr *Filter = nullptr;
4609 if (const auto *FilterClause = S.getSingleClause<OMPFilterClause>())
4610 Filter = FilterClause->getThreadID();
4611 CGF.CGM.getOpenMPRuntime().emitMaskedRegion(CGF, CodeGen, S.getBeginLoc(),
4612 Filter);
4613}
4614
4616 if (CGM.getLangOpts().OpenMPIRBuilder) {
4617 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4618 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4619
4620 const Stmt *MaskedRegionBodyStmt = S.getAssociatedStmt();
4621 const Expr *Filter = nullptr;
4622 if (const auto *FilterClause = S.getSingleClause<OMPFilterClause>())
4623 Filter = FilterClause->getThreadID();
4624 llvm::Value *FilterVal = Filter
4625 ? EmitScalarExpr(Filter, CGM.Int32Ty)
4626 : llvm::ConstantInt::get(CGM.Int32Ty, /*V=*/0);
4627
4628 auto FiniCB = [this](InsertPointTy IP) {
4630 return llvm::Error::success();
4631 };
4632
4633 auto BodyGenCB = [MaskedRegionBodyStmt, this](InsertPointTy AllocaIP,
4634 InsertPointTy CodeGenIP) {
4636 *this, MaskedRegionBodyStmt, AllocaIP, CodeGenIP, "masked");
4637 return llvm::Error::success();
4638 };
4639
4640 LexicalScope Scope(*this, S.getSourceRange());
4641 EmitStopPoint(&S);
4642 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
4643 OMPBuilder.createMasked(Builder, BodyGenCB, FiniCB, FilterVal));
4644 Builder.restoreIP(AfterIP);
4645
4646 return;
4647 }
4648 LexicalScope Scope(*this, S.getSourceRange());
4649 EmitStopPoint(&S);
4650 emitMasked(*this, S);
4651}
4652
4653void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) {
4654 if (CGM.getLangOpts().OpenMPIRBuilder) {
4655 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4656 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4657
4658 const Stmt *CriticalRegionBodyStmt = S.getAssociatedStmt();
4659 const Expr *Hint = nullptr;
4660 if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
4661 Hint = HintClause->getHint();
4662
4663 // TODO: This is slightly different from what's currently being done in
4664 // clang. Fix the Int32Ty to IntPtrTy (pointer width size) when everything
4665 // about typing is final.
4666 llvm::Value *HintInst = nullptr;
4667 if (Hint)
4668 HintInst =
4669 Builder.CreateIntCast(EmitScalarExpr(Hint), CGM.Int32Ty, false);
4670
4671 auto FiniCB = [this](InsertPointTy IP) {
4673 return llvm::Error::success();
4674 };
4675
4676 auto BodyGenCB = [CriticalRegionBodyStmt, this](InsertPointTy AllocaIP,
4677 InsertPointTy CodeGenIP) {
4679 *this, CriticalRegionBodyStmt, AllocaIP, CodeGenIP, "critical");
4680 return llvm::Error::success();
4681 };
4682
4683 LexicalScope Scope(*this, S.getSourceRange());
4684 EmitStopPoint(&S);
4685 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
4686 cantFail(OMPBuilder.createCritical(Builder, BodyGenCB, FiniCB,
4687 S.getDirectiveName().getAsString(),
4688 HintInst));
4689 Builder.restoreIP(AfterIP);
4690
4691 return;
4692 }
4693
4694 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4695 Action.Enter(CGF);
4696 CGF.EmitStmt(S.getAssociatedStmt());
4697 };
4698 const Expr *Hint = nullptr;
4699 if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
4700 Hint = HintClause->getHint();
4701 LexicalScope Scope(*this, S.getSourceRange());
4702 EmitStopPoint(&S);
4703 CGM.getOpenMPRuntime().emitCriticalRegion(*this,
4704 S.getDirectiveName().getAsString(),
4705 CodeGen, S.getBeginLoc(), Hint);
4706}
4707
4709 const OMPParallelForDirective &S) {
4710 // Emit directive as a combined directive that consists of two implicit
4711 // directives: 'parallel' with 'for' directive.
4712 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4713 Action.Enter(CGF);
4714 emitOMPCopyinClause(CGF, S);
4715 (void)emitWorksharingDirective(CGF, S, S.hasCancel());
4716 };
4717 {
4718 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
4721 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
4722 OMPLoopScope LoopScope(CGF, S);
4723 return CGF.EmitScalarExpr(S.getNumIterations());
4724 };
4725 bool IsInscan = llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),
4726 [](const OMPReductionClause *C) {
4727 return C->getModifier() == OMPC_REDUCTION_inscan;
4728 });
4729 if (IsInscan)
4730 emitScanBasedDirectiveDecls(*this, S, NumIteratorsGen);
4731 auto LPCRegion =
4733 emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,
4735 if (IsInscan)
4736 emitScanBasedDirectiveFinals(*this, S, NumIteratorsGen);
4737 }
4738 // Check for outer lastprivate conditional update.
4740}
4741
4743 const OMPParallelForSimdDirective &S) {
4744 // Emit directive as a combined directive that consists of two implicit
4745 // directives: 'parallel' with 'for' directive.
4746 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4747 Action.Enter(CGF);
4748 emitOMPCopyinClause(CGF, S);
4749 (void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
4750 };
4751 {
4752 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
4755 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
4756 OMPLoopScope LoopScope(CGF, S);
4757 return CGF.EmitScalarExpr(S.getNumIterations());
4758 };
4759 bool IsInscan = llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),
4760 [](const OMPReductionClause *C) {
4761 return C->getModifier() == OMPC_REDUCTION_inscan;
4762 });
4763 if (IsInscan)
4764 emitScanBasedDirectiveDecls(*this, S, NumIteratorsGen);
4765 auto LPCRegion =
4767 emitCommonOMPParallelDirective(*this, S, OMPD_for_simd, CodeGen,
4769 if (IsInscan)
4770 emitScanBasedDirectiveFinals(*this, S, NumIteratorsGen);
4771 }
4772 // Check for outer lastprivate conditional update.
4774}
4775
4777 const OMPParallelMasterDirective &S) {
4778 // Emit directive as a combined directive that consists of two implicit
4779 // directives: 'parallel' with 'master' directive.
4780 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4781 Action.Enter(CGF);
4782 OMPPrivateScope PrivateScope(CGF);
4783 emitOMPCopyinClause(CGF, S);
4784 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4785 CGF.EmitOMPPrivateClause(S, PrivateScope);
4786 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4787 (void)PrivateScope.Privatize();
4788 emitMaster(CGF, S);
4789 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4790 };
4791 {
4792 auto LPCRegion =
4794 emitCommonOMPParallelDirective(*this, S, OMPD_master, CodeGen,
4797 [](CodeGenFunction &) { return nullptr; });
4798 }
4799 // Check for outer lastprivate conditional update.
4801}
4802
4804 const OMPParallelMaskedDirective &S) {
4805 // Emit directive as a combined directive that consists of two implicit
4806 // directives: 'parallel' with 'masked' directive.
4807 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4808 Action.Enter(CGF);
4809 OMPPrivateScope PrivateScope(CGF);
4810 emitOMPCopyinClause(CGF, S);
4811 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4812 CGF.EmitOMPPrivateClause(S, PrivateScope);
4813 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4814 (void)PrivateScope.Privatize();
4815 emitMasked(CGF, S);
4816 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4817 };
4818 {
4819 auto LPCRegion =
4821 emitCommonOMPParallelDirective(*this, S, OMPD_masked, CodeGen,
4824 [](CodeGenFunction &) { return nullptr; });
4825 }
4826 // Check for outer lastprivate conditional update.
4828}
4829
4831 const OMPParallelSectionsDirective &S) {
4832 // Emit directive as a combined directive that consists of two implicit
4833 // directives: 'parallel' with 'sections' directive.
4834 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4835 Action.Enter(CGF);
4836 emitOMPCopyinClause(CGF, S);
4837 CGF.EmitSections(S);
4838 };
4839 {
4840 auto LPCRegion =
4842 emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen,
4844 }
4845 // Check for outer lastprivate conditional update.
4847}
4848
4849namespace {
4850/// Get the list of variables declared in the context of the untied tasks.
4851class CheckVarsEscapingUntiedTaskDeclContext final
4852 : public ConstStmtVisitor<CheckVarsEscapingUntiedTaskDeclContext> {
4854
4855public:
4856 explicit CheckVarsEscapingUntiedTaskDeclContext() = default;
4857 ~CheckVarsEscapingUntiedTaskDeclContext() = default;
4858 void VisitDeclStmt(const DeclStmt *S) {
4859 if (!S)
4860 return;
4861 // Need to privatize only local vars, static locals can be processed as is.
4862 for (const Decl *D : S->decls()) {
4863 if (const auto *VD = dyn_cast_or_null<VarDecl>(D))
4864 if (VD->hasLocalStorage())
4865 PrivateDecls.push_back(VD);
4866 }
4867 }
4868 void VisitOMPExecutableDirective(const OMPExecutableDirective *) {}
4869 void VisitCapturedStmt(const CapturedStmt *) {}
4870 void VisitLambdaExpr(const LambdaExpr *) {}
4871 void VisitBlockExpr(const BlockExpr *) {}
4872 void VisitStmt(const Stmt *S) {
4873 if (!S)
4874 return;
4875 for (const Stmt *Child : S->children())
4876 if (Child)
4877 Visit(Child);
4878 }
4879
4880 /// Swaps list of vars with the provided one.
4881 ArrayRef<const VarDecl *> getPrivateDecls() const { return PrivateDecls; }
4882};
4883} // anonymous namespace
4884
4887
4888 // First look for 'omp_all_memory' and add this first.
4889 bool OmpAllMemory = false;
4890 if (llvm::any_of(
4891 S.getClausesOfKind<OMPDependClause>(), [](const OMPDependClause *C) {
4892 return C->getDependencyKind() == OMPC_DEPEND_outallmemory ||
4893 C->getDependencyKind() == OMPC_DEPEND_inoutallmemory;
4894 })) {
4895 OmpAllMemory = true;
4896 // Since both OMPC_DEPEND_outallmemory and OMPC_DEPEND_inoutallmemory are
4897 // equivalent to the runtime, always use OMPC_DEPEND_outallmemory to
4898 // simplify.
4900 Data.Dependences.emplace_back(OMPC_DEPEND_outallmemory,
4901 /*IteratorExpr=*/nullptr);
4902 // Add a nullptr Expr to simplify the codegen in emitDependData.
4903 DD.DepExprs.push_back(nullptr);
4904 }
4905 // Add remaining dependences skipping any 'out' or 'inout' if they are
4906 // overridden by 'omp_all_memory'.
4907 for (const auto *C : S.getClausesOfKind<OMPDependClause>()) {
4908 OpenMPDependClauseKind Kind = C->getDependencyKind();
4909 if (Kind == OMPC_DEPEND_outallmemory || Kind == OMPC_DEPEND_inoutallmemory)
4910 continue;
4911 if (OmpAllMemory && (Kind == OMPC_DEPEND_out || Kind == OMPC_DEPEND_inout))
4912 continue;
4914 Data.Dependences.emplace_back(C->getDependencyKind(), C->getModifier());
4915 DD.DepExprs.append(C->varlist_begin(), C->varlist_end());
4916 }
4917}
4918
4920 const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion,
4921 const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen,
4923 // Emit outlined function for task construct.
4924 const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);
4925 auto I = CS->getCapturedDecl()->param_begin();
4926 auto PartId = std::next(I);
4927 auto TaskT = std::next(I, 4);
4928 // Check if the task is final
4929 if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
4930 // If the condition constant folds and can be elided, try to avoid emitting
4931 // the condition and the dead arm of the if/else.
4932 const Expr *Cond = Clause->getCondition();
4933 bool CondConstant;
4934 if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
4935 Data.Final.setInt(CondConstant);
4936 else
4937 Data.Final.setPointer(EvaluateExprAsBool(Cond));
4938 } else {
4939 // By default the task is not final.
4940 Data.Final.setInt(/*IntVal=*/false);
4941 }
4942 // Check if the task has 'priority' clause.
4943 if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {
4944 const Expr *Prio = Clause->getPriority();
4945 Data.Priority.setInt(/*IntVal=*/true);
4946 Data.Priority.setPointer(EmitScalarConversion(
4947 EmitScalarExpr(Prio), Prio->getType(),
4948 getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1),
4949 Prio->getExprLoc()));
4950 }
4951 // The first function argument for tasks is a thread id, the second one is a
4952 // part id (0 for tied tasks, >=0 for untied task).
4953 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
4954 // Get list of private variables.
4955 for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
4956 auto IRef = C->varlist_begin();
4957 for (const Expr *IInit : C->private_copies()) {
4958 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4959 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4960 Data.PrivateVars.push_back(*IRef);
4961 Data.PrivateCopies.push_back(IInit);
4962 }
4963 ++IRef;
4964 }
4965 }
4966 EmittedAsPrivate.clear();
4967 // Get list of firstprivate variables.
4968 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
4969 auto IRef = C->varlist_begin();
4970 auto IElemInitRef = C->inits().begin();
4971 for (const Expr *IInit : C->private_copies()) {
4972 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4973 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4974 Data.FirstprivateVars.push_back(*IRef);
4975 Data.FirstprivateCopies.push_back(IInit);
4976 Data.FirstprivateInits.push_back(*IElemInitRef);
4977 }
4978 ++IRef;
4979 ++IElemInitRef;
4980 }
4981 }
4982 // Get list of lastprivate variables (for taskloops).
4983 llvm::MapVector<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
4984 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
4985 auto IRef = C->varlist_begin();
4986 auto ID = C->destination_exprs().begin();
4987 for (const Expr *IInit : C->private_copies()) {
4988 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
4989 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
4990 Data.LastprivateVars.push_back(*IRef);
4991 Data.LastprivateCopies.push_back(IInit);
4992 }
4993 LastprivateDstsOrigs.insert(
4994 std::make_pair(cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
4995 cast<DeclRefExpr>(*IRef)));
4996 ++IRef;
4997 ++ID;
4998 }
4999 }
5002 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
5003 Data.ReductionVars.append(C->varlist_begin(), C->varlist_end());
5004 Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
5005 Data.ReductionCopies.append(C->privates().begin(), C->privates().end());
5006 Data.ReductionOps.append(C->reduction_ops().begin(),
5007 C->reduction_ops().end());
5008 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
5009 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
5010 }
5011 Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
5012 *this, S.getBeginLoc(), LHSs, RHSs, Data);
5013 // Build list of dependences.
5015 // Get list of local vars for untied tasks.
5016 if (!Data.Tied) {
5017 CheckVarsEscapingUntiedTaskDeclContext Checker;
5018 Checker.Visit(S.getInnermostCapturedStmt()->getCapturedStmt());
5019 Data.PrivateLocals.append(Checker.getPrivateDecls().begin(),
5020 Checker.getPrivateDecls().end());
5021 }
5022 auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
5023 CapturedRegion](CodeGenFunction &CGF,
5024 PrePostActionTy &Action) {
5025 llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
5026 std::pair<Address, Address>>
5027 UntiedLocalVars;
5028 // Set proper addresses for generated private copies.
5030 // Generate debug info for variables present in shared clause.
5031 if (auto *DI = CGF.getDebugInfo()) {
5032 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields =
5033 CGF.CapturedStmtInfo->getCaptureFields();
5034 llvm::Value *ContextValue = CGF.CapturedStmtInfo->getContextValue();
5035 if (CaptureFields.size() && ContextValue) {
5036 unsigned CharWidth = CGF.getContext().getCharWidth();
5037 // The shared variables are packed together as members of structure.
5038 // So the address of each shared variable can be computed by adding
5039 // offset of it (within record) to the base address of record. For each
5040 // shared variable, debug intrinsic llvm.dbg.declare is generated with
5041 // appropriate expressions (DIExpression).
5042 // Ex:
5043 // %12 = load %struct.anon*, %struct.anon** %__context.addr.i
5044 // call void @llvm.dbg.declare(metadata %struct.anon* %12,
5045 // metadata !svar1,
5046 // metadata !DIExpression(DW_OP_deref))
5047 // call void @llvm.dbg.declare(metadata %struct.anon* %12,
5048 // metadata !svar2,
5049 // metadata !DIExpression(DW_OP_plus_uconst, 8, DW_OP_deref))
5050 for (auto It = CaptureFields.begin(); It != CaptureFields.end(); ++It) {
5051 const VarDecl *SharedVar = It->first;
5052 RecordDecl *CaptureRecord = It->second->getParent();
5053 const ASTRecordLayout &Layout =
5054 CGF.getContext().getASTRecordLayout(CaptureRecord);
5055 unsigned Offset =
5056 Layout.getFieldOffset(It->second->getFieldIndex()) / CharWidth;
5057 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
5058 (void)DI->EmitDeclareOfAutoVariable(SharedVar, ContextValue,
5059 CGF.Builder, false);
5060 // Get the call dbg.declare instruction we just created and update
5061 // its DIExpression to add offset to base address.
5062 auto UpdateExpr = [](llvm::LLVMContext &Ctx, auto *Declare,
5063 unsigned Offset) {
5065 // Add offset to the base address if non zero.
5066 if (Offset) {
5067 Ops.push_back(llvm::dwarf::DW_OP_plus_uconst);
5068 Ops.push_back(Offset);
5069 }
5070 Ops.push_back(llvm::dwarf::DW_OP_deref);
5071 Declare->setExpression(llvm::DIExpression::get(Ctx, Ops));
5072 };
5073 llvm::Instruction &Last = CGF.Builder.GetInsertBlock()->back();
5074 if (auto DDI = dyn_cast<llvm::DbgVariableIntrinsic>(&Last))
5075 UpdateExpr(DDI->getContext(), DDI, Offset);
5076 // If we're emitting using the new debug info format into a block
5077 // without a terminator, the record will be "trailing".
5078 assert(!Last.isTerminator() && "unexpected terminator");
5079 if (auto *Marker =
5080 CGF.Builder.GetInsertBlock()->getTrailingDbgRecords()) {
5081 for (llvm::DbgVariableRecord &DVR : llvm::reverse(
5082 llvm::filterDbgVars(Marker->getDbgRecordRange()))) {
5083 UpdateExpr(Last.getContext(), &DVR, Offset);
5084 break;
5085 }
5086 }
5087 }
5088 }
5089 }
5091 if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||
5092 !Data.LastprivateVars.empty() || !Data.PrivateLocals.empty()) {
5093 enum { PrivatesParam = 2, CopyFnParam = 3 };
5094 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
5095 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
5096 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
5097 CS->getCapturedDecl()->getParam(PrivatesParam)));
5098 // Map privates.
5102 CallArgs.push_back(PrivatesPtr);
5103 ParamTypes.push_back(PrivatesPtr->getType());
5104 for (const Expr *E : Data.PrivateVars) {
5105 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5106 RawAddress PrivatePtr = CGF.CreateMemTemp(
5107 CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
5108 PrivatePtrs.emplace_back(VD, PrivatePtr);
5109 CallArgs.push_back(PrivatePtr.getPointer());
5110 ParamTypes.push_back(PrivatePtr.getType());
5111 }
5112 for (const Expr *E : Data.FirstprivateVars) {
5113 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5114 RawAddress PrivatePtr =
5115 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
5116 ".firstpriv.ptr.addr");
5117 PrivatePtrs.emplace_back(VD, PrivatePtr);
5118 FirstprivatePtrs.emplace_back(VD, PrivatePtr);
5119 CallArgs.push_back(PrivatePtr.getPointer());
5120 ParamTypes.push_back(PrivatePtr.getType());
5121 }
5122 for (const Expr *E : Data.LastprivateVars) {
5123 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5124 RawAddress PrivatePtr =
5125 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
5126 ".lastpriv.ptr.addr");
5127 PrivatePtrs.emplace_back(VD, PrivatePtr);
5128 CallArgs.push_back(PrivatePtr.getPointer());
5129 ParamTypes.push_back(PrivatePtr.getType());
5130 }
5131 for (const VarDecl *VD : Data.PrivateLocals) {
5133 if (VD->getType()->isLValueReferenceType())
5134 Ty = CGF.getContext().getPointerType(Ty);
5135 if (isAllocatableDecl(VD))
5136 Ty = CGF.getContext().getPointerType(Ty);
5137 RawAddress PrivatePtr = CGF.CreateMemTemp(
5138 CGF.getContext().getPointerType(Ty), ".local.ptr.addr");
5139 auto Result = UntiedLocalVars.insert(
5140 std::make_pair(VD, std::make_pair(PrivatePtr, Address::invalid())));
5141 // If key exists update in place.
5142 if (Result.second == false)
5143 *Result.first = std::make_pair(
5144 VD, std::make_pair(PrivatePtr, Address::invalid()));
5145 CallArgs.push_back(PrivatePtr.getPointer());
5146 ParamTypes.push_back(PrivatePtr.getType());
5147 }
5148 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),
5149 ParamTypes, /*isVarArg=*/false);
5150 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
5151 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
5152 for (const auto &Pair : LastprivateDstsOrigs) {
5153 const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
5154 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(OrigVD),
5155 /*RefersToEnclosingVariableOrCapture=*/
5156 CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr,
5157 Pair.second->getType(), VK_LValue,
5158 Pair.second->getExprLoc());
5159 Scope.addPrivate(Pair.first, CGF.EmitLValue(&DRE).getAddress());
5160 }
5161 for (const auto &Pair : PrivatePtrs) {
5162 Address Replacement = Address(
5163 CGF.Builder.CreateLoad(Pair.second),
5164 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
5165 CGF.getContext().getDeclAlign(Pair.first));
5166 Scope.addPrivate(Pair.first, Replacement);
5167 if (auto *DI = CGF.getDebugInfo())
5168 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
5169 (void)DI->EmitDeclareOfAutoVariable(
5170 Pair.first, Pair.second.getBasePointer(), CGF.Builder,
5171 /*UsePointerValue*/ true);
5172 }
5173 // Adjust mapping for internal locals by mapping actual memory instead of
5174 // a pointer to this memory.
5175 for (auto &Pair : UntiedLocalVars) {
5176 QualType VDType = Pair.first->getType().getNonReferenceType();
5177 if (Pair.first->getType()->isLValueReferenceType())
5178 VDType = CGF.getContext().getPointerType(VDType);
5179 if (isAllocatableDecl(Pair.first)) {
5180 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
5181 Address Replacement(
5182 Ptr,
5183 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(VDType)),
5184 CGF.getPointerAlign());
5185 Pair.second.first = Replacement;
5186 Ptr = CGF.Builder.CreateLoad(Replacement);
5187 Replacement = Address(Ptr, CGF.ConvertTypeForMem(VDType),
5188 CGF.getContext().getDeclAlign(Pair.first));
5189 Pair.second.second = Replacement;
5190 } else {
5191 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
5192 Address Replacement(Ptr, CGF.ConvertTypeForMem(VDType),
5193 CGF.getContext().getDeclAlign(Pair.first));
5194 Pair.second.first = Replacement;
5195 }
5196 }
5197 }
5198 if (Data.Reductions) {
5199 OMPPrivateScope FirstprivateScope(CGF);
5200 for (const auto &Pair : FirstprivatePtrs) {
5201 Address Replacement(
5202 CGF.Builder.CreateLoad(Pair.second),
5203 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
5204 CGF.getContext().getDeclAlign(Pair.first));
5205 FirstprivateScope.addPrivate(Pair.first, Replacement);
5206 }
5207 (void)FirstprivateScope.Privatize();
5208 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
5209 ReductionCodeGen RedCG(Data.ReductionVars, Data.ReductionVars,
5210 Data.ReductionCopies, Data.ReductionOps);
5211 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
5212 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
5213 for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
5214 RedCG.emitSharedOrigLValue(CGF, Cnt);
5215 RedCG.emitAggregateType(CGF, Cnt);
5216 // FIXME: This must removed once the runtime library is fixed.
5217 // Emit required threadprivate variables for
5218 // initializer/combiner/finalizer.
5219 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
5220 RedCG, Cnt);
5221 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
5222 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
5223 Replacement = Address(
5224 CGF.EmitScalarConversion(Replacement.emitRawPointer(CGF),
5225 CGF.getContext().VoidPtrTy,
5226 CGF.getContext().getPointerType(
5227 Data.ReductionCopies[Cnt]->getType()),
5228 Data.ReductionCopies[Cnt]->getExprLoc()),
5229 CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()),
5230 Replacement.getAlignment());
5231 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
5232 Scope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);
5233 }
5234 }
5235 // Privatize all private variables except for in_reduction items.
5236 (void)Scope.Privatize();
5240 SmallVector<const Expr *, 4> TaskgroupDescriptors;
5241 for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
5242 auto IPriv = C->privates().begin();
5243 auto IRed = C->reduction_ops().begin();
5244 auto ITD = C->taskgroup_descriptors().begin();
5245 for (const Expr *Ref : C->varlist()) {
5246 InRedVars.emplace_back(Ref);
5247 InRedPrivs.emplace_back(*IPriv);
5248 InRedOps.emplace_back(*IRed);
5249 TaskgroupDescriptors.emplace_back(*ITD);
5250 std::advance(IPriv, 1);
5251 std::advance(IRed, 1);
5252 std::advance(ITD, 1);
5253 }
5254 }
5255 // Privatize in_reduction items here, because taskgroup descriptors must be
5256 // privatized earlier.
5257 OMPPrivateScope InRedScope(CGF);
5258 if (!InRedVars.empty()) {
5259 ReductionCodeGen RedCG(InRedVars, InRedVars, InRedPrivs, InRedOps);
5260 for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
5261 RedCG.emitSharedOrigLValue(CGF, Cnt);
5262 RedCG.emitAggregateType(CGF, Cnt);
5263 // The taskgroup descriptor variable is always implicit firstprivate and
5264 // privatized already during processing of the firstprivates.
5265 // FIXME: This must removed once the runtime library is fixed.
5266 // Emit required threadprivate variables for
5267 // initializer/combiner/finalizer.
5268 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
5269 RedCG, Cnt);
5270 llvm::Value *ReductionsPtr;
5271 if (const Expr *TRExpr = TaskgroupDescriptors[Cnt]) {
5272 ReductionsPtr = CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr),
5273 TRExpr->getExprLoc());
5274 } else {
5275 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5276 }
5277 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
5278 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
5279 Replacement = Address(
5280 CGF.EmitScalarConversion(
5281 Replacement.emitRawPointer(CGF), CGF.getContext().VoidPtrTy,
5282 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
5283 InRedPrivs[Cnt]->getExprLoc()),
5284 CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()),
5285 Replacement.getAlignment());
5286 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
5287 InRedScope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);
5288 }
5289 }
5290 (void)InRedScope.Privatize();
5291
5293 UntiedLocalVars);
5294 Action.Enter(CGF);
5295 BodyGen(CGF);
5296 };
5298 llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
5299 S, *I, *PartId, *TaskT, EKind, CodeGen, Data.Tied, Data.NumberOfParts);
5300 OMPLexicalScope Scope(*this, S, std::nullopt,
5301 !isOpenMPParallelDirective(EKind) &&
5302 !isOpenMPSimdDirective(EKind));
5303 TaskGen(*this, OutlinedFn, Data);
5304}
5305
5306static ImplicitParamDecl *
5308 QualType Ty, CapturedDecl *CD,
5309 SourceLocation Loc) {
5310 auto *OrigVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
5312 auto *OrigRef = DeclRefExpr::Create(
5314 /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
5315 auto *PrivateVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
5317 auto *PrivateRef = DeclRefExpr::Create(
5318 C, NestedNameSpecifierLoc(), SourceLocation(), PrivateVD,
5319 /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
5320 QualType ElemType = C.getBaseElementType(Ty);
5321 auto *InitVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, ElemType,
5323 auto *InitRef = DeclRefExpr::Create(
5325 /*RefersToEnclosingVariableOrCapture=*/false, Loc, ElemType, VK_LValue);
5326 PrivateVD->setInitStyle(VarDecl::CInit);
5327 PrivateVD->setInit(ImplicitCastExpr::Create(C, ElemType, CK_LValueToRValue,
5328 InitRef, /*BasePath=*/nullptr,
5330 Data.FirstprivateVars.emplace_back(OrigRef);
5331 Data.FirstprivateCopies.emplace_back(PrivateRef);
5332 Data.FirstprivateInits.emplace_back(InitRef);
5333 return OrigVD;
5334}
5335
5337 const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen,
5338 OMPTargetDataInfo &InputInfo) {
5339 // Emit outlined function for task construct.
5340 const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
5341 Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
5342 CanQualType SharedsTy =
5344 auto I = CS->getCapturedDecl()->param_begin();
5345 auto PartId = std::next(I);
5346 auto TaskT = std::next(I, 4);
5348 // The task is not final.
5349 Data.Final.setInt(/*IntVal=*/false);
5350 // Get list of firstprivate variables.
5351 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
5352 auto IRef = C->varlist_begin();
5353 auto IElemInitRef = C->inits().begin();
5354 for (auto *IInit : C->private_copies()) {
5355 Data.FirstprivateVars.push_back(*IRef);
5356 Data.FirstprivateCopies.push_back(IInit);
5357 Data.FirstprivateInits.push_back(*IElemInitRef);
5358 ++IRef;
5359 ++IElemInitRef;
5360 }
5361 }
5364 for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
5365 Data.ReductionVars.append(C->varlist_begin(), C->varlist_end());
5366 Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
5367 Data.ReductionCopies.append(C->privates().begin(), C->privates().end());
5368 Data.ReductionOps.append(C->reduction_ops().begin(),
5369 C->reduction_ops().end());
5370 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
5371 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
5372 }
5373 OMPPrivateScope TargetScope(*this);
5374 VarDecl *BPVD = nullptr;
5375 VarDecl *PVD = nullptr;
5376 VarDecl *SVD = nullptr;
5377 VarDecl *MVD = nullptr;
5378 if (InputInfo.NumberOfTargetItems > 0) {
5379 auto *CD = CapturedDecl::Create(
5380 getContext(), getContext().getTranslationUnitDecl(), /*NumParams=*/0);
5381 llvm::APInt ArrSize(/*numBits=*/32, InputInfo.NumberOfTargetItems);
5382 QualType BaseAndPointerAndMapperType = getContext().getConstantArrayType(
5383 getContext().VoidPtrTy, ArrSize, nullptr, ArraySizeModifier::Normal,
5384 /*IndexTypeQuals=*/0);
5386 getContext(), Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
5388 getContext(), Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
5390 getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1),
5391 ArrSize, nullptr, ArraySizeModifier::Normal,
5392 /*IndexTypeQuals=*/0);
5393 SVD = createImplicitFirstprivateForType(getContext(), Data, SizesType, CD,
5394 S.getBeginLoc());
5395 TargetScope.addPrivate(BPVD, InputInfo.BasePointersArray);
5396 TargetScope.addPrivate(PVD, InputInfo.PointersArray);
5397 TargetScope.addPrivate(SVD, InputInfo.SizesArray);
5398 // If there is no user-defined mapper, the mapper array will be nullptr. In
5399 // this case, we don't need to privatize it.
5400 if (!isa_and_nonnull<llvm::ConstantPointerNull>(
5401 InputInfo.MappersArray.emitRawPointer(*this))) {
5403 getContext(), Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
5404 TargetScope.addPrivate(MVD, InputInfo.MappersArray);
5405 }
5406 }
5407 (void)TargetScope.Privatize();
5410 auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD, MVD, EKind,
5411 &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) {
5412 // Set proper addresses for generated private copies.
5414 if (!Data.FirstprivateVars.empty()) {
5415 enum { PrivatesParam = 2, CopyFnParam = 3 };
5416 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
5417 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
5418 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
5419 CS->getCapturedDecl()->getParam(PrivatesParam)));
5420 // Map privates.
5424 CallArgs.push_back(PrivatesPtr);
5425 ParamTypes.push_back(PrivatesPtr->getType());
5426 for (const Expr *E : Data.FirstprivateVars) {
5427 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5428 RawAddress PrivatePtr =
5429 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
5430 ".firstpriv.ptr.addr");
5431 PrivatePtrs.emplace_back(VD, PrivatePtr);
5432 CallArgs.push_back(PrivatePtr.getPointer());
5433 ParamTypes.push_back(PrivatePtr.getType());
5434 }
5435 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),
5436 ParamTypes, /*isVarArg=*/false);
5437 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
5438 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
5439 for (const auto &Pair : PrivatePtrs) {
5440 Address Replacement(
5441 CGF.Builder.CreateLoad(Pair.second),
5442 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),
5443 CGF.getContext().getDeclAlign(Pair.first));
5444 Scope.addPrivate(Pair.first, Replacement);
5445 }
5446 }
5447 CGF.processInReduction(S, Data, CGF, CS, Scope);
5448 if (InputInfo.NumberOfTargetItems > 0) {
5449 InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP(
5450 CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0);
5451 InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP(
5452 CGF.GetAddrOfLocalVar(PVD), /*Index=*/0);
5453 InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP(
5454 CGF.GetAddrOfLocalVar(SVD), /*Index=*/0);
5455 // If MVD is nullptr, the mapper array is not privatized
5456 if (MVD)
5457 InputInfo.MappersArray = CGF.Builder.CreateConstArrayGEP(
5458 CGF.GetAddrOfLocalVar(MVD), /*Index=*/0);
5459 }
5460
5461 Action.Enter(CGF);
5462 OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false);
5463 auto *TL = S.getSingleClause<OMPThreadLimitClause>();
5464 if (CGF.CGM.getLangOpts().OpenMP >= 51 &&
5465 needsTaskBasedThreadLimit(EKind) && TL) {
5466 // Emit __kmpc_set_thread_limit() to set the thread_limit for the task
5467 // enclosing this target region. This will indirectly set the thread_limit
5468 // for every applicable construct within target region.
5469 CGF.CGM.getOpenMPRuntime().emitThreadLimitClause(
5470 CGF, TL->getThreadLimit().front(), S.getBeginLoc());
5471 }
5472 BodyGen(CGF);
5473 };
5474 llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
5475 S, *I, *PartId, *TaskT, EKind, CodeGen, /*Tied=*/true,
5476 Data.NumberOfParts);
5477 llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0);
5478 IntegerLiteral IfCond(getContext(), TrueOrFalse,
5479 getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
5480 SourceLocation());
5481 CGM.getOpenMPRuntime().emitTaskCall(*this, S.getBeginLoc(), S, OutlinedFn,
5482 SharedsTy, CapturedStruct, &IfCond, Data);
5483}
5484
5487 CodeGenFunction &CGF,
5488 const CapturedStmt *CS,
5491 if (Data.Reductions) {
5492 OpenMPDirectiveKind CapturedRegion = EKind;
5493 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
5494 ReductionCodeGen RedCG(Data.ReductionVars, Data.ReductionVars,
5495 Data.ReductionCopies, Data.ReductionOps);
5496 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
5498 for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
5499 RedCG.emitSharedOrigLValue(CGF, Cnt);
5500 RedCG.emitAggregateType(CGF, Cnt);
5501 // FIXME: This must removed once the runtime library is fixed.
5502 // Emit required threadprivate variables for
5503 // initializer/combiner/finalizer.
5504 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
5505 RedCG, Cnt);
5507 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
5508 Replacement = Address(
5509 CGF.EmitScalarConversion(Replacement.emitRawPointer(CGF),
5510 CGF.getContext().VoidPtrTy,
5512 Data.ReductionCopies[Cnt]->getType()),
5513 Data.ReductionCopies[Cnt]->getExprLoc()),
5514 CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()),
5515 Replacement.getAlignment());
5516 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
5517 Scope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);
5518 }
5519 }
5520 (void)Scope.Privatize();
5524 SmallVector<const Expr *, 4> TaskgroupDescriptors;
5525 for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
5526 auto IPriv = C->privates().begin();
5527 auto IRed = C->reduction_ops().begin();
5528 auto ITD = C->taskgroup_descriptors().begin();
5529 for (const Expr *Ref : C->varlist()) {
5530 InRedVars.emplace_back(Ref);
5531 InRedPrivs.emplace_back(*IPriv);
5532 InRedOps.emplace_back(*IRed);
5533 TaskgroupDescriptors.emplace_back(*ITD);
5534 std::advance(IPriv, 1);
5535 std::advance(IRed, 1);
5536 std::advance(ITD, 1);
5537 }
5538 }
5539 OMPPrivateScope InRedScope(CGF);
5540 if (!InRedVars.empty()) {
5541 ReductionCodeGen RedCG(InRedVars, InRedVars, InRedPrivs, InRedOps);
5542 for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
5543 RedCG.emitSharedOrigLValue(CGF, Cnt);
5544 RedCG.emitAggregateType(CGF, Cnt);
5545 // FIXME: This must removed once the runtime library is fixed.
5546 // Emit required threadprivate variables for
5547 // initializer/combiner/finalizer.
5548 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
5549 RedCG, Cnt);
5550 llvm::Value *ReductionsPtr;
5551 if (const Expr *TRExpr = TaskgroupDescriptors[Cnt]) {
5552 ReductionsPtr =
5553 CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr), TRExpr->getExprLoc());
5554 } else {
5555 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5556 }
5558 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
5559 Replacement = Address(
5561 Replacement.emitRawPointer(CGF), CGF.getContext().VoidPtrTy,
5562 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
5563 InRedPrivs[Cnt]->getExprLoc()),
5564 CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()),
5565 Replacement.getAlignment());
5566 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
5567 InRedScope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);
5568 }
5569 }
5570 (void)InRedScope.Privatize();
5571}
5572
5573void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) {
5574 // Emit outlined function for task construct.
5575 const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
5576 Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
5577 CanQualType SharedsTy =
5579 const Expr *IfCond = nullptr;
5580 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
5581 if (C->getNameModifier() == OMPD_unknown ||
5582 C->getNameModifier() == OMPD_task) {
5583 IfCond = C->getCondition();
5584 break;
5585 }
5586 }
5587
5589 // Check if we should emit tied or untied task.
5590 Data.Tied = !S.getSingleClause<OMPUntiedClause>();
5591 auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
5592 CGF.EmitStmt(CS->getCapturedStmt());
5593 };
5594 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
5595 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
5596 const OMPTaskDataTy &Data) {
5597 CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn,
5598 SharedsTy, CapturedStruct, IfCond,
5599 Data);
5600 };
5601 auto LPCRegion =
5603 EmitOMPTaskBasedDirective(S, OMPD_task, BodyGen, TaskGen, Data);
5604}
5605
5607 const OMPTaskyieldDirective &S) {
5608 CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getBeginLoc());
5609}
5610
5612 const OMPMessageClause *MC = S.getSingleClause<OMPMessageClause>();
5613 Expr *ME = MC ? MC->getMessageString() : nullptr;
5614 const OMPSeverityClause *SC = S.getSingleClause<OMPSeverityClause>();
5615 bool IsFatal = false;
5616 if (!SC || SC->getSeverityKind() == OMPC_SEVERITY_fatal)
5617 IsFatal = true;
5618 CGM.getOpenMPRuntime().emitErrorCall(*this, S.getBeginLoc(), ME, IsFatal);
5619}
5620
5621void CodeGenFunction::EmitOMPBarrierDirective(const OMPBarrierDirective &S) {
5622 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_barrier);
5623}
5624
5625void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S) {
5627 // Build list of dependences
5629 Data.HasNowaitClause = S.hasClausesOfKind<OMPNowaitClause>();
5630 CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getBeginLoc(), Data);
5631}
5632
5633static bool isSupportedByOpenMPIRBuilder(const OMPTaskgroupDirective &T) {
5634 return T.clauses().empty();
5635}
5636
5638 const OMPTaskgroupDirective &S) {
5639 OMPLexicalScope Scope(*this, S, OMPD_unknown);
5640 if (CGM.getLangOpts().OpenMPIRBuilder && isSupportedByOpenMPIRBuilder(S)) {
5641 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
5642 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
5643 InsertPointTy AllocaIP(AllocaInsertPt->getParent(),
5644 AllocaInsertPt->getIterator());
5645
5646 auto BodyGenCB = [&, this](InsertPointTy AllocaIP,
5647 InsertPointTy CodeGenIP) {
5648 Builder.restoreIP(CodeGenIP);
5649 EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
5650 return llvm::Error::success();
5651 };
5653 if (!CapturedStmtInfo)
5654 CapturedStmtInfo = &CapStmtInfo;
5655 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
5656 cantFail(OMPBuilder.createTaskgroup(Builder, AllocaIP, BodyGenCB));
5657 Builder.restoreIP(AfterIP);
5658 return;
5659 }
5660 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5661 Action.Enter(CGF);
5662 if (const Expr *E = S.getReductionRef()) {
5666 for (const auto *C : S.getClausesOfKind<OMPTaskReductionClause>()) {
5667 Data.ReductionVars.append(C->varlist_begin(), C->varlist_end());
5668 Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
5669 Data.ReductionCopies.append(C->privates().begin(), C->privates().end());
5670 Data.ReductionOps.append(C->reduction_ops().begin(),
5671 C->reduction_ops().end());
5672 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
5673 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
5674 }
5675 llvm::Value *ReductionDesc =
5676 CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getBeginLoc(),
5677 LHSs, RHSs, Data);
5678 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5679 CGF.EmitVarDecl(*VD);
5680 CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD),
5681 /*Volatile=*/false, E->getType());
5682 }
5683 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
5684 };
5685 CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getBeginLoc());
5686}
5687
5688void CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &S) {
5689 llvm::AtomicOrdering AO = S.getSingleClause<OMPFlushClause>()
5690 ? llvm::AtomicOrdering::NotAtomic
5691 : llvm::AtomicOrdering::AcquireRelease;
5692 CGM.getOpenMPRuntime().emitFlush(
5693 *this,
5694 [&S]() -> ArrayRef<const Expr *> {
5695 if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>())
5696 return llvm::ArrayRef(FlushClause->varlist_begin(),
5697 FlushClause->varlist_end());
5698 return {};
5699 }(),
5700 S.getBeginLoc(), AO);
5701}
5702
5703void CodeGenFunction::EmitOMPDepobjDirective(const OMPDepobjDirective &S) {
5704 const auto *DO = S.getSingleClause<OMPDepobjClause>();
5705 LValue DOLVal = EmitLValue(DO->getDepobj());
5706 if (const auto *DC = S.getSingleClause<OMPDependClause>()) {
5707 // Build list and emit dependences
5710 for (auto &Dep : Data.Dependences) {
5711 Address DepAddr = CGM.getOpenMPRuntime().emitDepobjDependClause(
5712 *this, Dep, DC->getBeginLoc());
5713 EmitStoreOfScalar(DepAddr.emitRawPointer(*this), DOLVal);
5714 }
5715 return;
5716 }
5717 if (const auto *DC = S.getSingleClause<OMPDestroyClause>()) {
5718 CGM.getOpenMPRuntime().emitDestroyClause(*this, DOLVal, DC->getBeginLoc());
5719 return;
5720 }
5721 if (const auto *UC = S.getSingleClause<OMPUpdateClause>()) {
5722 CGM.getOpenMPRuntime().emitUpdateClause(
5723 *this, DOLVal, UC->getDependencyKind(), UC->getBeginLoc());
5724 return;
5725 }
5726}
5727
5730 return;
5732 bool IsInclusive = S.hasClausesOfKind<OMPInclusiveClause>();
5737 SmallVector<const Expr *, 4> ReductionOps;
5739 SmallVector<const Expr *, 4> CopyArrayTemps;
5740 SmallVector<const Expr *, 4> CopyArrayElems;
5741 for (const auto *C : ParentDir.getClausesOfKind<OMPReductionClause>()) {
5742 if (C->getModifier() != OMPC_REDUCTION_inscan)
5743 continue;
5744 Shareds.append(C->varlist_begin(), C->varlist_end());
5745 Privates.append(C->privates().begin(), C->privates().end());
5746 LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
5747 RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
5748 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
5749 CopyOps.append(C->copy_ops().begin(), C->copy_ops().end());
5750 CopyArrayTemps.append(C->copy_array_temps().begin(),
5751 C->copy_array_temps().end());
5752 CopyArrayElems.append(C->copy_array_elems().begin(),
5753 C->copy_array_elems().end());
5754 }
5755 if (ParentDir.getDirectiveKind() == OMPD_simd ||
5756 (getLangOpts().OpenMPSimd &&
5757 isOpenMPSimdDirective(ParentDir.getDirectiveKind()))) {
5758 // For simd directive and simd-based directives in simd only mode, use the
5759 // following codegen:
5760 // int x = 0;
5761 // #pragma omp simd reduction(inscan, +: x)
5762 // for (..) {
5763 // <first part>
5764 // #pragma omp scan inclusive(x)
5765 // <second part>
5766 // }
5767 // is transformed to:
5768 // int x = 0;
5769 // for (..) {
5770 // int x_priv = 0;
5771 // <first part>
5772 // x = x_priv + x;
5773 // x_priv = x;
5774 // <second part>
5775 // }
5776 // and
5777 // int x = 0;
5778 // #pragma omp simd reduction(inscan, +: x)
5779 // for (..) {
5780 // <first part>
5781 // #pragma omp scan exclusive(x)
5782 // <second part>
5783 // }
5784 // to
5785 // int x = 0;
5786 // for (..) {
5787 // int x_priv = 0;
5788 // <second part>
5789 // int temp = x;
5790 // x = x_priv + x;
5791 // x_priv = temp;
5792 // <first part>
5793 // }
5794 llvm::BasicBlock *OMPScanReduce = createBasicBlock("omp.inscan.reduce");
5795 EmitBranch(IsInclusive
5796 ? OMPScanReduce
5797 : BreakContinueStack.back().ContinueBlock.getBlock());
5799 {
5800 // New scope for correct construction/destruction of temp variables for
5801 // exclusive scan.
5802 LexicalScope Scope(*this, S.getSourceRange());
5804 EmitBlock(OMPScanReduce);
5805 if (!IsInclusive) {
5806 // Create temp var and copy LHS value to this temp value.
5807 // TMP = LHS;
5808 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
5809 const Expr *PrivateExpr = Privates[I];
5810 const Expr *TempExpr = CopyArrayTemps[I];
5812 *cast<VarDecl>(cast<DeclRefExpr>(TempExpr)->getDecl()));
5813 LValue DestLVal = EmitLValue(TempExpr);
5814 LValue SrcLVal = EmitLValue(LHSs[I]);
5815 EmitOMPCopy(PrivateExpr->getType(), DestLVal.getAddress(),
5816 SrcLVal.getAddress(),
5817 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
5818 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
5819 CopyOps[I]);
5820 }
5821 }
5822 CGM.getOpenMPRuntime().emitReduction(
5823 *this, ParentDir.getEndLoc(), Privates, LHSs, RHSs, ReductionOps,
5824 {/*WithNowait=*/true, /*SimpleReduction=*/true,
5825 /*IsPrivateVarReduction*/ {}, OMPD_simd});
5826 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
5827 const Expr *PrivateExpr = Privates[I];
5828 LValue DestLVal;
5829 LValue SrcLVal;
5830 if (IsInclusive) {
5831 DestLVal = EmitLValue(RHSs[I]);
5832 SrcLVal = EmitLValue(LHSs[I]);
5833 } else {
5834 const Expr *TempExpr = CopyArrayTemps[I];
5835 DestLVal = EmitLValue(RHSs[I]);
5836 SrcLVal = EmitLValue(TempExpr);
5837 }
5839 PrivateExpr->getType(), DestLVal.getAddress(), SrcLVal.getAddress(),
5840 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
5841 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()), CopyOps[I]);
5842 }
5843 }
5845 OMPScanExitBlock = IsInclusive
5846 ? BreakContinueStack.back().ContinueBlock.getBlock()
5847 : OMPScanReduce;
5849 return;
5850 }
5851 if (!IsInclusive) {
5852 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
5854 }
5855 if (OMPFirstScanLoop) {
5856 // Emit buffer[i] = red; at the end of the input phase.
5857 const auto *IVExpr = cast<OMPLoopDirective>(ParentDir)
5858 .getIterationVariable()
5859 ->IgnoreParenImpCasts();
5860 LValue IdxLVal = EmitLValue(IVExpr);
5861 llvm::Value *IdxVal = EmitLoadOfScalar(IdxLVal, IVExpr->getExprLoc());
5862 IdxVal = Builder.CreateIntCast(IdxVal, SizeTy, /*isSigned=*/false);
5863 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
5864 const Expr *PrivateExpr = Privates[I];
5865 const Expr *OrigExpr = Shareds[I];
5866 const Expr *CopyArrayElem = CopyArrayElems[I];
5867 OpaqueValueMapping IdxMapping(
5868 *this,
5870 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
5871 RValue::get(IdxVal));
5872 LValue DestLVal = EmitLValue(CopyArrayElem);
5873 LValue SrcLVal = EmitLValue(OrigExpr);
5875 PrivateExpr->getType(), DestLVal.getAddress(), SrcLVal.getAddress(),
5876 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
5877 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()), CopyOps[I]);
5878 }
5879 }
5880 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
5881 if (IsInclusive) {
5883 EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
5884 }
5886 if (!OMPFirstScanLoop) {
5887 // Emit red = buffer[i]; at the entrance to the scan phase.
5888 const auto *IVExpr = cast<OMPLoopDirective>(ParentDir)
5889 .getIterationVariable()
5890 ->IgnoreParenImpCasts();
5891 LValue IdxLVal = EmitLValue(IVExpr);
5892 llvm::Value *IdxVal = EmitLoadOfScalar(IdxLVal, IVExpr->getExprLoc());
5893 IdxVal = Builder.CreateIntCast(IdxVal, SizeTy, /*isSigned=*/false);
5894 llvm::BasicBlock *ExclusiveExitBB = nullptr;
5895 if (!IsInclusive) {
5896 llvm::BasicBlock *ContBB = createBasicBlock("omp.exclusive.dec");
5897 ExclusiveExitBB = createBasicBlock("omp.exclusive.copy.exit");
5898 llvm::Value *Cmp = Builder.CreateIsNull(IdxVal);
5899 Builder.CreateCondBr(Cmp, ExclusiveExitBB, ContBB);
5900 EmitBlock(ContBB);
5901 // Use idx - 1 iteration for exclusive scan.
5902 IdxVal = Builder.CreateNUWSub(IdxVal, llvm::ConstantInt::get(SizeTy, 1));
5903 }
5904 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
5905 const Expr *PrivateExpr = Privates[I];
5906 const Expr *OrigExpr = Shareds[I];
5907 const Expr *CopyArrayElem = CopyArrayElems[I];
5908 OpaqueValueMapping IdxMapping(
5909 *this,
5911 cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
5912 RValue::get(IdxVal));
5913 LValue SrcLVal = EmitLValue(CopyArrayElem);
5914 LValue DestLVal = EmitLValue(OrigExpr);
5916 PrivateExpr->getType(), DestLVal.getAddress(), SrcLVal.getAddress(),
5917 cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
5918 cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()), CopyOps[I]);
5919 }
5920 if (!IsInclusive) {
5921 EmitBlock(ExclusiveExitBB);
5922 }
5923 }
5927}
5928
5930 const CodeGenLoopTy &CodeGenLoop,
5931 Expr *IncExpr) {
5932 // Emit the loop iteration variable.
5933 const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
5934 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
5935 EmitVarDecl(*IVDecl);
5936
5937 // Emit the iterations count variable.
5938 // If it is not a variable, Sema decided to calculate iterations count on each
5939 // iteration (e.g., it is foldable into a constant).
5940 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
5941 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
5942 // Emit calculation of the iterations count.
5943 EmitIgnoredExpr(S.getCalcLastIteration());
5944 }
5945
5946 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
5947
5948 bool HasLastprivateClause = false;
5949 // Check pre-condition.
5950 {
5951 OMPLoopScope PreInitScope(*this, S);
5952 // Skip the entire loop if we don't meet the precondition.
5953 // If the condition constant folds and can be elided, avoid emitting the
5954 // whole loop.
5955 bool CondConstant;
5956 llvm::BasicBlock *ContBlock = nullptr;
5957 if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
5958 if (!CondConstant)
5959 return;
5960 } else {
5961 llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
5962 ContBlock = createBasicBlock("omp.precond.end");
5963 emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
5964 getProfileCount(&S));
5965 EmitBlock(ThenBlock);
5967 }
5968
5969 emitAlignedClause(*this, S);
5970 // Emit 'then' code.
5971 {
5972 // Emit helper vars inits.
5973
5975 *this, cast<DeclRefExpr>(
5976 (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
5977 ? S.getCombinedLowerBoundVariable()
5978 : S.getLowerBoundVariable())));
5980 *this, cast<DeclRefExpr>(
5981 (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
5982 ? S.getCombinedUpperBoundVariable()
5983 : S.getUpperBoundVariable())));
5984 LValue ST =
5985 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
5986 LValue IL =
5987 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
5988
5989 OMPPrivateScope LoopScope(*this);
5990 if (EmitOMPFirstprivateClause(S, LoopScope)) {
5991 // Emit implicit barrier to synchronize threads and avoid data races
5992 // on initialization of firstprivate variables and post-update of
5993 // lastprivate variables.
5994 CGM.getOpenMPRuntime().emitBarrierCall(
5995 *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
5996 /*ForceSimpleCall=*/true);
5997 }
5998 EmitOMPPrivateClause(S, LoopScope);
5999 if (isOpenMPSimdDirective(S.getDirectiveKind()) &&
6000 !isOpenMPParallelDirective(S.getDirectiveKind()) &&
6001 !isOpenMPTeamsDirective(S.getDirectiveKind()))
6002 EmitOMPReductionClauseInit(S, LoopScope);
6003 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
6004 EmitOMPPrivateLoopCounters(S, LoopScope);
6005 (void)LoopScope.Privatize();
6006 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
6007 CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);
6008
6009 // Detect the distribute schedule kind and chunk.
6010 llvm::Value *Chunk = nullptr;
6012 if (const auto *C = S.getSingleClause<OMPDistScheduleClause>()) {
6013 ScheduleKind = C->getDistScheduleKind();
6014 if (const Expr *Ch = C->getChunkSize()) {
6015 Chunk = EmitScalarExpr(Ch);
6016 Chunk = EmitScalarConversion(Chunk, Ch->getType(),
6017 S.getIterationVariable()->getType(),
6018 S.getBeginLoc());
6019 }
6020 } else {
6021 // Default behaviour for dist_schedule clause.
6022 CGM.getOpenMPRuntime().getDefaultDistScheduleAndChunk(
6023 *this, S, ScheduleKind, Chunk);
6024 }
6025 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
6026 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
6027
6028 // OpenMP [2.10.8, distribute Construct, Description]
6029 // If dist_schedule is specified, kind must be static. If specified,
6030 // iterations are divided into chunks of size chunk_size, chunks are
6031 // assigned to the teams of the league in a round-robin fashion in the
6032 // order of the team number. When no chunk_size is specified, the
6033 // iteration space is divided into chunks that are approximately equal
6034 // in size, and at most one chunk is distributed to each team of the
6035 // league. The size of the chunks is unspecified in this case.
6036 bool StaticChunked =
6037 RT.isStaticChunked(ScheduleKind, /* Chunked */ Chunk != nullptr) &&
6038 isOpenMPLoopBoundSharingDirective(S.getDirectiveKind());
6039 if (RT.isStaticNonchunked(ScheduleKind,
6040 /* Chunked */ Chunk != nullptr) ||
6041 StaticChunked) {
6043 IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(),
6044 LB.getAddress(), UB.getAddress(), ST.getAddress(),
6045 StaticChunked ? Chunk : nullptr);
6046 RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind,
6047 StaticInit);
6050 // UB = min(UB, GlobalUB);
6052 ? S.getCombinedEnsureUpperBound()
6053 : S.getEnsureUpperBound());
6054 // IV = LB;
6056 ? S.getCombinedInit()
6057 : S.getInit());
6058
6059 const Expr *Cond =
6060 isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
6061 ? S.getCombinedCond()
6062 : S.getCond();
6063
6064 if (StaticChunked)
6065 Cond = S.getCombinedDistCond();
6066
6067 // For static unchunked schedules generate:
6068 //
6069 // 1. For distribute alone, codegen
6070 // while (idx <= UB) {
6071 // BODY;
6072 // ++idx;
6073 // }
6074 //
6075 // 2. When combined with 'for' (e.g. as in 'distribute parallel for')
6076 // while (idx <= UB) {
6077 // <CodeGen rest of pragma>(LB, UB);
6078 // idx += ST;
6079 // }
6080 //
6081 // For static chunk one schedule generate:
6082 //
6083 // while (IV <= GlobalUB) {
6084 // <CodeGen rest of pragma>(LB, UB);
6085 // LB += ST;
6086 // UB += ST;
6087 // UB = min(UB, GlobalUB);
6088 // IV = LB;
6089 // }
6090 //
6092 *this, S,
6093 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
6094 if (isOpenMPSimdDirective(S.getDirectiveKind()))
6095 CGF.EmitOMPSimdInit(S);
6096 },
6097 [&S, &LoopScope, Cond, IncExpr, LoopExit, &CodeGenLoop,
6098 StaticChunked](CodeGenFunction &CGF, PrePostActionTy &) {
6099 CGF.EmitOMPInnerLoop(
6100 S, LoopScope.requiresCleanups(), Cond, IncExpr,
6101 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
6102 CodeGenLoop(CGF, S, LoopExit);
6103 },
6104 [&S, StaticChunked](CodeGenFunction &CGF) {
6105 if (StaticChunked) {
6106 CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound());
6107 CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound());
6108 CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound());
6109 CGF.EmitIgnoredExpr(S.getCombinedInit());
6110 }
6111 });
6112 });
6113 EmitBlock(LoopExit.getBlock());
6114 // Tell the runtime we are done.
6115 RT.emitForStaticFinish(*this, S.getEndLoc(), OMPD_distribute);
6116 } else {
6117 // Emit the outer loop, which requests its work chunk [LB..UB] from
6118 // runtime and runs the inner loop to process it.
6119 const OMPLoopArguments LoopArguments = {
6120 LB.getAddress(), UB.getAddress(), ST.getAddress(), IL.getAddress(),
6121 Chunk};
6122 EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
6123 CodeGenLoop);
6124 }
6125 if (isOpenMPSimdDirective(S.getDirectiveKind())) {
6126 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
6127 return CGF.Builder.CreateIsNotNull(
6128 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
6129 });
6130 }
6131 if (isOpenMPSimdDirective(S.getDirectiveKind()) &&
6132 !isOpenMPParallelDirective(S.getDirectiveKind()) &&
6133 !isOpenMPTeamsDirective(S.getDirectiveKind())) {
6134 EmitOMPReductionClauseFinal(S, OMPD_simd);
6135 // Emit post-update of the reduction variables if IsLastIter != 0.
6137 *this, S, [IL, &S](CodeGenFunction &CGF) {
6138 return CGF.Builder.CreateIsNotNull(
6139 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
6140 });
6141 }
6142 // Emit final copy of the lastprivate variables if IsLastIter != 0.
6143 if (HasLastprivateClause) {
6145 S, /*NoFinals=*/false,
6146 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
6147 }
6148 }
6149
6150 // We're now done with the loop, so jump to the continuation block.
6151 if (ContBlock) {
6152 EmitBranch(ContBlock);
6153 EmitBlock(ContBlock, true);
6154 }
6155 }
6156}
6157
6158// Pass OMPLoopDirective (instead of OMPDistributeDirective) to make this
6159// function available for "loop bind(teams)", which maps to "distribute".
6161 CodeGenFunction &CGF,
6162 CodeGenModule &CGM) {
6163 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
6165 };
6166 OMPLexicalScope Scope(CGF, S, OMPD_unknown);
6167 CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, CodeGen);
6168}
6169
6174
6175static llvm::Function *
6177 const OMPExecutableDirective &D) {
6178 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
6180 CGF.CapturedStmtInfo = &CapStmtInfo;
6181 llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S, D);
6182 Fn->setDoesNotRecurse();
6183 return Fn;
6184}
6185
6186template <typename T>
6187static void emitRestoreIP(CodeGenFunction &CGF, const T *C,
6188 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP,
6189 llvm::OpenMPIRBuilder &OMPBuilder) {
6190
6191 unsigned NumLoops = C->getNumLoops();
6193 /*DestWidth=*/64, /*Signed=*/1);
6195 for (unsigned I = 0; I < NumLoops; I++) {
6196 const Expr *CounterVal = C->getLoopData(I);
6197 assert(CounterVal);
6198 llvm::Value *StoreValue = CGF.EmitScalarConversion(
6199 CGF.EmitScalarExpr(CounterVal), CounterVal->getType(), Int64Ty,
6200 CounterVal->getExprLoc());
6201 StoreValues.emplace_back(StoreValue);
6202 }
6203 OMPDoacrossKind<T> ODK;
6204 bool IsDependSource = ODK.isSource(C);
6205 CGF.Builder.restoreIP(
6206 OMPBuilder.createOrderedDepend(CGF.Builder, AllocaIP, NumLoops,
6207 StoreValues, ".cnt.addr", IsDependSource));
6208}
6209
6210void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &S) {
6211 if (CGM.getLangOpts().OpenMPIRBuilder) {
6212 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
6213 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
6214
6215 if (S.hasClausesOfKind<OMPDependClause>() ||
6216 S.hasClausesOfKind<OMPDoacrossClause>()) {
6217 // The ordered directive with depend clause.
6218 assert(!S.hasAssociatedStmt() && "No associated statement must be in "
6219 "ordered depend|doacross construct.");
6220 InsertPointTy AllocaIP(AllocaInsertPt->getParent(),
6221 AllocaInsertPt->getIterator());
6222 for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
6223 emitRestoreIP(*this, DC, AllocaIP, OMPBuilder);
6224 for (const auto *DC : S.getClausesOfKind<OMPDoacrossClause>())
6225 emitRestoreIP(*this, DC, AllocaIP, OMPBuilder);
6226 } else {
6227 // The ordered directive with threads or simd clause, or without clause.
6228 // Without clause, it behaves as if the threads clause is specified.
6229 const auto *C = S.getSingleClause<OMPSIMDClause>();
6230
6231 auto FiniCB = [this](InsertPointTy IP) {
6233 return llvm::Error::success();
6234 };
6235
6236 auto BodyGenCB = [&S, C, this](InsertPointTy AllocaIP,
6237 InsertPointTy CodeGenIP) {
6238 Builder.restoreIP(CodeGenIP);
6239
6240 const CapturedStmt *CS = S.getInnermostCapturedStmt();
6241 if (C) {
6242 llvm::BasicBlock *FiniBB = splitBBWithSuffix(
6243 Builder, /*CreateBranch=*/false, ".ordered.after");
6245 GenerateOpenMPCapturedVars(*CS, CapturedVars);
6246 llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS, S);
6247 assert(S.getBeginLoc().isValid() &&
6248 "Outlined function call location must be valid.");
6249 ApplyDebugLocation::CreateDefaultArtificial(*this, S.getBeginLoc());
6250 OMPBuilderCBHelpers::EmitCaptureStmt(*this, CodeGenIP, *FiniBB,
6251 OutlinedFn, CapturedVars);
6252 } else {
6254 *this, CS->getCapturedStmt(), AllocaIP, CodeGenIP, "ordered");
6255 }
6256 return llvm::Error::success();
6257 };
6258
6259 OMPLexicalScope Scope(*this, S, OMPD_unknown);
6260 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
6261 OMPBuilder.createOrderedThreadsSimd(Builder, BodyGenCB, FiniCB, !C));
6262 Builder.restoreIP(AfterIP);
6263 }
6264 return;
6265 }
6266
6267 if (S.hasClausesOfKind<OMPDependClause>()) {
6268 assert(!S.hasAssociatedStmt() &&
6269 "No associated statement must be in ordered depend construct.");
6270 for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
6271 CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);
6272 return;
6273 }
6274 if (S.hasClausesOfKind<OMPDoacrossClause>()) {
6275 assert(!S.hasAssociatedStmt() &&
6276 "No associated statement must be in ordered doacross construct.");
6277 for (const auto *DC : S.getClausesOfKind<OMPDoacrossClause>())
6278 CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);
6279 return;
6280 }
6281 const auto *C = S.getSingleClause<OMPSIMDClause>();
6282 auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
6283 PrePostActionTy &Action) {
6284 const CapturedStmt *CS = S.getInnermostCapturedStmt();
6285 if (C) {
6287 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
6288 llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS, S);
6289 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
6290 OutlinedFn, CapturedVars);
6291 } else {
6292 Action.Enter(CGF);
6293 CGF.EmitStmt(CS->getCapturedStmt());
6294 }
6295 };
6296 OMPLexicalScope Scope(*this, S, OMPD_unknown);
6297 CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getBeginLoc(), !C);
6298}
6299
6300static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val,
6301 QualType SrcType, QualType DestType,
6302 SourceLocation Loc) {
6303 assert(CGF.hasScalarEvaluationKind(DestType) &&
6304 "DestType must have scalar evaluation kind.");
6305 assert(!Val.isAggregate() && "Must be a scalar or complex.");
6306 return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType,
6307 DestType, Loc)
6309 Val.getComplexVal(), SrcType, DestType, Loc);
6310}
6311
6314 QualType DestType, SourceLocation Loc) {
6315 assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
6316 "DestType must have complex evaluation kind.");
6318 if (Val.isScalar()) {
6319 // Convert the input element to the element type of the complex.
6320 QualType DestElementType =
6321 DestType->castAs<ComplexType>()->getElementType();
6322 llvm::Value *ScalarVal = CGF.EmitScalarConversion(
6323 Val.getScalarVal(), SrcType, DestElementType, Loc);
6324 ComplexVal = CodeGenFunction::ComplexPairTy(
6325 ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
6326 } else {
6327 assert(Val.isComplex() && "Must be a scalar or complex.");
6328 QualType SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
6329 QualType DestElementType =
6330 DestType->castAs<ComplexType>()->getElementType();
6331 ComplexVal.first = CGF.EmitScalarConversion(
6332 Val.getComplexVal().first, SrcElementType, DestElementType, Loc);
6333 ComplexVal.second = CGF.EmitScalarConversion(
6334 Val.getComplexVal().second, SrcElementType, DestElementType, Loc);
6335 }
6336 return ComplexVal;
6337}
6338
6339static void emitSimpleAtomicStore(CodeGenFunction &CGF, llvm::AtomicOrdering AO,
6340 LValue LVal, RValue RVal) {
6341 if (LVal.isGlobalReg())
6342 CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal);
6343 else
6344 CGF.EmitAtomicStore(RVal, LVal, AO, LVal.isVolatile(), /*isInit=*/false);
6345}
6346
6348 llvm::AtomicOrdering AO, LValue LVal,
6349 SourceLocation Loc) {
6350 if (LVal.isGlobalReg())
6351 return CGF.EmitLoadOfLValue(LVal, Loc);
6352 return CGF.EmitAtomicLoad(
6353 LVal, Loc, llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO),
6354 LVal.isVolatile());
6355}
6356
6358 QualType RValTy, SourceLocation Loc) {
6359 switch (getEvaluationKind(LVal.getType())) {
6360 case TEK_Scalar:
6362 *this, RVal, RValTy, LVal.getType(), Loc)),
6363 LVal);
6364 break;
6365 case TEK_Complex:
6367 convertToComplexValue(*this, RVal, RValTy, LVal.getType(), Loc), LVal,
6368 /*isInit=*/false);
6369 break;
6370 case TEK_Aggregate:
6371 llvm_unreachable("Must be a scalar or complex.");
6372 }
6373}
6374
6375static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO,
6376 const Expr *X, const Expr *V,
6377 SourceLocation Loc) {
6378 // v = x;
6379 assert(V->isLValue() && "V of 'omp atomic read' is not lvalue");
6380 assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");
6381 LValue XLValue = CGF.EmitLValue(X);
6382 LValue VLValue = CGF.EmitLValue(V);
6383 RValue Res = emitSimpleAtomicLoad(CGF, AO, XLValue, Loc);
6384 // OpenMP, 2.17.7, atomic Construct
6385 // If the read or capture clause is specified and the acquire, acq_rel, or
6386 // seq_cst clause is specified then the strong flush on exit from the atomic
6387 // operation is also an acquire flush.
6388 switch (AO) {
6389 case llvm::AtomicOrdering::Acquire:
6390 case llvm::AtomicOrdering::AcquireRelease:
6391 case llvm::AtomicOrdering::SequentiallyConsistent:
6392 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, {}, Loc,
6393 llvm::AtomicOrdering::Acquire);
6394 break;
6395 case llvm::AtomicOrdering::Monotonic:
6396 case llvm::AtomicOrdering::Release:
6397 break;
6398 case llvm::AtomicOrdering::NotAtomic:
6399 case llvm::AtomicOrdering::Unordered:
6400 llvm_unreachable("Unexpected ordering.");
6401 }
6402 CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc);
6404}
6405
6407 llvm::AtomicOrdering AO, const Expr *X,
6408 const Expr *E, SourceLocation Loc) {
6409 // x = expr;
6410 assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");
6411 emitSimpleAtomicStore(CGF, AO, CGF.EmitLValue(X), CGF.EmitAnyExpr(E));
6413 // OpenMP, 2.17.7, atomic Construct
6414 // If the write, update, or capture clause is specified and the release,
6415 // acq_rel, or seq_cst clause is specified then the strong flush on entry to
6416 // the atomic operation is also a release flush.
6417 switch (AO) {
6418 case llvm::AtomicOrdering::Release:
6419 case llvm::AtomicOrdering::AcquireRelease:
6420 case llvm::AtomicOrdering::SequentiallyConsistent:
6421 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, {}, Loc,
6422 llvm::AtomicOrdering::Release);
6423 break;
6424 case llvm::AtomicOrdering::Acquire:
6425 case llvm::AtomicOrdering::Monotonic:
6426 break;
6427 case llvm::AtomicOrdering::NotAtomic:
6428 case llvm::AtomicOrdering::Unordered:
6429 llvm_unreachable("Unexpected ordering.");
6430 }
6431}
6432
6433static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
6434 RValue Update,
6436 llvm::AtomicOrdering AO,
6437 bool IsXLHSInRHSPart) {
6438 ASTContext &Context = CGF.getContext();
6439 // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x'
6440 // expression is simple and atomic is allowed for the given type for the
6441 // target platform.
6442 if (BO == BO_Comma || !Update.isScalar() || !X.isSimple() ||
6443 (!isa<llvm::ConstantInt>(Update.getScalarVal()) &&
6444 (Update.getScalarVal()->getType() != X.getAddress().getElementType())) ||
6445 !Context.getTargetInfo().hasBuiltinAtomic(
6446 Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment())))
6447 return std::make_pair(false, RValue::get(nullptr));
6448
6449 auto &&CheckAtomicSupport = [&CGF](llvm::Type *T, BinaryOperatorKind BO) {
6450 if (T->isIntegerTy())
6451 return true;
6452
6453 if (T->isFloatingPointTy() && (BO == BO_Add || BO == BO_Sub))
6454 return llvm::isPowerOf2_64(CGF.CGM.getDataLayout().getTypeStoreSize(T));
6455
6456 return false;
6457 };
6458
6459 if (!CheckAtomicSupport(Update.getScalarVal()->getType(), BO) ||
6460 !CheckAtomicSupport(X.getAddress().getElementType(), BO))
6461 return std::make_pair(false, RValue::get(nullptr));
6462
6463 bool IsInteger = X.getAddress().getElementType()->isIntegerTy();
6464 llvm::AtomicRMWInst::BinOp RMWOp;
6465 switch (BO) {
6466 case BO_Add:
6467 RMWOp = IsInteger ? llvm::AtomicRMWInst::Add : llvm::AtomicRMWInst::FAdd;
6468 break;
6469 case BO_Sub:
6470 if (!IsXLHSInRHSPart)
6471 return std::make_pair(false, RValue::get(nullptr));
6472 RMWOp = IsInteger ? llvm::AtomicRMWInst::Sub : llvm::AtomicRMWInst::FSub;
6473 break;
6474 case BO_And:
6475 RMWOp = llvm::AtomicRMWInst::And;
6476 break;
6477 case BO_Or:
6478 RMWOp = llvm::AtomicRMWInst::Or;
6479 break;
6480 case BO_Xor:
6481 RMWOp = llvm::AtomicRMWInst::Xor;
6482 break;
6483 case BO_LT:
6484 if (IsInteger)
6485 RMWOp = X.getType()->hasSignedIntegerRepresentation()
6486 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
6487 : llvm::AtomicRMWInst::Max)
6488 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
6489 : llvm::AtomicRMWInst::UMax);
6490 else
6491 RMWOp = IsXLHSInRHSPart ? llvm::AtomicRMWInst::FMin
6492 : llvm::AtomicRMWInst::FMax;
6493 break;
6494 case BO_GT:
6495 if (IsInteger)
6496 RMWOp = X.getType()->hasSignedIntegerRepresentation()
6497 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
6498 : llvm::AtomicRMWInst::Min)
6499 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
6500 : llvm::AtomicRMWInst::UMin);
6501 else
6502 RMWOp = IsXLHSInRHSPart ? llvm::AtomicRMWInst::FMax
6503 : llvm::AtomicRMWInst::FMin;
6504 break;
6505 case BO_Assign:
6506 RMWOp = llvm::AtomicRMWInst::Xchg;
6507 break;
6508 case BO_Mul:
6509 case BO_Div:
6510 case BO_Rem:
6511 case BO_Shl:
6512 case BO_Shr:
6513 case BO_LAnd:
6514 case BO_LOr:
6515 return std::make_pair(false, RValue::get(nullptr));
6516 case BO_PtrMemD:
6517 case BO_PtrMemI:
6518 case BO_LE:
6519 case BO_GE:
6520 case BO_EQ:
6521 case BO_NE:
6522 case BO_Cmp:
6523 case BO_AddAssign:
6524 case BO_SubAssign:
6525 case BO_AndAssign:
6526 case BO_OrAssign:
6527 case BO_XorAssign:
6528 case BO_MulAssign:
6529 case BO_DivAssign:
6530 case BO_RemAssign:
6531 case BO_ShlAssign:
6532 case BO_ShrAssign:
6533 case BO_Comma:
6534 llvm_unreachable("Unsupported atomic update operation");
6535 }
6536 llvm::Value *UpdateVal = Update.getScalarVal();
6537 if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
6538 if (IsInteger)
6539 UpdateVal = CGF.Builder.CreateIntCast(
6540 IC, X.getAddress().getElementType(),
6541 X.getType()->hasSignedIntegerRepresentation());
6542 else
6543 UpdateVal = CGF.Builder.CreateCast(llvm::Instruction::CastOps::UIToFP, IC,
6544 X.getAddress().getElementType());
6545 }
6546 llvm::AtomicRMWInst *Res =
6547 CGF.emitAtomicRMWInst(RMWOp, X.getAddress(), UpdateVal, AO);
6548 return std::make_pair(true, RValue::get(Res));
6549}
6550
6553 llvm::AtomicOrdering AO, SourceLocation Loc,
6554 const llvm::function_ref<RValue(RValue)> CommonGen) {
6555 // Update expressions are allowed to have the following forms:
6556 // x binop= expr; -> xrval + expr;
6557 // x++, ++x -> xrval + 1;
6558 // x--, --x -> xrval - 1;
6559 // x = x binop expr; -> xrval binop expr
6560 // x = expr Op x; - > expr binop xrval;
6561 auto Res = emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart);
6562 if (!Res.first) {
6563 if (X.isGlobalReg()) {
6564 // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop
6565 // 'xrval'.
6566 EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X);
6567 } else {
6568 // Perform compare-and-swap procedure.
6569 EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified());
6570 }
6571 }
6572 return Res;
6573}
6574
6576 llvm::AtomicOrdering AO, const Expr *X,
6577 const Expr *E, const Expr *UE,
6578 bool IsXLHSInRHSPart, SourceLocation Loc) {
6579 assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
6580 "Update expr in 'atomic update' must be a binary operator.");
6581 const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
6582 // Update expressions are allowed to have the following forms:
6583 // x binop= expr; -> xrval + expr;
6584 // x++, ++x -> xrval + 1;
6585 // x--, --x -> xrval - 1;
6586 // x = x binop expr; -> xrval binop expr
6587 // x = expr Op x; - > expr binop xrval;
6588 assert(X->isLValue() && "X of 'omp atomic update' is not lvalue");
6589 LValue XLValue = CGF.EmitLValue(X);
6590 RValue ExprRValue = CGF.EmitAnyExpr(E);
6591 const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
6592 const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
6593 const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
6594 const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
6595 auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) {
6596 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
6597 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
6598 return CGF.EmitAnyExpr(UE);
6599 };
6601 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
6603 // OpenMP, 2.17.7, atomic Construct
6604 // If the write, update, or capture clause is specified and the release,
6605 // acq_rel, or seq_cst clause is specified then the strong flush on entry to
6606 // the atomic operation is also a release flush.
6607 switch (AO) {
6608 case llvm::AtomicOrdering::Release:
6609 case llvm::AtomicOrdering::AcquireRelease:
6610 case llvm::AtomicOrdering::SequentiallyConsistent:
6611 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, {}, Loc,
6612 llvm::AtomicOrdering::Release);
6613 break;
6614 case llvm::AtomicOrdering::Acquire:
6615 case llvm::AtomicOrdering::Monotonic:
6616 break;
6617 case llvm::AtomicOrdering::NotAtomic:
6618 case llvm::AtomicOrdering::Unordered:
6619 llvm_unreachable("Unexpected ordering.");
6620 }
6621}
6622
6624 QualType SourceType, QualType ResType,
6625 SourceLocation Loc) {
6626 switch (CGF.getEvaluationKind(ResType)) {
6627 case TEK_Scalar:
6628 return RValue::get(
6629 convertToScalarValue(CGF, Value, SourceType, ResType, Loc));
6630 case TEK_Complex: {
6631 auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc);
6632 return RValue::getComplex(Res.first, Res.second);
6633 }
6634 case TEK_Aggregate:
6635 break;
6636 }
6637 llvm_unreachable("Must be a scalar or complex.");
6638}
6639
6641 llvm::AtomicOrdering AO,
6642 bool IsPostfixUpdate, const Expr *V,
6643 const Expr *X, const Expr *E,
6644 const Expr *UE, bool IsXLHSInRHSPart,
6645 SourceLocation Loc) {
6646 assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue");
6647 assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue");
6648 RValue NewVVal;
6649 LValue VLValue = CGF.EmitLValue(V);
6650 LValue XLValue = CGF.EmitLValue(X);
6651 RValue ExprRValue = CGF.EmitAnyExpr(E);
6652 QualType NewVValType;
6653 if (UE) {
6654 // 'x' is updated with some additional value.
6655 assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
6656 "Update expr in 'atomic capture' must be a binary operator.");
6657 const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
6658 // Update expressions are allowed to have the following forms:
6659 // x binop= expr; -> xrval + expr;
6660 // x++, ++x -> xrval + 1;
6661 // x--, --x -> xrval - 1;
6662 // x = x binop expr; -> xrval binop expr
6663 // x = expr Op x; - > expr binop xrval;
6664 const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
6665 const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
6666 const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
6667 NewVValType = XRValExpr->getType();
6668 const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
6669 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
6670 IsPostfixUpdate](RValue XRValue) {
6671 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
6672 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
6673 RValue Res = CGF.EmitAnyExpr(UE);
6674 NewVVal = IsPostfixUpdate ? XRValue : Res;
6675 return Res;
6676 };
6677 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
6678 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
6680 if (Res.first) {
6681 // 'atomicrmw' instruction was generated.
6682 if (IsPostfixUpdate) {
6683 // Use old value from 'atomicrmw'.
6684 NewVVal = Res.second;
6685 } else {
6686 // 'atomicrmw' does not provide new value, so evaluate it using old
6687 // value of 'x'.
6688 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
6689 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
6690 NewVVal = CGF.EmitAnyExpr(UE);
6691 }
6692 }
6693 } else {
6694 // 'x' is simply rewritten with some 'expr'.
6695 NewVValType = X->getType().getNonReferenceType();
6696 ExprRValue = convertToType(CGF, ExprRValue, E->getType(),
6697 X->getType().getNonReferenceType(), Loc);
6698 auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) {
6699 NewVVal = XRValue;
6700 return ExprRValue;
6701 };
6702 // Try to perform atomicrmw xchg, otherwise simple exchange.
6703 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
6704 XLValue, ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO,
6705 Loc, Gen);
6707 if (Res.first) {
6708 // 'atomicrmw' instruction was generated.
6709 NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
6710 }
6711 }
6712 // Emit post-update store to 'v' of old/new 'x' value.
6713 CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc);
6715 // OpenMP 5.1 removes the required flush for capture clause.
6716 if (CGF.CGM.getLangOpts().OpenMP < 51) {
6717 // OpenMP, 2.17.7, atomic Construct
6718 // If the write, update, or capture clause is specified and the release,
6719 // acq_rel, or seq_cst clause is specified then the strong flush on entry to
6720 // the atomic operation is also a release flush.
6721 // If the read or capture clause is specified and the acquire, acq_rel, or
6722 // seq_cst clause is specified then the strong flush on exit from the atomic
6723 // operation is also an acquire flush.
6724 switch (AO) {
6725 case llvm::AtomicOrdering::Release:
6726 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, {}, Loc,
6727 llvm::AtomicOrdering::Release);
6728 break;
6729 case llvm::AtomicOrdering::Acquire:
6730 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, {}, Loc,
6731 llvm::AtomicOrdering::Acquire);
6732 break;
6733 case llvm::AtomicOrdering::AcquireRelease:
6734 case llvm::AtomicOrdering::SequentiallyConsistent:
6736 CGF, {}, Loc, llvm::AtomicOrdering::AcquireRelease);
6737 break;
6738 case llvm::AtomicOrdering::Monotonic:
6739 break;
6740 case llvm::AtomicOrdering::NotAtomic:
6741 case llvm::AtomicOrdering::Unordered:
6742 llvm_unreachable("Unexpected ordering.");
6743 }
6744 }
6745}
6746
6748 CodeGenFunction &CGF, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO,
6749 const Expr *X, const Expr *V, const Expr *R, const Expr *E, const Expr *D,
6750 const Expr *CE, bool IsXBinopExpr, bool IsPostfixUpdate, bool IsFailOnly,
6751 SourceLocation Loc) {
6752 llvm::OpenMPIRBuilder &OMPBuilder =
6754
6755 OMPAtomicCompareOp Op;
6756 assert(isa<BinaryOperator>(CE) && "CE is not a BinaryOperator");
6757 switch (cast<BinaryOperator>(CE)->getOpcode()) {
6758 case BO_EQ:
6759 Op = OMPAtomicCompareOp::EQ;
6760 break;
6761 case BO_LT:
6762 Op = OMPAtomicCompareOp::MIN;
6763 break;
6764 case BO_GT:
6765 Op = OMPAtomicCompareOp::MAX;
6766 break;
6767 default:
6768 llvm_unreachable("unsupported atomic compare binary operator");
6769 }
6770
6771 LValue XLVal = CGF.EmitLValue(X);
6772 Address XAddr = XLVal.getAddress();
6773
6774 auto EmitRValueWithCastIfNeeded = [&CGF, Loc](const Expr *X, const Expr *E) {
6775 if (X->getType() == E->getType())
6776 return CGF.EmitScalarExpr(E);
6777 const Expr *NewE = E->IgnoreImplicitAsWritten();
6778 llvm::Value *V = CGF.EmitScalarExpr(NewE);
6779 if (NewE->getType() == X->getType())
6780 return V;
6781 return CGF.EmitScalarConversion(V, NewE->getType(), X->getType(), Loc);
6782 };
6783
6784 llvm::Value *EVal = EmitRValueWithCastIfNeeded(X, E);
6785 llvm::Value *DVal = D ? EmitRValueWithCastIfNeeded(X, D) : nullptr;
6786 if (auto *CI = dyn_cast<llvm::ConstantInt>(EVal))
6787 EVal = CGF.Builder.CreateIntCast(
6788 CI, XLVal.getAddress().getElementType(),
6790 if (DVal)
6791 if (auto *CI = dyn_cast<llvm::ConstantInt>(DVal))
6792 DVal = CGF.Builder.CreateIntCast(
6793 CI, XLVal.getAddress().getElementType(),
6795
6796 llvm::OpenMPIRBuilder::AtomicOpValue XOpVal{
6797 XAddr.emitRawPointer(CGF), XAddr.getElementType(),
6798 X->getType()->hasSignedIntegerRepresentation(),
6799 X->getType().isVolatileQualified()};
6800 llvm::OpenMPIRBuilder::AtomicOpValue VOpVal, ROpVal;
6801 if (V) {
6802 LValue LV = CGF.EmitLValue(V);
6803 Address Addr = LV.getAddress();
6804 VOpVal = {Addr.emitRawPointer(CGF), Addr.getElementType(),
6805 V->getType()->hasSignedIntegerRepresentation(),
6806 V->getType().isVolatileQualified()};
6807 }
6808 if (R) {
6809 LValue LV = CGF.EmitLValue(R);
6810 Address Addr = LV.getAddress();
6811 ROpVal = {Addr.emitRawPointer(CGF), Addr.getElementType(),
6814 }
6815
6816 if (FailAO == llvm::AtomicOrdering::NotAtomic) {
6817 // fail clause was not mentioned on the
6818 // "#pragma omp atomic compare" construct.
6819 CGF.Builder.restoreIP(OMPBuilder.createAtomicCompare(
6820 CGF.Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr,
6822 } else
6823 CGF.Builder.restoreIP(OMPBuilder.createAtomicCompare(
6824 CGF.Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr,
6825 IsPostfixUpdate, IsFailOnly, FailAO));
6826}
6827
6829 llvm::AtomicOrdering AO,
6830 llvm::AtomicOrdering FailAO, bool IsPostfixUpdate,
6831 const Expr *X, const Expr *V, const Expr *R,
6832 const Expr *E, const Expr *UE, const Expr *D,
6833 const Expr *CE, bool IsXLHSInRHSPart,
6834 bool IsFailOnly, SourceLocation Loc) {
6835 switch (Kind) {
6836 case OMPC_read:
6837 emitOMPAtomicReadExpr(CGF, AO, X, V, Loc);
6838 break;
6839 case OMPC_write:
6840 emitOMPAtomicWriteExpr(CGF, AO, X, E, Loc);
6841 break;
6842 case OMPC_unknown:
6843 case OMPC_update:
6844 emitOMPAtomicUpdateExpr(CGF, AO, X, E, UE, IsXLHSInRHSPart, Loc);
6845 break;
6846 case OMPC_capture:
6847 emitOMPAtomicCaptureExpr(CGF, AO, IsPostfixUpdate, V, X, E, UE,
6848 IsXLHSInRHSPart, Loc);
6849 break;
6850 case OMPC_compare: {
6851 emitOMPAtomicCompareExpr(CGF, AO, FailAO, X, V, R, E, D, CE,
6853 break;
6854 }
6855 default:
6856 llvm_unreachable("Clause is not allowed in 'omp atomic'.");
6857 }
6858}
6859
6860void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) {
6861 llvm::AtomicOrdering AO = CGM.getOpenMPRuntime().getDefaultMemoryOrdering();
6862 // Fail Memory Clause Ordering.
6863 llvm::AtomicOrdering FailAO = llvm::AtomicOrdering::NotAtomic;
6864 bool MemOrderingSpecified = false;
6865 if (S.getSingleClause<OMPSeqCstClause>()) {
6866 AO = llvm::AtomicOrdering::SequentiallyConsistent;
6867 MemOrderingSpecified = true;
6868 } else if (S.getSingleClause<OMPAcqRelClause>()) {
6869 AO = llvm::AtomicOrdering::AcquireRelease;
6870 MemOrderingSpecified = true;
6871 } else if (S.getSingleClause<OMPAcquireClause>()) {
6872 AO = llvm::AtomicOrdering::Acquire;
6873 MemOrderingSpecified = true;
6874 } else if (S.getSingleClause<OMPReleaseClause>()) {
6875 AO = llvm::AtomicOrdering::Release;
6876 MemOrderingSpecified = true;
6877 } else if (S.getSingleClause<OMPRelaxedClause>()) {
6878 AO = llvm::AtomicOrdering::Monotonic;
6879 MemOrderingSpecified = true;
6880 }
6881 llvm::SmallSet<OpenMPClauseKind, 2> KindsEncountered;
6882 OpenMPClauseKind Kind = OMPC_unknown;
6883 for (const OMPClause *C : S.clauses()) {
6884 // Find first clause (skip seq_cst|acq_rel|aqcuire|release|relaxed clause,
6885 // if it is first).
6886 OpenMPClauseKind K = C->getClauseKind();
6887 // TBD
6888 if (K == OMPC_weak)
6889 return;
6890 if (K == OMPC_seq_cst || K == OMPC_acq_rel || K == OMPC_acquire ||
6891 K == OMPC_release || K == OMPC_relaxed || K == OMPC_hint)
6892 continue;
6893 Kind = K;
6894 KindsEncountered.insert(K);
6895 }
6896 // We just need to correct Kind here. No need to set a bool saying it is
6897 // actually compare capture because we can tell from whether V and R are
6898 // nullptr.
6899 if (KindsEncountered.contains(OMPC_compare) &&
6900 KindsEncountered.contains(OMPC_capture))
6901 Kind = OMPC_compare;
6902 if (!MemOrderingSpecified) {
6903 llvm::AtomicOrdering DefaultOrder =
6904 CGM.getOpenMPRuntime().getDefaultMemoryOrdering();
6905 if (DefaultOrder == llvm::AtomicOrdering::Monotonic ||
6906 DefaultOrder == llvm::AtomicOrdering::SequentiallyConsistent ||
6907 (DefaultOrder == llvm::AtomicOrdering::AcquireRelease &&
6908 Kind == OMPC_capture)) {
6909 AO = DefaultOrder;
6910 } else if (DefaultOrder == llvm::AtomicOrdering::AcquireRelease) {
6911 if (Kind == OMPC_unknown || Kind == OMPC_update || Kind == OMPC_write) {
6912 AO = llvm::AtomicOrdering::Release;
6913 } else if (Kind == OMPC_read) {
6914 assert(Kind == OMPC_read && "Unexpected atomic kind.");
6915 AO = llvm::AtomicOrdering::Acquire;
6916 }
6917 }
6918 }
6919
6920 if (KindsEncountered.contains(OMPC_compare) &&
6921 KindsEncountered.contains(OMPC_fail)) {
6922 Kind = OMPC_compare;
6923 const auto *FailClause = S.getSingleClause<OMPFailClause>();
6924 if (FailClause) {
6925 OpenMPClauseKind FailParameter = FailClause->getFailParameter();
6926 if (FailParameter == llvm::omp::OMPC_relaxed)
6927 FailAO = llvm::AtomicOrdering::Monotonic;
6928 else if (FailParameter == llvm::omp::OMPC_acquire)
6929 FailAO = llvm::AtomicOrdering::Acquire;
6930 else if (FailParameter == llvm::omp::OMPC_seq_cst)
6931 FailAO = llvm::AtomicOrdering::SequentiallyConsistent;
6932 }
6933 }
6934
6935 LexicalScope Scope(*this, S.getSourceRange());
6936 EmitStopPoint(S.getAssociatedStmt());
6937 emitOMPAtomicExpr(*this, Kind, AO, FailAO, S.isPostfixUpdate(), S.getX(),
6938 S.getV(), S.getR(), S.getExpr(), S.getUpdateExpr(),
6939 S.getD(), S.getCondExpr(), S.isXLHSInRHSPart(),
6940 S.isFailOnly(), S.getBeginLoc());
6941}
6942
6944 const OMPExecutableDirective &S,
6945 const RegionCodeGenTy &CodeGen) {
6946 assert(isOpenMPTargetExecutionDirective(S.getDirectiveKind()));
6947 CodeGenModule &CGM = CGF.CGM;
6948
6949 // On device emit this construct as inlined code.
6950 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
6951 OMPLexicalScope Scope(CGF, S, OMPD_target);
6953 CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) {
6954 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
6955 });
6956 return;
6957 }
6958
6960 llvm::Function *Fn = nullptr;
6961 llvm::Constant *FnID = nullptr;
6962
6963 const Expr *IfCond = nullptr;
6964 // Check for the at most one if clause associated with the target region.
6965 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
6966 if (C->getNameModifier() == OMPD_unknown ||
6967 C->getNameModifier() == OMPD_target) {
6968 IfCond = C->getCondition();
6969 break;
6970 }
6971 }
6972
6973 // Check if we have any device clause associated with the directive.
6974 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device(
6975 nullptr, OMPC_DEVICE_unknown);
6976 if (auto *C = S.getSingleClause<OMPDeviceClause>())
6977 Device.setPointerAndInt(C->getDevice(), C->getModifier());
6978
6979 // Check if we have an if clause whose conditional always evaluates to false
6980 // or if we do not have any targets specified. If so the target region is not
6981 // an offload entry point.
6982 bool IsOffloadEntry = true;
6983 if (IfCond) {
6984 bool Val;
6985 if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val)
6986 IsOffloadEntry = false;
6987 }
6988 if (CGM.getLangOpts().OMPTargetTriples.empty())
6989 IsOffloadEntry = false;
6990
6991 if (CGM.getLangOpts().OpenMPOffloadMandatory && !IsOffloadEntry) {
6992 unsigned DiagID = CGM.getDiags().getCustomDiagID(
6994 "No offloading entry generated while offloading is mandatory.");
6995 CGM.getDiags().Report(DiagID);
6996 }
6997
6998 assert(CGF.CurFuncDecl && "No parent declaration for target region!");
6999 StringRef ParentName;
7000 // In case we have Ctors/Dtors we use the complete type variant to produce
7001 // the mangling of the device outlined kernel.
7002 if (const auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl))
7003 ParentName = CGM.getMangledName(GlobalDecl(D, Ctor_Complete));
7004 else if (const auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl))
7005 ParentName = CGM.getMangledName(GlobalDecl(D, Dtor_Complete));
7006 else
7007 ParentName =
7009
7010 // Emit target region as a standalone region.
7011 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID,
7012 IsOffloadEntry, CodeGen);
7013 OMPLexicalScope Scope(CGF, S, OMPD_task);
7014 auto &&SizeEmitter =
7015 [IsOffloadEntry](CodeGenFunction &CGF,
7016 const OMPLoopDirective &D) -> llvm::Value * {
7017 if (IsOffloadEntry) {
7018 OMPLoopScope(CGF, D);
7019 // Emit calculation of the iterations count.
7020 llvm::Value *NumIterations = CGF.EmitScalarExpr(D.getNumIterations());
7021 NumIterations = CGF.Builder.CreateIntCast(NumIterations, CGF.Int64Ty,
7022 /*isSigned=*/false);
7023 return NumIterations;
7024 }
7025 return nullptr;
7026 };
7027 CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device,
7028 SizeEmitter);
7029}
7030
7032 PrePostActionTy &Action) {
7033 Action.Enter(CGF);
7034 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7035 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
7036 CGF.EmitOMPPrivateClause(S, PrivateScope);
7037 (void)PrivateScope.Privatize();
7038 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
7040
7041 CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt());
7042 CGF.EnsureInsertPoint();
7043}
7044
7046 StringRef ParentName,
7047 const OMPTargetDirective &S) {
7048 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7049 emitTargetRegion(CGF, S, Action);
7050 };
7051 llvm::Function *Fn;
7052 llvm::Constant *Addr;
7053 // Emit target region as a standalone region.
7054 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7055 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
7056 assert(Fn && Addr && "Target device function emission failed.");
7057}
7058
7060 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7061 emitTargetRegion(CGF, S, Action);
7062 };
7064}
7065
7067 const OMPExecutableDirective &S,
7068 OpenMPDirectiveKind InnermostKind,
7069 const RegionCodeGenTy &CodeGen) {
7070 const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);
7071 llvm::Function *OutlinedFn =
7073 CGF, S, *CS->getCapturedDecl()->param_begin(), InnermostKind,
7074 CodeGen);
7075
7076 const auto *NT = S.getSingleClause<OMPNumTeamsClause>();
7077 const auto *TL = S.getSingleClause<OMPThreadLimitClause>();
7078 if (NT || TL) {
7079 const Expr *NumTeams = NT ? NT->getNumTeams().front() : nullptr;
7080 const Expr *ThreadLimit = TL ? TL->getThreadLimit().front() : nullptr;
7081
7082 CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,
7083 S.getBeginLoc());
7084 }
7085
7086 OMPTeamsScope Scope(CGF, S);
7088 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
7089 CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getBeginLoc(), OutlinedFn,
7090 CapturedVars);
7091}
7092
7094 // Emit teams region as a standalone region.
7095 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7096 Action.Enter(CGF);
7097 OMPPrivateScope PrivateScope(CGF);
7098 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
7099 CGF.EmitOMPPrivateClause(S, PrivateScope);
7100 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7101 (void)PrivateScope.Privatize();
7102 CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt());
7103 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7104 };
7105 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
7107 [](CodeGenFunction &) { return nullptr; });
7108}
7109
7111 const OMPTargetTeamsDirective &S) {
7112 auto *CS = S.getCapturedStmt(OMPD_teams);
7113 Action.Enter(CGF);
7114 // Emit teams region as a standalone region.
7115 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
7116 Action.Enter(CGF);
7117 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7118 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
7119 CGF.EmitOMPPrivateClause(S, PrivateScope);
7120 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7121 (void)PrivateScope.Privatize();
7122 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
7124 CGF.EmitStmt(CS->getCapturedStmt());
7125 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7126 };
7127 emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen);
7129 [](CodeGenFunction &) { return nullptr; });
7130}
7131
7133 CodeGenModule &CGM, StringRef ParentName,
7134 const OMPTargetTeamsDirective &S) {
7135 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7136 emitTargetTeamsRegion(CGF, Action, S);
7137 };
7138 llvm::Function *Fn;
7139 llvm::Constant *Addr;
7140 // Emit target region as a standalone region.
7141 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7142 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
7143 assert(Fn && Addr && "Target device function emission failed.");
7144}
7145
7147 const OMPTargetTeamsDirective &S) {
7148 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7149 emitTargetTeamsRegion(CGF, Action, S);
7150 };
7152}
7153
7154static void
7157 Action.Enter(CGF);
7158 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7160 };
7161
7162 // Emit teams region as a standalone region.
7163 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7164 PrePostActionTy &Action) {
7165 Action.Enter(CGF);
7166 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7167 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7168 (void)PrivateScope.Privatize();
7169 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
7170 CodeGenDistribute);
7171 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7172 };
7173 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen);
7175 [](CodeGenFunction &) { return nullptr; });
7176}
7177
7179 CodeGenModule &CGM, StringRef ParentName,
7181 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7182 emitTargetTeamsDistributeRegion(CGF, Action, S);
7183 };
7184 llvm::Function *Fn;
7185 llvm::Constant *Addr;
7186 // Emit target region as a standalone region.
7187 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7188 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
7189 assert(Fn && Addr && "Target device function emission failed.");
7190}
7191
7194 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7195 emitTargetTeamsDistributeRegion(CGF, Action, S);
7196 };
7198}
7199
7201 CodeGenFunction &CGF, PrePostActionTy &Action,
7203 Action.Enter(CGF);
7204 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7206 };
7207
7208 // Emit teams region as a standalone region.
7209 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7210 PrePostActionTy &Action) {
7211 Action.Enter(CGF);
7212 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7213 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7214 (void)PrivateScope.Privatize();
7215 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
7216 CodeGenDistribute);
7217 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7218 };
7219 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen);
7221 [](CodeGenFunction &) { return nullptr; });
7222}
7223
7225 CodeGenModule &CGM, StringRef ParentName,
7227 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7229 };
7230 llvm::Function *Fn;
7231 llvm::Constant *Addr;
7232 // Emit target region as a standalone region.
7233 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7234 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
7235 assert(Fn && Addr && "Target device function emission failed.");
7236}
7237
7240 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7242 };
7244}
7245
7247 const OMPTeamsDistributeDirective &S) {
7248
7249 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7251 };
7252
7253 // Emit teams region as a standalone region.
7254 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7255 PrePostActionTy &Action) {
7256 Action.Enter(CGF);
7257 OMPPrivateScope PrivateScope(CGF);
7258 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7259 (void)PrivateScope.Privatize();
7260 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
7261 CodeGenDistribute);
7262 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7263 };
7264 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
7266 [](CodeGenFunction &) { return nullptr; });
7267}
7268
7271 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7273 };
7274
7275 // Emit teams region as a standalone region.
7276 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7277 PrePostActionTy &Action) {
7278 Action.Enter(CGF);
7279 OMPPrivateScope PrivateScope(CGF);
7280 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7281 (void)PrivateScope.Privatize();
7282 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
7283 CodeGenDistribute);
7284 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7285 };
7286 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_simd, CodeGen);
7288 [](CodeGenFunction &) { return nullptr; });
7289}
7290
7293 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7295 S.getDistInc());
7296 };
7297
7298 // Emit teams region as a standalone region.
7299 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7300 PrePostActionTy &Action) {
7301 Action.Enter(CGF);
7302 OMPPrivateScope PrivateScope(CGF);
7303 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7304 (void)PrivateScope.Privatize();
7305 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
7306 CodeGenDistribute);
7307 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7308 };
7309 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
7311 [](CodeGenFunction &) { return nullptr; });
7312}
7313
7316 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7318 S.getDistInc());
7319 };
7320
7321 // Emit teams region as a standalone region.
7322 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7323 PrePostActionTy &Action) {
7324 Action.Enter(CGF);
7325 OMPPrivateScope PrivateScope(CGF);
7326 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7327 (void)PrivateScope.Privatize();
7329 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
7330 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7331 };
7332 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for_simd,
7333 CodeGen);
7335 [](CodeGenFunction &) { return nullptr; });
7336}
7337
7339 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
7340 llvm::Value *Device = nullptr;
7341 llvm::Value *NumDependences = nullptr;
7342 llvm::Value *DependenceList = nullptr;
7343
7344 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
7345 Device = EmitScalarExpr(C->getDevice());
7346
7347 // Build list and emit dependences
7350 if (!Data.Dependences.empty()) {
7351 Address DependenciesArray = Address::invalid();
7352 std::tie(NumDependences, DependenciesArray) =
7353 CGM.getOpenMPRuntime().emitDependClause(*this, Data.Dependences,
7354 S.getBeginLoc());
7355 DependenceList = DependenciesArray.emitRawPointer(*this);
7356 }
7357 Data.HasNowaitClause = S.hasClausesOfKind<OMPNowaitClause>();
7358
7359 assert(!(Data.HasNowaitClause && !(S.getSingleClause<OMPInitClause>() ||
7360 S.getSingleClause<OMPDestroyClause>() ||
7361 S.getSingleClause<OMPUseClause>())) &&
7362 "OMPNowaitClause clause is used separately in OMPInteropDirective.");
7363
7364 auto ItOMPInitClause = S.getClausesOfKind<OMPInitClause>();
7365 if (!ItOMPInitClause.empty()) {
7366 // Look at the multiple init clauses
7367 for (const OMPInitClause *C : ItOMPInitClause) {
7368 llvm::Value *InteropvarPtr =
7369 EmitLValue(C->getInteropVar()).getPointer(*this);
7370 llvm::omp::OMPInteropType InteropType =
7371 llvm::omp::OMPInteropType::Unknown;
7372 if (C->getIsTarget()) {
7373 InteropType = llvm::omp::OMPInteropType::Target;
7374 } else {
7375 assert(C->getIsTargetSync() &&
7376 "Expected interop-type target/targetsync");
7377 InteropType = llvm::omp::OMPInteropType::TargetSync;
7378 }
7379 OMPBuilder.createOMPInteropInit(Builder, InteropvarPtr, InteropType,
7380 Device, NumDependences, DependenceList,
7381 Data.HasNowaitClause);
7382 }
7383 }
7384 auto ItOMPDestroyClause = S.getClausesOfKind<OMPDestroyClause>();
7385 if (!ItOMPDestroyClause.empty()) {
7386 // Look at the multiple destroy clauses
7387 for (const OMPDestroyClause *C : ItOMPDestroyClause) {
7388 llvm::Value *InteropvarPtr =
7389 EmitLValue(C->getInteropVar()).getPointer(*this);
7390 OMPBuilder.createOMPInteropDestroy(Builder, InteropvarPtr, Device,
7391 NumDependences, DependenceList,
7392 Data.HasNowaitClause);
7393 }
7394 }
7395 auto ItOMPUseClause = S.getClausesOfKind<OMPUseClause>();
7396 if (!ItOMPUseClause.empty()) {
7397 // Look at the multiple use clauses
7398 for (const OMPUseClause *C : ItOMPUseClause) {
7399 llvm::Value *InteropvarPtr =
7400 EmitLValue(C->getInteropVar()).getPointer(*this);
7401 OMPBuilder.createOMPInteropUse(Builder, InteropvarPtr, Device,
7402 NumDependences, DependenceList,
7403 Data.HasNowaitClause);
7404 }
7405 }
7406}
7407
7410 PrePostActionTy &Action) {
7411 Action.Enter(CGF);
7412 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7414 S.getDistInc());
7415 };
7416
7417 // Emit teams region as a standalone region.
7418 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7419 PrePostActionTy &Action) {
7420 Action.Enter(CGF);
7421 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7422 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7423 (void)PrivateScope.Privatize();
7425 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
7426 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7427 };
7428
7429 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for,
7430 CodeGenTeams);
7432 [](CodeGenFunction &) { return nullptr; });
7433}
7434
7436 CodeGenModule &CGM, StringRef ParentName,
7438 // Emit SPMD target teams distribute parallel for region as a standalone
7439 // region.
7440 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7442 };
7443 llvm::Function *Fn;
7444 llvm::Constant *Addr;
7445 // Emit target region as a standalone region.
7446 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7447 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
7448 assert(Fn && Addr && "Target device function emission failed.");
7449}
7450
7458
7460 CodeGenFunction &CGF,
7462 PrePostActionTy &Action) {
7463 Action.Enter(CGF);
7464 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7466 S.getDistInc());
7467 };
7468
7469 // Emit teams region as a standalone region.
7470 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7471 PrePostActionTy &Action) {
7472 Action.Enter(CGF);
7473 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7474 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7475 (void)PrivateScope.Privatize();
7477 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
7478 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
7479 };
7480
7481 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd,
7482 CodeGenTeams);
7484 [](CodeGenFunction &) { return nullptr; });
7485}
7486
7488 CodeGenModule &CGM, StringRef ParentName,
7490 // Emit SPMD target teams distribute parallel for simd region as a standalone
7491 // region.
7492 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7494 };
7495 llvm::Function *Fn;
7496 llvm::Constant *Addr;
7497 // Emit target region as a standalone region.
7498 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7499 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
7500 assert(Fn && Addr && "Target device function emission failed.");
7501}
7502
7510
7513 CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getBeginLoc(),
7514 S.getCancelRegion());
7515}
7516
7518 const Expr *IfCond = nullptr;
7519 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
7520 if (C->getNameModifier() == OMPD_unknown ||
7521 C->getNameModifier() == OMPD_cancel) {
7522 IfCond = C->getCondition();
7523 break;
7524 }
7525 }
7526 if (CGM.getLangOpts().OpenMPIRBuilder) {
7527 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
7528 // TODO: This check is necessary as we only generate `omp parallel` through
7529 // the OpenMPIRBuilder for now.
7530 if (S.getCancelRegion() == OMPD_parallel ||
7531 S.getCancelRegion() == OMPD_sections ||
7532 S.getCancelRegion() == OMPD_section) {
7533 llvm::Value *IfCondition = nullptr;
7534 if (IfCond)
7535 IfCondition = EmitScalarExpr(IfCond,
7536 /*IgnoreResultAssign=*/true);
7537 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
7538 OMPBuilder.createCancel(Builder, IfCondition, S.getCancelRegion()));
7539 return Builder.restoreIP(AfterIP);
7540 }
7541 }
7542
7543 CGM.getOpenMPRuntime().emitCancelCall(*this, S.getBeginLoc(), IfCond,
7544 S.getCancelRegion());
7545}
7546
7549 if (Kind == OMPD_parallel || Kind == OMPD_task ||
7550 Kind == OMPD_target_parallel || Kind == OMPD_taskloop ||
7551 Kind == OMPD_master_taskloop || Kind == OMPD_parallel_master_taskloop)
7552 return ReturnBlock;
7553 assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
7554 Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
7555 Kind == OMPD_distribute_parallel_for ||
7556 Kind == OMPD_target_parallel_for ||
7557 Kind == OMPD_teams_distribute_parallel_for ||
7558 Kind == OMPD_target_teams_distribute_parallel_for);
7559 return OMPCancelStack.getExitBlock();
7560}
7561
7563 const OMPUseDevicePtrClause &C, OMPPrivateScope &PrivateScope,
7564 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
7565 CaptureDeviceAddrMap) {
7566 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
7567 for (const Expr *OrigVarIt : C.varlist()) {
7568 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(OrigVarIt)->getDecl());
7569 if (!Processed.insert(OrigVD).second)
7570 continue;
7571
7572 // In order to identify the right initializer we need to match the
7573 // declaration used by the mapping logic. In some cases we may get
7574 // OMPCapturedExprDecl that refers to the original declaration.
7575 const ValueDecl *MatchingVD = OrigVD;
7576 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
7577 // OMPCapturedExprDecl are used to privative fields of the current
7578 // structure.
7579 const auto *ME = cast<MemberExpr>(OED->getInit());
7580 assert(isa<CXXThisExpr>(ME->getBase()->IgnoreImpCasts()) &&
7581 "Base should be the current struct!");
7582 MatchingVD = ME->getMemberDecl();
7583 }
7584
7585 // If we don't have information about the current list item, move on to
7586 // the next one.
7587 auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
7588 if (InitAddrIt == CaptureDeviceAddrMap.end())
7589 continue;
7590
7591 llvm::Type *Ty = ConvertTypeForMem(OrigVD->getType().getNonReferenceType());
7592
7593 // Return the address of the private variable.
7594 bool IsRegistered = PrivateScope.addPrivate(
7595 OrigVD,
7596 Address(InitAddrIt->second, Ty,
7597 getContext().getTypeAlignInChars(getContext().VoidPtrTy)));
7598 assert(IsRegistered && "firstprivate var already registered as private");
7599 // Silence the warning about unused variable.
7600 (void)IsRegistered;
7601 }
7602}
7603
7604static const VarDecl *getBaseDecl(const Expr *Ref) {
7605 const Expr *Base = Ref->IgnoreParenImpCasts();
7606 while (const auto *OASE = dyn_cast<ArraySectionExpr>(Base))
7607 Base = OASE->getBase()->IgnoreParenImpCasts();
7608 while (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Base))
7609 Base = ASE->getBase()->IgnoreParenImpCasts();
7610 return cast<VarDecl>(cast<DeclRefExpr>(Base)->getDecl());
7611}
7612
7614 const OMPUseDeviceAddrClause &C, OMPPrivateScope &PrivateScope,
7615 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
7616 CaptureDeviceAddrMap) {
7617 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
7618 for (const Expr *Ref : C.varlist()) {
7619 const VarDecl *OrigVD = getBaseDecl(Ref);
7620 if (!Processed.insert(OrigVD).second)
7621 continue;
7622 // In order to identify the right initializer we need to match the
7623 // declaration used by the mapping logic. In some cases we may get
7624 // OMPCapturedExprDecl that refers to the original declaration.
7625 const ValueDecl *MatchingVD = OrigVD;
7626 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
7627 // OMPCapturedExprDecl are used to privative fields of the current
7628 // structure.
7629 const auto *ME = cast<MemberExpr>(OED->getInit());
7630 assert(isa<CXXThisExpr>(ME->getBase()) &&
7631 "Base should be the current struct!");
7632 MatchingVD = ME->getMemberDecl();
7633 }
7634
7635 // If we don't have information about the current list item, move on to
7636 // the next one.
7637 auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
7638 if (InitAddrIt == CaptureDeviceAddrMap.end())
7639 continue;
7640
7641 llvm::Type *Ty = ConvertTypeForMem(OrigVD->getType().getNonReferenceType());
7642
7643 Address PrivAddr =
7644 Address(InitAddrIt->second, Ty,
7645 getContext().getTypeAlignInChars(getContext().VoidPtrTy));
7646 // For declrefs and variable length array need to load the pointer for
7647 // correct mapping, since the pointer to the data was passed to the runtime.
7648 if (isa<DeclRefExpr>(Ref->IgnoreParenImpCasts()) ||
7649 MatchingVD->getType()->isArrayType()) {
7651 OrigVD->getType().getNonReferenceType());
7652 PrivAddr =
7654 PtrTy->castAs<PointerType>());
7655 }
7656
7657 (void)PrivateScope.addPrivate(OrigVD, PrivAddr);
7658 }
7659}
7660
7661// Generate the instructions for '#pragma omp target data' directive.
7663 const OMPTargetDataDirective &S) {
7664 CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true,
7665 /*SeparateBeginEndCalls=*/true);
7666
7667 // Create a pre/post action to signal the privatization of the device pointer.
7668 // This action can be replaced by the OpenMP runtime code generation to
7669 // deactivate privatization.
7670 bool PrivatizeDevicePointers = false;
7671 class DevicePointerPrivActionTy : public PrePostActionTy {
7672 bool &PrivatizeDevicePointers;
7673
7674 public:
7675 explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers)
7676 : PrivatizeDevicePointers(PrivatizeDevicePointers) {}
7677 void Enter(CodeGenFunction &CGF) override {
7678 PrivatizeDevicePointers = true;
7679 }
7680 };
7681 DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
7682
7683 auto &&CodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
7684 auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7685 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
7686 };
7687
7688 // Codegen that selects whether to generate the privatization code or not.
7689 auto &&PrivCodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
7690 RegionCodeGenTy RCG(InnermostCodeGen);
7691 PrivatizeDevicePointers = false;
7692
7693 // Call the pre-action to change the status of PrivatizeDevicePointers if
7694 // needed.
7695 Action.Enter(CGF);
7696
7697 if (PrivatizeDevicePointers) {
7698 OMPPrivateScope PrivateScope(CGF);
7699 // Emit all instances of the use_device_ptr clause.
7700 for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
7701 CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope,
7703 for (const auto *C : S.getClausesOfKind<OMPUseDeviceAddrClause>())
7704 CGF.EmitOMPUseDeviceAddrClause(*C, PrivateScope,
7706 (void)PrivateScope.Privatize();
7707 RCG(CGF);
7708 } else {
7709 // If we don't have target devices, don't bother emitting the data
7710 // mapping code.
7711 std::optional<OpenMPDirectiveKind> CaptureRegion;
7712 if (CGM.getLangOpts().OMPTargetTriples.empty()) {
7713 // Emit helper decls of the use_device_ptr/use_device_addr clauses.
7714 for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
7715 for (const Expr *E : C->varlist()) {
7716 const Decl *D = cast<DeclRefExpr>(E)->getDecl();
7717 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
7718 CGF.EmitVarDecl(*OED);
7719 }
7720 for (const auto *C : S.getClausesOfKind<OMPUseDeviceAddrClause>())
7721 for (const Expr *E : C->varlist()) {
7722 const Decl *D = getBaseDecl(E);
7723 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
7724 CGF.EmitVarDecl(*OED);
7725 }
7726 } else {
7727 CaptureRegion = OMPD_unknown;
7728 }
7729
7730 OMPLexicalScope Scope(CGF, S, CaptureRegion);
7731 RCG(CGF);
7732 }
7733 };
7734
7735 // Forward the provided action to the privatization codegen.
7736 RegionCodeGenTy PrivRCG(PrivCodeGen);
7737 PrivRCG.setAction(Action);
7738
7739 // Notwithstanding the body of the region is emitted as inlined directive,
7740 // we don't use an inline scope as changes in the references inside the
7741 // region are expected to be visible outside, so we do not privative them.
7742 OMPLexicalScope Scope(CGF, S);
7743 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data,
7744 PrivRCG);
7745 };
7746
7748
7749 // If we don't have target devices, don't bother emitting the data mapping
7750 // code.
7751 if (CGM.getLangOpts().OMPTargetTriples.empty()) {
7752 RCG(*this);
7753 return;
7754 }
7755
7756 // Check if we have any if clause associated with the directive.
7757 const Expr *IfCond = nullptr;
7758 if (const auto *C = S.getSingleClause<OMPIfClause>())
7759 IfCond = C->getCondition();
7760
7761 // Check if we have any device clause associated with the directive.
7762 const Expr *Device = nullptr;
7763 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
7764 Device = C->getDevice();
7765
7766 // Set the action to signal privatization of device pointers.
7767 RCG.setAction(PrivAction);
7768
7769 // Emit region code.
7770 CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG,
7771 Info);
7772}
7773
7775 const OMPTargetEnterDataDirective &S) {
7776 // If we don't have target devices, don't bother emitting the data mapping
7777 // code.
7778 if (CGM.getLangOpts().OMPTargetTriples.empty())
7779 return;
7780
7781 // Check if we have any if clause associated with the directive.
7782 const Expr *IfCond = nullptr;
7783 if (const auto *C = S.getSingleClause<OMPIfClause>())
7784 IfCond = C->getCondition();
7785
7786 // Check if we have any device clause associated with the directive.
7787 const Expr *Device = nullptr;
7788 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
7789 Device = C->getDevice();
7790
7791 OMPLexicalScope Scope(*this, S, OMPD_task);
7792 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
7793}
7794
7796 const OMPTargetExitDataDirective &S) {
7797 // If we don't have target devices, don't bother emitting the data mapping
7798 // code.
7799 if (CGM.getLangOpts().OMPTargetTriples.empty())
7800 return;
7801
7802 // Check if we have any if clause associated with the directive.
7803 const Expr *IfCond = nullptr;
7804 if (const auto *C = S.getSingleClause<OMPIfClause>())
7805 IfCond = C->getCondition();
7806
7807 // Check if we have any device clause associated with the directive.
7808 const Expr *Device = nullptr;
7809 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
7810 Device = C->getDevice();
7811
7812 OMPLexicalScope Scope(*this, S, OMPD_task);
7813 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
7814}
7815
7818 PrePostActionTy &Action) {
7819 // Get the captured statement associated with the 'parallel' region.
7820 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
7821 Action.Enter(CGF);
7822 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
7823 Action.Enter(CGF);
7824 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7825 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
7826 CGF.EmitOMPPrivateClause(S, PrivateScope);
7827 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
7828 (void)PrivateScope.Privatize();
7829 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
7831 // TODO: Add support for clauses.
7832 CGF.EmitStmt(CS->getCapturedStmt());
7833 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
7834 };
7835 emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen,
7838 [](CodeGenFunction &) { return nullptr; });
7839}
7840
7842 CodeGenModule &CGM, StringRef ParentName,
7843 const OMPTargetParallelDirective &S) {
7844 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7845 emitTargetParallelRegion(CGF, S, Action);
7846 };
7847 llvm::Function *Fn;
7848 llvm::Constant *Addr;
7849 // Emit target region as a standalone region.
7850 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7851 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
7852 assert(Fn && Addr && "Target device function emission failed.");
7853}
7854
7856 const OMPTargetParallelDirective &S) {
7857 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7858 emitTargetParallelRegion(CGF, S, Action);
7859 };
7861}
7862
7865 PrePostActionTy &Action) {
7866 Action.Enter(CGF);
7867 // Emit directive as a combined directive that consists of two implicit
7868 // directives: 'parallel' with 'for' directive.
7869 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7870 Action.Enter(CGF);
7872 CGF, OMPD_target_parallel_for, S.hasCancel());
7873 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
7875 };
7876 emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,
7878}
7879
7881 CodeGenModule &CGM, StringRef ParentName,
7883 // Emit SPMD target parallel for region as a standalone region.
7884 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7885 emitTargetParallelForRegion(CGF, S, Action);
7886 };
7887 llvm::Function *Fn;
7888 llvm::Constant *Addr;
7889 // Emit target region as a standalone region.
7890 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7891 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
7892 assert(Fn && Addr && "Target device function emission failed.");
7893}
7894
7897 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7898 emitTargetParallelForRegion(CGF, S, Action);
7899 };
7901}
7902
7903static void
7906 PrePostActionTy &Action) {
7907 Action.Enter(CGF);
7908 // Emit directive as a combined directive that consists of two implicit
7909 // directives: 'parallel' with 'for' directive.
7910 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7911 Action.Enter(CGF);
7912 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
7914 };
7915 emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen,
7917}
7918
7920 CodeGenModule &CGM, StringRef ParentName,
7922 // Emit SPMD target parallel for region as a standalone region.
7923 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7924 emitTargetParallelForSimdRegion(CGF, S, Action);
7925 };
7926 llvm::Function *Fn;
7927 llvm::Constant *Addr;
7928 // Emit target region as a standalone region.
7929 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7930 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
7931 assert(Fn && Addr && "Target device function emission failed.");
7932}
7933
7936 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7937 emitTargetParallelForSimdRegion(CGF, S, Action);
7938 };
7940}
7941
7942/// Emit a helper variable and return corresponding lvalue.
7943static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,
7944 const ImplicitParamDecl *PVD,
7946 const auto *VDecl = cast<VarDecl>(Helper->getDecl());
7947 Privates.addPrivate(VDecl, CGF.GetAddrOfLocalVar(PVD));
7948}
7949
7951 assert(isOpenMPTaskLoopDirective(S.getDirectiveKind()));
7952 // Emit outlined function for task construct.
7953 const CapturedStmt *CS = S.getCapturedStmt(OMPD_taskloop);
7954 Address CapturedStruct = Address::invalid();
7955 {
7956 OMPLexicalScope Scope(*this, S, OMPD_taskloop, /*EmitPreInitStmt=*/false);
7957 CapturedStruct = GenerateCapturedStmtArgument(*CS);
7958 }
7959 CanQualType SharedsTy =
7961 const Expr *IfCond = nullptr;
7962 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
7963 if (C->getNameModifier() == OMPD_unknown ||
7964 C->getNameModifier() == OMPD_taskloop) {
7965 IfCond = C->getCondition();
7966 break;
7967 }
7968 }
7969
7971 // Check if taskloop must be emitted without taskgroup.
7972 Data.Nogroup = S.getSingleClause<OMPNogroupClause>();
7973 // TODO: Check if we should emit tied or untied task.
7974 Data.Tied = true;
7975 // Set scheduling for taskloop
7976 if (const auto *Clause = S.getSingleClause<OMPGrainsizeClause>()) {
7977 // grainsize clause
7978 Data.Schedule.setInt(/*IntVal=*/false);
7979 Data.Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize()));
7980 Data.HasModifier =
7981 (Clause->getModifier() == OMPC_GRAINSIZE_strict) ? true : false;
7982 } else if (const auto *Clause = S.getSingleClause<OMPNumTasksClause>()) {
7983 // num_tasks clause
7984 Data.Schedule.setInt(/*IntVal=*/true);
7985 Data.Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks()));
7986 Data.HasModifier =
7987 (Clause->getModifier() == OMPC_NUMTASKS_strict) ? true : false;
7988 }
7989
7990 auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
7991 // if (PreCond) {
7992 // for (IV in 0..LastIteration) BODY;
7993 // <Final counter/linear vars updates>;
7994 // }
7995 //
7996
7997 // Emit: if (PreCond) - begin.
7998 // If the condition constant folds and can be elided, avoid emitting the
7999 // whole loop.
8000 bool CondConstant;
8001 llvm::BasicBlock *ContBlock = nullptr;
8002 OMPLoopScope PreInitScope(CGF, S);
8003 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
8004 if (!CondConstant)
8005 return;
8006 } else {
8007 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then");
8008 ContBlock = CGF.createBasicBlock("taskloop.if.end");
8009 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
8010 CGF.getProfileCount(&S));
8011 CGF.EmitBlock(ThenBlock);
8012 CGF.incrementProfileCounter(&S);
8013 }
8014
8015 (void)CGF.EmitOMPLinearClauseInit(S);
8016
8017 OMPPrivateScope LoopScope(CGF);
8018 // Emit helper vars inits.
8019 enum { LowerBound = 5, UpperBound, Stride, LastIter };
8020 auto *I = CS->getCapturedDecl()->param_begin();
8021 auto *LBP = std::next(I, LowerBound);
8022 auto *UBP = std::next(I, UpperBound);
8023 auto *STP = std::next(I, Stride);
8024 auto *LIP = std::next(I, LastIter);
8025 mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP,
8026 LoopScope);
8027 mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP,
8028 LoopScope);
8029 mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope);
8030 mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP,
8031 LoopScope);
8032 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
8033 CGF.EmitOMPLinearClause(S, LoopScope);
8034 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
8035 (void)LoopScope.Privatize();
8036 // Emit the loop iteration variable.
8037 const Expr *IVExpr = S.getIterationVariable();
8038 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
8039 CGF.EmitVarDecl(*IVDecl);
8040 CGF.EmitIgnoredExpr(S.getInit());
8041
8042 // Emit the iterations count variable.
8043 // If it is not a variable, Sema decided to calculate iterations count on
8044 // each iteration (e.g., it is foldable into a constant).
8045 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
8046 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
8047 // Emit calculation of the iterations count.
8048 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
8049 }
8050
8051 {
8052 OMPLexicalScope Scope(CGF, S, OMPD_taskloop, /*EmitPreInitStmt=*/false);
8054 CGF, S,
8055 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
8056 if (isOpenMPSimdDirective(S.getDirectiveKind()))
8057 CGF.EmitOMPSimdInit(S);
8058 },
8059 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
8060 CGF.EmitOMPInnerLoop(
8061 S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(),
8062 [&S](CodeGenFunction &CGF) {
8063 emitOMPLoopBodyWithStopPoint(CGF, S,
8064 CodeGenFunction::JumpDest());
8065 },
8066 [](CodeGenFunction &) {});
8067 });
8068 }
8069 // Emit: if (PreCond) - end.
8070 if (ContBlock) {
8071 CGF.EmitBranch(ContBlock);
8072 CGF.EmitBlock(ContBlock, true);
8073 }
8074 // Emit final copy of the lastprivate variables if IsLastIter != 0.
8075 if (HasLastprivateClause) {
8076 CGF.EmitOMPLastprivateClauseFinal(
8077 S, isOpenMPSimdDirective(S.getDirectiveKind()),
8078 CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
8079 CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
8080 (*LIP)->getType(), S.getBeginLoc())));
8081 }
8082 LoopScope.restoreMap();
8083 CGF.EmitOMPLinearClauseFinal(S, [LIP, &S](CodeGenFunction &CGF) {
8084 return CGF.Builder.CreateIsNotNull(
8085 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
8086 (*LIP)->getType(), S.getBeginLoc()));
8087 });
8088 };
8089 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
8090 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
8091 const OMPTaskDataTy &Data) {
8092 auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,
8093 &Data](CodeGenFunction &CGF, PrePostActionTy &) {
8094 OMPLoopScope PreInitScope(CGF, S);
8095 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S,
8096 OutlinedFn, SharedsTy,
8097 CapturedStruct, IfCond, Data);
8098 };
8099 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
8100 CodeGen);
8101 };
8102 if (Data.Nogroup) {
8103 EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, Data);
8104 } else {
8105 CGM.getOpenMPRuntime().emitTaskgroupRegion(
8106 *this,
8107 [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF,
8108 PrePostActionTy &Action) {
8109 Action.Enter(CGF);
8110 CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen,
8111 Data);
8112 },
8113 S.getBeginLoc());
8114 }
8115}
8116
8122
8124 const OMPTaskLoopSimdDirective &S) {
8125 auto LPCRegion =
8127 OMPLexicalScope Scope(*this, S);
8129}
8130
8132 const OMPMasterTaskLoopDirective &S) {
8133 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8134 Action.Enter(CGF);
8136 };
8137 auto LPCRegion =
8139 OMPLexicalScope Scope(*this, S, std::nullopt, /*EmitPreInitStmt=*/false);
8140 CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc());
8141}
8142
8144 const OMPMaskedTaskLoopDirective &S) {
8145 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8146 Action.Enter(CGF);
8148 };
8149 auto LPCRegion =
8151 OMPLexicalScope Scope(*this, S, std::nullopt, /*EmitPreInitStmt=*/false);
8152 CGM.getOpenMPRuntime().emitMaskedRegion(*this, CodeGen, S.getBeginLoc());
8153}
8154
8157 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8158 Action.Enter(CGF);
8160 };
8161 auto LPCRegion =
8163 OMPLexicalScope Scope(*this, S);
8164 CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc());
8165}
8166
8169 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8170 Action.Enter(CGF);
8172 };
8173 auto LPCRegion =
8175 OMPLexicalScope Scope(*this, S);
8176 CGM.getOpenMPRuntime().emitMaskedRegion(*this, CodeGen, S.getBeginLoc());
8177}
8178
8181 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8182 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8183 PrePostActionTy &Action) {
8184 Action.Enter(CGF);
8186 };
8187 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
8188 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen,
8189 S.getBeginLoc());
8190 };
8191 auto LPCRegion =
8193 emitCommonOMPParallelDirective(*this, S, OMPD_master_taskloop, CodeGen,
8195}
8196
8199 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8200 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8201 PrePostActionTy &Action) {
8202 Action.Enter(CGF);
8204 };
8205 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
8206 CGM.getOpenMPRuntime().emitMaskedRegion(CGF, TaskLoopCodeGen,
8207 S.getBeginLoc());
8208 };
8209 auto LPCRegion =
8211 emitCommonOMPParallelDirective(*this, S, OMPD_masked_taskloop, CodeGen,
8213}
8214
8217 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8218 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8219 PrePostActionTy &Action) {
8220 Action.Enter(CGF);
8222 };
8223 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
8224 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen,
8225 S.getBeginLoc());
8226 };
8227 auto LPCRegion =
8229 emitCommonOMPParallelDirective(*this, S, OMPD_master_taskloop_simd, CodeGen,
8231}
8232
8235 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8236 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8237 PrePostActionTy &Action) {
8238 Action.Enter(CGF);
8240 };
8241 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
8242 CGM.getOpenMPRuntime().emitMaskedRegion(CGF, TaskLoopCodeGen,
8243 S.getBeginLoc());
8244 };
8245 auto LPCRegion =
8247 emitCommonOMPParallelDirective(*this, S, OMPD_masked_taskloop_simd, CodeGen,
8249}
8250
8251// Generate the instructions for '#pragma omp target update' directive.
8253 const OMPTargetUpdateDirective &S) {
8254 // If we don't have target devices, don't bother emitting the data mapping
8255 // code.
8256 if (CGM.getLangOpts().OMPTargetTriples.empty())
8257 return;
8258
8259 // Check if we have any if clause associated with the directive.
8260 const Expr *IfCond = nullptr;
8261 if (const auto *C = S.getSingleClause<OMPIfClause>())
8262 IfCond = C->getCondition();
8263
8264 // Check if we have any device clause associated with the directive.
8265 const Expr *Device = nullptr;
8266 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
8267 Device = C->getDevice();
8268
8269 OMPLexicalScope Scope(*this, S, OMPD_task);
8270 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
8271}
8272
8274 const OMPGenericLoopDirective &S) {
8275 // Always expect a bind clause on the loop directive. It it wasn't
8276 // in the source, it should have been added in sema.
8277
8279 if (const auto *C = S.getSingleClause<OMPBindClause>())
8280 BindKind = C->getBindKind();
8281
8282 switch (BindKind) {
8283 case OMPC_BIND_parallel: // for
8284 return emitOMPForDirective(S, *this, CGM, /*HasCancel=*/false);
8285 case OMPC_BIND_teams: // distribute
8286 return emitOMPDistributeDirective(S, *this, CGM);
8287 case OMPC_BIND_thread: // simd
8288 return emitOMPSimdDirective(S, *this, CGM);
8289 case OMPC_BIND_unknown:
8290 break;
8291 }
8292
8293 // Unimplemented, just inline the underlying statement for now.
8294 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8295 // Emit the loop iteration variable.
8296 const Stmt *CS =
8297 cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt();
8298 const auto *ForS = dyn_cast<ForStmt>(CS);
8299 if (ForS && !isa<DeclStmt>(ForS->getInit())) {
8300 OMPPrivateScope LoopScope(CGF);
8301 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
8302 (void)LoopScope.Privatize();
8303 CGF.EmitStmt(CS);
8304 LoopScope.restoreMap();
8305 } else {
8306 CGF.EmitStmt(CS);
8307 }
8308 };
8309 OMPLexicalScope Scope(*this, S, OMPD_unknown);
8310 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_loop, CodeGen);
8311}
8312
8314 const OMPLoopDirective &S) {
8315 // Emit combined directive as if its constituent constructs are 'parallel'
8316 // and 'for'.
8317 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8318 Action.Enter(CGF);
8319 emitOMPCopyinClause(CGF, S);
8320 (void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
8321 };
8322 {
8323 auto LPCRegion =
8325 emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,
8327 }
8328 // Check for outer lastprivate conditional update.
8330}
8331
8334 // To be consistent with current behavior of 'target teams loop', emit
8335 // 'teams loop' as if its constituent constructs are 'teams' and 'distribute'.
8336 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
8338 };
8339
8340 // Emit teams region as a standalone region.
8341 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
8342 PrePostActionTy &Action) {
8343 Action.Enter(CGF);
8344 OMPPrivateScope PrivateScope(CGF);
8345 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
8346 (void)PrivateScope.Privatize();
8347 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
8348 CodeGenDistribute);
8349 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
8350 };
8351 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
8353 [](CodeGenFunction &) { return nullptr; });
8354}
8355
8356#ifndef NDEBUG
8358 std::string StatusMsg,
8359 const OMPExecutableDirective &D) {
8360 bool IsDevice = CGF.CGM.getLangOpts().OpenMPIsTargetDevice;
8361 if (IsDevice)
8362 StatusMsg += ": DEVICE";
8363 else
8364 StatusMsg += ": HOST";
8365 SourceLocation L = D.getBeginLoc();
8366 auto &SM = CGF.getContext().getSourceManager();
8367 PresumedLoc PLoc = SM.getPresumedLoc(L);
8368 const char *FileName = PLoc.isValid() ? PLoc.getFilename() : nullptr;
8369 unsigned LineNo =
8370 PLoc.isValid() ? PLoc.getLine() : SM.getExpansionLineNumber(L);
8371 llvm::dbgs() << StatusMsg << ": " << FileName << ": " << LineNo << "\n";
8372}
8373#endif
8374
8376 CodeGenFunction &CGF, PrePostActionTy &Action,
8378 Action.Enter(CGF);
8379 // Emit 'teams loop' as if its constituent constructs are 'distribute,
8380 // 'parallel, and 'for'.
8381 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
8383 S.getDistInc());
8384 };
8385
8386 // Emit teams region as a standalone region.
8387 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
8388 PrePostActionTy &Action) {
8389 Action.Enter(CGF);
8390 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
8391 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
8392 (void)PrivateScope.Privatize();
8394 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
8395 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
8396 };
8397 DEBUG_WITH_TYPE(TTL_CODEGEN_TYPE,
8399 CGF, TTL_CODEGEN_TYPE " as parallel for", S));
8400 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for,
8401 CodeGenTeams);
8403 [](CodeGenFunction &) { return nullptr; });
8404}
8405
8407 CodeGenFunction &CGF, PrePostActionTy &Action,
8409 Action.Enter(CGF);
8410 // Emit 'teams loop' as if its constituent construct is 'distribute'.
8411 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
8413 };
8414
8415 // Emit teams region as a standalone region.
8416 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
8417 PrePostActionTy &Action) {
8418 Action.Enter(CGF);
8419 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
8420 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
8421 (void)PrivateScope.Privatize();
8423 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
8424 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
8425 };
8426 DEBUG_WITH_TYPE(TTL_CODEGEN_TYPE,
8428 CGF, TTL_CODEGEN_TYPE " as distribute", S));
8429 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen);
8431 [](CodeGenFunction &) { return nullptr; });
8432}
8433
8436 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8437 if (S.canBeParallelFor())
8439 else
8441 };
8443}
8444
8446 CodeGenModule &CGM, StringRef ParentName,
8448 // Emit SPMD target parallel loop region as a standalone region.
8449 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8450 if (S.canBeParallelFor())
8452 else
8454 };
8455 llvm::Function *Fn;
8456 llvm::Constant *Addr;
8457 // Emit target region as a standalone region.
8458 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
8459 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
8460 assert(Fn && Addr &&
8461 "Target device function emission failed for 'target teams loop'.");
8462}
8463
8466 PrePostActionTy &Action) {
8467 Action.Enter(CGF);
8468 // Emit as 'parallel for'.
8469 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8470 Action.Enter(CGF);
8472 CGF, OMPD_target_parallel_loop, /*hasCancel=*/false);
8473 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
8475 };
8476 emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,
8478}
8479
8481 CodeGenModule &CGM, StringRef ParentName,
8483 // Emit target parallel loop region as a standalone region.
8484 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8486 };
8487 llvm::Function *Fn;
8488 llvm::Constant *Addr;
8489 // Emit target region as a standalone region.
8490 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
8491 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
8492 assert(Fn && Addr && "Target device function emission failed.");
8493}
8494
8495/// Emit combined directive 'target parallel loop' as if its constituent
8496/// constructs are 'target', 'parallel', and 'for'.
8499 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8501 };
8503}
8504
8506 const OMPExecutableDirective &D) {
8507 if (const auto *SD = dyn_cast<OMPScanDirective>(&D)) {
8509 return;
8510 }
8511 if (!D.hasAssociatedStmt() || !D.getAssociatedStmt())
8512 return;
8513 auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) {
8514 OMPPrivateScope GlobalsScope(CGF);
8515 if (isOpenMPTaskingDirective(D.getDirectiveKind())) {
8516 // Capture global firstprivates to avoid crash.
8517 for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
8518 for (const Expr *Ref : C->varlist()) {
8519 const auto *DRE = cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
8520 if (!DRE)
8521 continue;
8522 const auto *VD = dyn_cast<VarDecl>(DRE->getDecl());
8523 if (!VD || VD->hasLocalStorage())
8524 continue;
8525 if (!CGF.LocalDeclMap.count(VD)) {
8526 LValue GlobLVal = CGF.EmitLValue(Ref);
8527 GlobalsScope.addPrivate(VD, GlobLVal.getAddress());
8528 }
8529 }
8530 }
8531 }
8532 if (isOpenMPSimdDirective(D.getDirectiveKind())) {
8533 (void)GlobalsScope.Privatize();
8534 ParentLoopDirectiveForScanRegion ScanRegion(CGF, D);
8536 } else {
8537 if (const auto *LD = dyn_cast<OMPLoopDirective>(&D)) {
8538 for (const Expr *E : LD->counters()) {
8539 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
8540 if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(VD)) {
8541 LValue GlobLVal = CGF.EmitLValue(E);
8542 GlobalsScope.addPrivate(VD, GlobLVal.getAddress());
8543 }
8544 if (isa<OMPCapturedExprDecl>(VD)) {
8545 // Emit only those that were not explicitly referenced in clauses.
8546 if (!CGF.LocalDeclMap.count(VD))
8547 CGF.EmitVarDecl(*VD);
8548 }
8549 }
8550 for (const auto *C : D.getClausesOfKind<OMPOrderedClause>()) {
8551 if (!C->getNumForLoops())
8552 continue;
8553 for (unsigned I = LD->getLoopsNumber(),
8554 E = C->getLoopNumIterations().size();
8555 I < E; ++I) {
8556 if (const auto *VD = dyn_cast<OMPCapturedExprDecl>(
8557 cast<DeclRefExpr>(C->getLoopCounter(I))->getDecl())) {
8558 // Emit only those that were not explicitly referenced in clauses.
8559 if (!CGF.LocalDeclMap.count(VD))
8560 CGF.EmitVarDecl(*VD);
8561 }
8562 }
8563 }
8564 }
8565 (void)GlobalsScope.Privatize();
8566 CGF.EmitStmt(D.getInnermostCapturedStmt()->getCapturedStmt());
8567 }
8568 };
8569 if (D.getDirectiveKind() == OMPD_atomic ||
8570 D.getDirectiveKind() == OMPD_critical ||
8571 D.getDirectiveKind() == OMPD_section ||
8572 D.getDirectiveKind() == OMPD_master ||
8573 D.getDirectiveKind() == OMPD_masked ||
8574 D.getDirectiveKind() == OMPD_unroll ||
8575 D.getDirectiveKind() == OMPD_assume) {
8576 EmitStmt(D.getAssociatedStmt());
8577 } else {
8578 auto LPCRegion =
8580 OMPSimdLexicalScope Scope(*this, D);
8581 CGM.getOpenMPRuntime().emitInlinedDirective(
8582 *this,
8583 isOpenMPSimdDirective(D.getDirectiveKind()) ? OMPD_simd
8584 : D.getDirectiveKind(),
8585 CodeGen);
8586 }
8587 // Check for outer lastprivate conditional update.
8589}
8590
8592 EmitStmt(S.getAssociatedStmt());
8593}
Defines the clang::ASTContext interface.
#define V(N, I)
static bool isAllocatableDecl(const VarDecl *VD)
static const VarDecl * getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE)
static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S, PrePostActionTy &Action)
static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S, PrePostActionTy &Action)
static const VarDecl * getBaseDecl(const Expr *Ref)
static void emitTargetTeamsGenericLoopRegionAsParallel(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsGenericLoopDirective &S)
static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, const Expr *X, const Expr *V, SourceLocation Loc)
static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, bool IsPostfixUpdate, const Expr *V, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
static void emitScanBasedDirective(CodeGenFunction &CGF, const OMPLoopDirective &S, llvm::function_ref< llvm::Value *(CodeGenFunction &)> NumIteratorsGen, llvm::function_ref< void(CodeGenFunction &)> FirstGen, llvm::function_ref< void(CodeGenFunction &)> SecondGen)
Emits the code for the directive with inscan reductions.
static void emitSimpleAtomicStore(CodeGenFunction &CGF, llvm::AtomicOrdering AO, LValue LVal, RValue RVal)
static bool isSupportedByOpenMPIRBuilder(const OMPTaskgroupDirective &T)
static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, QualType DstType, StringRef Name, LValue AddrLV)
static void emitDistributeParallelForDistributeInnerBoundParams(CodeGenFunction &CGF, const OMPExecutableDirective &S, llvm::SmallVectorImpl< llvm::Value * > &CapturedVars)
static void emitScanBasedDirectiveFinals(CodeGenFunction &CGF, const OMPLoopDirective &S, llvm::function_ref< llvm::Value *(CodeGenFunction &)> NumIteratorsGen)
Copies final inscan reductions values to the original variables.
static void checkForLastprivateConditionalUpdate(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static std::pair< LValue, LValue > emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
The following two functions generate expressions for the loop lower and upper bounds in case of stati...
static void emitTargetParallelForRegion(CodeGenFunction &CGF, const OMPTargetParallelForDirective &S, PrePostActionTy &Action)
static LValue EmitOMPHelperVar(CodeGenFunction &CGF, const DeclRefExpr *Helper)
Emit a helper variable and return corresponding lvalue.
static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
static llvm::Value * convertToScalarValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
static llvm::Function * emitOutlinedOrderedFunction(CodeGenModule &CGM, const CapturedStmt *S, const OMPExecutableDirective &D)
static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
static std::pair< bool, RValue > emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, RValue Update, BinaryOperatorKind BO, llvm::AtomicOrdering AO, bool IsXLHSInRHSPart)
static std::pair< LValue, LValue > emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitTargetTeamsGenericLoopRegionAsDistribute(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsGenericLoopDirective &S)
static void emitTargetParallelRegion(CodeGenFunction &CGF, const OMPTargetParallelDirective &S, PrePostActionTy &Action)
static std::pair< llvm::Value *, llvm::Value * > emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
When dealing with dispatch schedules (e.g.
static void emitMaster(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitRestoreIP(CodeGenFunction &CGF, const T *C, llvm::OpenMPIRBuilder::InsertPointTy AllocaIP, llvm::OpenMPIRBuilder &OMPBuilder)
static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, const RegionCodeGenTy &CodeGen)
static void emitSimdlenSafelenClause(CodeGenFunction &CGF, const OMPExecutableDirective &D)
static void emitAlignedClause(CodeGenFunction &CGF, const OMPExecutableDirective &D)
static bool isSimdSupportedByOpenMPIRBuilder(const OMPLoopDirective &S)
static void emitCommonOMPParallelDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, const CodeGenBoundParametersTy &CodeGenBoundParameters)
static void applyConservativeSimdOrderedDirective(const Stmt &AssociatedStmt, LoopInfoStack &LoopStack)
static bool emitWorksharingDirective(CodeGenFunction &CGF, const OMPLoopDirective &S, bool HasCancel)
static void emitPostUpdateForReductionClause(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc, const unsigned IVSize, const bool IVSigned)
static void emitTargetTeamsLoopCodegenStatus(CodeGenFunction &CGF, std::string StatusMsg, const OMPExecutableDirective &D)
static bool isForSupportedByOpenMPIRBuilder(const OMPLoopDirective &S, bool HasCancel)
static RValue emitSimpleAtomicLoad(CodeGenFunction &CGF, llvm::AtomicOrdering AO, LValue LVal, SourceLocation Loc)
static std::pair< llvm::Value *, llvm::Value * > emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
if the 'for' loop has a dispatch schedule (e.g.
static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO, bool IsPostfixUpdate, const Expr *X, const Expr *V, const Expr *R, const Expr *E, const Expr *UE, const Expr *D, const Expr *CE, bool IsXLHSInRHSPart, bool IsFailOnly, SourceLocation Loc)
#define TTL_CODEGEN_TYPE
static CodeGenFunction::ComplexPairTy convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
static ImplicitParamDecl * createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data, QualType Ty, CapturedDecl *CD, SourceLocation Loc)
static EmittedClosureTy emitCapturedStmtFunc(CodeGenFunction &ParentCGF, const CapturedStmt *S)
Emit a captured statement and return the function as well as its captured closure context.
static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
static void emitOMPDistributeDirective(const OMPLoopDirective &S, CodeGenFunction &CGF, CodeGenModule &CGM)
static void emitOMPCopyinClause(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitTargetTeamsDistributeParallelForRegion(CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S, PrePostActionTy &Action)
static bool hasOrderedDirective(const Stmt *S)
static llvm::CallInst * emitCapturedStmtCall(CodeGenFunction &ParentCGF, EmittedClosureTy Cap, llvm::ArrayRef< llvm::Value * > Args)
Emit a call to a previously captured closure.
static void emitMasked(CodeGenFunction &CGF, const OMPExecutableDirective &S)
static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop, int MaxLevel, int Level=0)
static void emitOMPForDirective(const OMPLoopDirective &S, CodeGenFunction &CGF, CodeGenModule &CGM, bool HasCancel)
static void emitEmptyBoundParameters(CodeGenFunction &, const OMPExecutableDirective &, llvm::SmallVectorImpl< llvm::Value * > &)
static void emitTargetParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetParallelForSimdDirective &S, PrePostActionTy &Action)
static void emitOMPSimdDirective(const OMPLoopDirective &S, CodeGenFunction &CGF, CodeGenModule &CGM)
static void emitOMPAtomicCompareExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO, const Expr *X, const Expr *V, const Expr *R, const Expr *E, const Expr *D, const Expr *CE, bool IsXBinopExpr, bool IsPostfixUpdate, bool IsFailOnly, SourceLocation Loc)
std::pair< llvm::Function *, llvm::Value * > EmittedClosureTy
static OpenMPDirectiveKind getEffectiveDirectiveKind(const OMPExecutableDirective &S)
static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDirective &S)
static void buildDependences(const OMPExecutableDirective &S, OMPTaskDataTy &Data)
static RValue convertToType(CodeGenFunction &CGF, RValue Value, QualType SourceType, QualType ResType, SourceLocation Loc)
static void emitScanBasedDirectiveDecls(CodeGenFunction &CGF, const OMPLoopDirective &S, llvm::function_ref< llvm::Value *(CodeGenFunction &)> NumIteratorsGen)
Emits internal temp array declarations for the directive with inscan reductions.
static void emitTargetTeamsDistributeParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForSimdDirective &S, PrePostActionTy &Action)
static void emitTargetTeamsDistributeSimdRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeSimdDirective &S)
static llvm::MapVector< llvm::Value *, llvm::Value * > GetAlignedMapping(const OMPLoopDirective &S, CodeGenFunction &CGF)
static llvm::omp::ScheduleKind convertClauseKindToSchedKind(OpenMPScheduleClauseKind ScheduleClauseKind)
static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, const ImplicitParamDecl *PVD, CodeGenFunction::OMPPrivateScope &Privates)
Emit a helper variable and return corresponding lvalue.
static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
static void emitTargetParallelGenericLoopRegion(CodeGenFunction &CGF, const OMPTargetParallelGenericLoopDirective &S, PrePostActionTy &Action)
static QualType getCanonicalParamType(ASTContext &C, QualType T)
static void emitCommonSimdLoop(CodeGenFunction &CGF, const OMPLoopDirective &S, const RegionCodeGenTy &SimdInitGen, const RegionCodeGenTy &BodyCodeGen)
static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, const Twine &Name, llvm::Value *Init=nullptr)
static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, const Expr *X, const Expr *E, SourceLocation Loc)
static llvm::Function * emitOutlinedFunctionPrologue(CodeGenFunction &CGF, FunctionArgList &Args, llvm::MapVector< const Decl *, std::pair< const VarDecl *, Address > > &LocalAddrs, llvm::DenseMap< const Decl *, std::pair< const Expr *, llvm::Value * > > &VLASizes, llvm::Value *&CXXThisValue, const FunctionOptions &FO)
static void emitInnerParallelForWhenCombined(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
static void emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeDirective &S)
This file defines OpenMP nodes for declarative directives.
TokenType getType() const
Returns the token's type, e.g.
FormatToken * Next
The next token in the unwrapped line.
static const Decl * getCanonicalDecl(const Decl *D)
#define X(type, name)
Definition Value.h:97
#define SM(sm)
This file defines OpenMP AST classes for clauses.
Defines some OpenMP-specific enums and functions.
Defines the PrettyStackTraceEntry class, which is used to make crashes give more contextual informati...
Defines the SourceManager interface.
This file defines OpenMP AST classes for executable directives and clauses.
This represents 'pragma omp cancel' directive.
OpenMPDirectiveKind getCancelRegion() const
Get cancellation region for the current cancellation point.
This represents 'pragma omp cancellation point' directive.
OpenMPDirectiveKind getCancelRegion() const
Get cancellation region for the current cancellation point.
The base class for all transformation directives of canonical loop sequences (currently only 'fuse')
This represents 'pragma omp distribute' directive.
This represents 'pragma omp distribute parallel for' composite directive.
This represents 'pragma omp distribute parallel for simd' composite directive.
This represents 'pragma omp distribute simd' composite directive.
This represents 'pragma omp error' directive.
Represents the 'pragma omp fuse' loop transformation directive.
Stmt * getTransformedStmt() const
Gets the associated loops after the transformation.
This represents 'pragma omp loop' directive.
Represents the 'pragma omp interchange' loop transformation directive.
Stmt * getTransformedStmt() const
Gets the associated loops after the transformation.
This represents 'pragma omp interop' directive.
This represents 'pragma omp masked' directive.
This represents 'pragma omp masked taskloop' directive.
This represents 'pragma omp masked taskloop simd' directive.
This represents 'pragma omp master taskloop' directive.
This represents 'pragma omp master taskloop simd' directive.
This represents 'pragma omp metadirective' directive.
Stmt * getIfStmt() const
This represents 'pragma omp parallel masked taskloop' directive.
This represents 'pragma omp parallel masked taskloop simd' directive.
This represents 'pragma omp parallel master taskloop' directive.
This represents 'pragma omp parallel master taskloop simd' directive.
Represents the 'pragma omp reverse' loop transformation directive.
Stmt * getTransformedStmt() const
Gets/sets the associated loops after the transformation, i.e.
This represents 'pragma omp scan' directive.
This represents the 'pragma omp stripe' loop transformation directive.
Stmt * getTransformedStmt() const
Gets/sets the associated loops after striping.
This represents 'pragma omp target data' directive.
This represents 'pragma omp target' directive.
This represents 'pragma omp target enter data' directive.
This represents 'pragma omp target exit data' directive.
This represents 'pragma omp target parallel' directive.
This represents 'pragma omp target parallel for' directive.
bool hasCancel() const
Return true if current directive has inner cancel directive.
This represents 'pragma omp target parallel for simd' directive.
This represents 'pragma omp target parallel loop' directive.
This represents 'pragma omp target simd' directive.
This represents 'pragma omp target teams' directive.
This represents 'pragma omp target teams distribute' combined directive.
This represents 'pragma omp target teams distribute parallel for' combined directive.
This represents 'pragma omp target teams distribute parallel for simd' combined directive.
This represents 'pragma omp target teams distribute simd' combined directive.
This represents 'pragma omp target teams loop' directive.
bool canBeParallelFor() const
Return true if current loop directive's associated loop can be a parallel for.
This represents 'pragma omp target update' directive.
This represents 'pragma omp taskloop' directive.
This represents 'pragma omp taskloop simd' directive.
This represents 'pragma omp teams' directive.
This represents 'pragma omp teams distribute' directive.
This represents 'pragma omp teams distribute parallel for' composite directive.
This represents 'pragma omp teams distribute parallel for simd' composite directive.
This represents 'pragma omp teams distribute simd' combined directive.
This represents 'pragma omp teams loop' directive.
This represents the 'pragma omp tile' loop transformation directive.
Stmt * getTransformedStmt() const
Gets/sets the associated loops after tiling.
This represents the 'pragma omp unroll' loop transformation directive.
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition ASTContext.h:220
SourceManager & getSourceManager()
Definition ASTContext.h:833
TranslationUnitDecl * getTranslationUnitDecl() const
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
CanQualType VoidPtrTy
IdentifierTable & Idents
Definition ASTContext.h:772
QualType getConstantArrayType(QualType EltTy, const llvm::APInt &ArySize, const Expr *SizeExpr, ArraySizeModifier ASM, unsigned IndexTypeQuals) const
Return the unique reference to the type for a constant array of the specified element type.
QualType getUIntPtrType() const
Return a type compatible with "uintptr_t" (C99 7.18.1.4), as defined by the target.
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth,...
TypeSourceInfo * getTrivialTypeSourceInfo(QualType T, SourceLocation Loc=SourceLocation()) const
Allocate a TypeSourceInfo where all locations have been initialized to a given location,...
unsigned getOpenMPDefaultSimdAlign(QualType T) const
Get default simd alignment of the specified complete type in bits.
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
CanQualType VoidTy
QualType getFunctionType(QualType ResultTy, ArrayRef< QualType > Args, const FunctionProtoType::ExtProtoInfo &EPI) const
Return a normal function type with a typed argument list.
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
CanQualType getCanonicalTagType(const TagDecl *TD) const
ASTRecordLayout - This class contains layout information for one RecordDecl, which is a struct/union/...
uint64_t getFieldOffset(unsigned FieldNo) const
getFieldOffset - Get the offset of the given field index, in bits.
Represents an array type, per C99 6.7.5.2 - Array Declarators.
Definition TypeBase.h:3722
Represents an attribute applied to a statement.
Definition Stmt.h:2203
ArrayRef< const Attr * > getAttrs() const
Definition Stmt.h:2235
static BinaryOperator * Create(const ASTContext &C, Expr *lhs, Expr *rhs, Opcode opc, QualType ResTy, ExprValueKind VK, ExprObjectKind OK, SourceLocation opLoc, FPOptionsOverride FPFeatures)
Definition Expr.cpp:4979
Represents the body of a CapturedStmt, and serves as its DeclContext.
Definition Decl.h:4923
unsigned getContextParamPosition() const
Definition Decl.h:4990
bool isNothrow() const
Definition Decl.cpp:5573
static CapturedDecl * Create(ASTContext &C, DeclContext *DC, unsigned NumParams)
Definition Decl.cpp:5558
param_iterator param_end() const
Retrieve an iterator one past the last parameter decl.
Definition Decl.h:4998
param_iterator param_begin() const
Retrieve an iterator pointing to the first parameter decl.
Definition Decl.h:4996
Stmt * getBody() const override
getBody - If this Decl represents a declaration for a body of code, such as a function or method defi...
Definition Decl.cpp:5570
ImplicitParamDecl * getParam(unsigned i) const
Definition Decl.h:4963
This captures a statement into a function.
Definition Stmt.h:3886
SourceLocation getEndLoc() const LLVM_READONLY
Definition Stmt.h:4085
CapturedDecl * getCapturedDecl()
Retrieve the outlined function declaration.
Definition Stmt.cpp:1451
child_range children()
Definition Stmt.cpp:1442
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Definition Stmt.h:4007
Stmt * getCapturedStmt()
Retrieve the statement being captured.
Definition Stmt.h:3990
capture_init_iterator capture_init_begin()
Retrieve the first initialization argument.
Definition Stmt.h:4063
SourceLocation getBeginLoc() const LLVM_READONLY
Definition Stmt.h:4081
capture_init_iterator capture_init_end()
Retrieve the iterator pointing one past the last initialization argument.
Definition Stmt.h:4073
capture_range captures()
Definition Stmt.h:4024
Expr *const * const_capture_init_iterator
Const iterator that walks over the capture initialization arguments.
Definition Stmt.h:4050
CharUnits - This is an opaque type for sizes expressed in character units.
Definition CharUnits.h:38
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
Definition CharUnits.h:185
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
Definition CharUnits.h:214
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
Definition CharUnits.h:63
CharUnits alignTo(const CharUnits &Align) const
alignTo - Returns the next integer (mod 2**64) that is greater than or equal to this quantity and is ...
Definition CharUnits.h:201
Like RawAddress, an abstract representation of an aligned address, but the pointer contained in this ...
Definition Address.h:128
static Address invalid()
Definition Address.h:176
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
Definition Address.h:253
CharUnits getAlignment() const
Definition Address.h:194
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Definition Address.h:209
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
Definition Address.h:276
Address withAlignment(CharUnits NewAlignment) const
Return address with different alignment, but same pointer and element type.
Definition Address.h:269
llvm::PointerType * getType() const
Return the type of the pointer value.
Definition Address.h:204
static AggValueSlot ignored()
ignored - Returns an aggregate value slot indicating that the aggregate value is being ignored.
Definition CGValue.h:572
static ApplyDebugLocation CreateDefaultArtificial(CodeGenFunction &CGF, SourceLocation TemporaryLocation)
Apply TemporaryLocation if it is valid.
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, llvm::Type *ElementTy, const llvm::Twine &Name="")
Definition CGBuilder.h:207
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
Definition CGBuilder.h:112
CGFunctionInfo - Class to encapsulate the information about a function definition.
Manages list of lastprivate conditional decls for the specified directive.
static LastprivateConditionalRAII disable(CodeGenFunction &CGF, const OMPExecutableDirective &S)
Manages list of nontemporal decls for the specified directive.
Struct that keeps all the relevant information that should be kept throughout a 'target data' region.
llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap
Map between the a declaration of a capture and the corresponding new llvm address where the runtime r...
Manages list of nontemporal decls for the specified directive.
virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
Emit task region for the task directive.
virtual llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST)
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars, const Expr *IfCond, llvm::Value *NumThreads, OpenMPNumThreadsClauseModifier NumThreadsModifier=OMPC_NUMTHREADS_unknown, OpenMPSeverityClauseKind Severity=OMPC_SEVERITY_fatal, const Expr *Message=nullptr)
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
virtual void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc, OpenMPNumThreadsClauseModifier Modifier=OMPC_NUMTHREADS_unknown, OpenMPSeverityClauseKind Severity=OMPC_SEVERITY_fatal, SourceLocation SeverityLoc=SourceLocation(), const Expr *Message=nullptr, SourceLocation MessageLoc=SourceLocation())
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32global_tid, kmp_int32 num_threads) ...
virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal)
Get the address of void * type of the privatue copy of the reduction item specified by the SharedLVal...
virtual void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc)
This is used for non static scheduled types and when the ordered clause is present on the loop constr...
virtual void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars)
Emits code for teams call of the OutlinedFn with variables captured in a record which address is stor...
virtual const VarDecl * translateParameter(const FieldDecl *FD, const VarDecl *NativeParam) const
Translates the native parameter of outlined function if this is required for target.
virtual llvm::Function * emitTeamsOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP teams directive D.
virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, ArrayRef< Expr * > NumIterations)
Emit initialization for doacross loop nesting support.
virtual void adjustTargetSpecificDataForLambdas(CodeGenFunction &CGF, const OMPExecutableDirective &D) const
Adjust some parameters for the target-based directives, like addresses of the variables captured by r...
virtual Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const
Gets the address of the native argument basing on the address of the target-specific parameter.
virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc)
Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32global_tid, kmp_int32 num_teams,...
virtual llvm::Value * emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, const OMPTaskDataTy &Data)
Emit a code for initialization of task reduction clause.
virtual void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr * > Vars, SourceLocation Loc, llvm::AtomicOrdering AO)
Emit flush of the variables specified in 'omp flush' directive.
virtual void emitProcBindClause(CodeGenFunction &CGF, llvm::omp::ProcBindKind ProcBind, SourceLocation Loc)
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32global_tid, int proc_bind) to generate...
virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false)
Emit an implicit/explicit barrier for OpenMP threads.
virtual void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values)
virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind)
Call the appropriate runtime routine to notify that we finished all the work with current loop.
void emitIfClause(CodeGenFunction &CGF, const Expr *Cond, const RegionCodeGenTy &ThenGen, const RegionCodeGenTy &ElseGen)
Emits code for OpenMP 'if' clause using specified CodeGen function.
virtual llvm::Function * emitParallelOutlinedFunction(CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP parallel directive D.
virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values)
Call the appropriate runtime routine to initialize it before start of loop.
virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static non-chunked.
virtual void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc)
Emits a master region.
virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Required to resolve existing problems in the runtime.
virtual void checkAndEmitLastprivateConditional(CodeGenFunction &CGF, const Expr *LHS)
Checks if the provided LVal is lastprivate conditional and emits the code to update the value of the ...
llvm::OpenMPIRBuilder & getOMPBuilder()
virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen)
Emit outilined function for 'target' directive.
virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned)
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
virtual void checkAndEmitSharedLastprivateConditional(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::DenseSet< CanonicalDeclPtr< const VarDecl > > &IgnoredDecls)
Checks if the lastprivate conditional was updated in inner region and writes the value.
virtual void emitInlinedDirective(CodeGenFunction &CGF, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool HasCancel=false)
Emit code for the directive that does not require outlining.
virtual bool isStaticChunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static chunked.
virtual void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, llvm::PointerIntPair< const Expr *, 2, OpenMPDeviceClauseModifier > Device, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter)
Emit the target offloading code associated with D.
virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const
Check if the specified ScheduleKind is dynamic.
virtual void emitMaskedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc, const Expr *Filter=nullptr)
Emits a masked region.
virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues)
Call the appropriate runtime routine to initialize it before start of loop.
Address getAllocatedAddress() const
Returns the raw, allocated address, which is not necessarily the address of the object itself.
API for captured statement code generation.
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
RAII for correct setting/restoring of CapturedStmtInfo.
LValue getReferenceLValue(CodeGenFunction &CGF, const Expr *RefExpr) const
void ForceCleanup()
Force the emission of cleanups now, instead of waiting until this object is destroyed.
RAII for preserving necessary info during inlined region body codegen.
RAII for preserving necessary info during Outlined region body codegen.
Controls insertion of cancellation exit blocks in worksharing constructs.
Save/restore original map of previously emitted local vars in case when we need to duplicate emission...
The class used to assign some variables some temporarily addresses.
bool apply(CodeGenFunction &CGF)
Applies new addresses to the list of the variables.
void restore(CodeGenFunction &CGF)
Restores original addresses of the variables.
bool setVarAddr(CodeGenFunction &CGF, const VarDecl *LocalVD, Address TempAddr)
Sets the address of the variable LocalVD to be TempAddr in function CGF.
The scope used to remap some variables as private in the OpenMP loop body (or other captured region e...
void restoreMap()
Restore all mapped variables w/o clean up.
bool Privatize()
Privatizes local variables previously registered as private.
bool addPrivate(const VarDecl *LocalVD, Address Addr)
Registers LocalVD variable as a private with Addr as the address of the corresponding private variabl...
An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
Enters a new scope for capturing cleanups, all of which will be executed once the scope is exited.
void ForceCleanup(std::initializer_list< llvm::Value ** > ValuesToReload={})
Force the emission of cleanups now, instead of waiting until this object is destroyed.
bool requiresCleanups() const
Determine whether this scope requires any cleanups.
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
void EmitOMPParallelMaskedTaskLoopDirective(const OMPParallelMaskedTaskLoopDirective &S)
void EmitOMPParallelMaskedDirective(const OMPParallelMaskedDirective &S)
void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S)
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount, Stmt::Likelihood LH=Stmt::LH_None, const Expr *ConditionalOp=nullptr, const VarDecl *ConditionalDecl=nullptr)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D, bool NoFinals, llvm::Value *IsLastIterCond=nullptr)
Emit final copying of lastprivate values to original variables at the end of the worksharing or simd ...
void processInReduction(const OMPExecutableDirective &S, OMPTaskDataTy &Data, CodeGenFunction &CGF, const CapturedStmt *CS, OMPPrivateScope &Scope)
JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target)
The given basic block lies in the current EH scope, but may be a target of a potentially scope-crossi...
void EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S)
void emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy, SourceLocation Loc)
static void EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelDirective &S)
void EmitOMPCanonicalLoop(const OMPCanonicalLoop *S)
Emit an OMPCanonicalLoop using the OpenMPIRBuilder.
void EmitOMPGenericLoopDirective(const OMPGenericLoopDirective &S)
void EmitOMPScanDirective(const OMPScanDirective &S)
static bool hasScalarEvaluationKind(QualType T)
llvm::function_ref< std::pair< llvm::Value *, llvm::Value * >(CodeGenFunction &, const OMPExecutableDirective &S, Address LB, Address UB)> CodeGenDispatchBoundsTy
LValue InitCapturedStruct(const CapturedStmt &S)
Definition CGStmt.cpp:3250
CGCapturedStmtInfo * CapturedStmtInfo
void EmitOMPDistributeDirective(const OMPDistributeDirective &S)
void EmitOMPParallelForDirective(const OMPParallelForDirective &S)
void EmitOMPMasterDirective(const OMPMasterDirective &S)
void EmitOMPParallelMasterTaskLoopSimdDirective(const OMPParallelMasterTaskLoopSimdDirective &S)
void EmitOMPSimdInit(const OMPLoopDirective &D)
Helpers for the OpenMP loop directives.
const OMPExecutableDirective * OMPParentLoopDirectiveForScan
Parent loop-based directive for scan directive.
void EmitOMPFlushDirective(const OMPFlushDirective &S)
static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetDirective &S)
Emit device code for the target directive.
bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S)
void EmitOMPTargetTeamsDistributeParallelForSimdDirective(const OMPTargetTeamsDistributeParallelForSimdDirective &S)
static void EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDirective &S)
Emit device code for the target teams directive.
void EmitOMPReductionClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope, bool ForInscan=false)
Emit initial code for reduction variables.
void EmitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &S)
void EmitAutoVarDecl(const VarDecl &D)
EmitAutoVarDecl - Emit an auto variable declaration.
Definition CGDecl.cpp:1348
static void EmitOMPTargetTeamsDistributeDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeDirective &S)
Emit device code for the target teams distribute directive.
void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S)
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
void EmitOMPTargetParallelForDirective(const OMPTargetParallelForDirective &S)
const LangOptions & getLangOpts() const
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var)
EmitAutoVarAlloca - Emit the alloca and debug information for a local variable.
Definition CGDecl.cpp:1482
void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO, const llvm::function_ref< RValue(RValue)> &UpdateOp, bool IsVolatile)
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Load a pointer with type PtrTy stored at address Ptr.
Definition CGExpr.cpp:3039
void EmitBranchThroughCleanup(JumpDest Dest)
EmitBranchThroughCleanup - Emit a branch from the current insert block through the normal cleanup han...
void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind)
Emit final update of reduction values to original variables at the end of the directive.
void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit)
Helper for the OpenMP loop directives.
void EmitOMPScopeDirective(const OMPScopeDirective &S)
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
llvm::AssertingVH< llvm::Instruction > AllocaInsertPt
AllocaInsertPoint - This is an instruction in the entry block before which we prefer to insert alloca...
void EmitAggregateAssign(LValue Dest, LValue Src, QualType EltTy)
Emit an aggregate assignment.
JumpDest ReturnBlock
ReturnBlock - Unified return block.
void EmitOMPTargetTeamsDistributeSimdDirective(const OMPTargetTeamsDistributeSimdDirective &S)
const llvm::function_ref< void(CodeGenFunction &, llvm::Function *, const OMPTaskDataTy &)> TaskGenTy
llvm::DebugLoc SourceLocToDebugLoc(SourceLocation Location)
Converts Location to a DebugLoc, if debug information is enabled.
bool EmitOMPCopyinClause(const OMPExecutableDirective &D)
Emit code for copyin clause in D directive.
void EmitOMPLinearClause(const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope)
Emit initial code for linear clauses.
llvm::BasicBlock * OMPBeforeScanBlock
void EmitOMPInterchangeDirective(const OMPInterchangeDirective &S)
void EmitOMPPrivateLoopCounters(const OMPLoopDirective &S, OMPPrivateScope &LoopScope)
Emit initial code for loop counters of loop-based directives.
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value * > &CapturedVars)
void EmitOMPDepobjDirective(const OMPDepobjDirective &S)
void EmitOMPMetaDirective(const OMPMetaDirective &S)
void EmitOMPCriticalDirective(const OMPCriticalDirective &S)
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
Definition CGExpr.cpp:242
void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S)
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
Definition CGExpr.cpp:2336
void EmitOMPCancelDirective(const OMPCancelDirective &S)
void EmitOMPBarrierDirective(const OMPBarrierDirective &S)
llvm::Value * EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified complex type to the specified destination type,...
void EmitOMPOrderedDirective(const OMPOrderedDirective &S)
bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB, const CodeGenLoopBoundsTy &CodeGenLoopBounds, const CodeGenDispatchBoundsTy &CGDispatchBounds)
Emit code for the worksharing loop-based directive.
LValue EmitOMPSharedLValue(const Expr *E)
Emits the lvalue for the expression with possibly captured variable.
llvm::CanonicalLoopInfo * EmitOMPCollapsedCanonicalLoopNest(const Stmt *S, int Depth)
Emit the Stmt S and return its topmost canonical loop, if any.
void EmitOMPSectionsDirective(const OMPSectionsDirective &S)
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
void EmitOMPInteropDirective(const OMPInteropDirective &S)
void EmitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &S)
void EmitOMPTargetParallelDirective(const OMPTargetParallelDirective &S)
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
llvm::Value * EvaluateExprAsBool(const Expr *E)
EvaluateExprAsBool - Perform the usual unary conversions on the specified expression and compare the ...
Definition CGExpr.cpp:223
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
void EmitOMPTargetParallelForSimdDirective(const OMPTargetParallelForSimdDirective &S)
void EmitOMPTargetParallelGenericLoopDirective(const OMPTargetParallelGenericLoopDirective &S)
Emit combined directive 'target parallel loop' as if its constituent constructs are 'target',...
void EmitOMPUseDeviceAddrClause(const OMPUseDeviceAddrClause &C, OMPPrivateScope &PrivateScope, const llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap)
void EmitOMPTeamsDistributeParallelForSimdDirective(const OMPTeamsDistributeParallelForSimdDirective &S)
void EmitOMPMaskedDirective(const OMPMaskedDirective &S)
llvm::Value * emitArrayLength(const ArrayType *arrayType, QualType &baseType, Address &addr)
emitArrayLength - Compute the length of an array, even if it's a VLA, and drill down to the base elem...
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
void EmitOMPTeamsDistributeSimdDirective(const OMPTeamsDistributeSimdDirective &S)
RValue EmitAtomicLoad(LValue LV, SourceLocation SL, AggValueSlot Slot=AggValueSlot::ignored())
void EmitOMPDistributeLoop(const OMPLoopDirective &S, const CodeGenLoopTy &CodeGenLoop, Expr *IncExpr)
Emit code for the distribute loop-based directive.
void EmitOMPMasterTaskLoopDirective(const OMPMasterTaskLoopDirective &S)
void EmitOMPReverseDirective(const OMPReverseDirective &S)
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
void EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S)
void EmitOMPTargetTeamsDistributeParallelForDirective(const OMPTargetTeamsDistributeParallelForDirective &S)
void EmitOMPMaskedTaskLoopDirective(const OMPMaskedTaskLoopDirective &S)
llvm::function_ref< std::pair< LValue, LValue >(CodeGenFunction &, const OMPExecutableDirective &S)> CodeGenLoopBoundsTy
void EmitOMPTargetExitDataDirective(const OMPTargetExitDataDirective &S)
void incrementProfileCounter(const Stmt *S, llvm::Value *StepV=nullptr)
Increment the profiler's counter for the given statement by StepV.
void EmitOMPTargetEnterDataDirective(const OMPTargetEnterDataDirective &S)
void EmitOMPMaskedTaskLoopSimdDirective(const OMPMaskedTaskLoopSimdDirective &S)
std::pair< bool, RValue > EmitOMPAtomicSimpleUpdateExpr(LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, llvm::AtomicOrdering AO, SourceLocation Loc, const llvm::function_ref< RValue(RValue)> CommonGen)
Emit atomic update code for constructs: X = X BO E or X = E BO E.
VlaSizePair getVLASize(const VariableArrayType *vla)
Returns an LLVM value that corresponds to the size, in non-variably-sized elements,...
void EmitOMPParallelDirective(const OMPParallelDirective &S)
void EmitOMPTaskDirective(const OMPTaskDirective &S)
void EmitOMPMasterTaskLoopSimdDirective(const OMPMasterTaskLoopSimdDirective &S)
void EmitOMPDistributeParallelForDirective(const OMPDistributeParallelForDirective &S)
void EmitOMPAssumeDirective(const OMPAssumeDirective &S)
int ExpectedOMPLoopDepth
Number of nested loop to be consumed by the last surrounding loop-associated directive.
void EmitOMPPrivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S)
void EmitStopPoint(const Stmt *S)
EmitStopPoint - Emit a debug stoppoint if we are emitting debug info.
Definition CGStmt.cpp:51
void EmitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &S)
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
void EmitOMPTargetTeamsGenericLoopDirective(const OMPTargetTeamsGenericLoopDirective &S)
void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit)
EmitStoreOfComplex - Store a complex number into the specified l-value.
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
void EmitAutoVarCleanups(const AutoVarEmission &emission)
Definition CGDecl.cpp:2202
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
Definition CGExpr.cpp:2533
SmallVector< llvm::CanonicalLoopInfo *, 4 > OMPLoopNestStack
List of recently emitted OMPCanonicalLoops.
void EmitOMPTeamsDistributeParallelForDirective(const OMPTeamsDistributeParallelForDirective &S)
llvm::AtomicRMWInst * emitAtomicRMWInst(llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val, llvm::AtomicOrdering Order=llvm::AtomicOrdering::SequentiallyConsistent, llvm::SyncScope::ID SSID=llvm::SyncScope::System, const AtomicExpr *AE=nullptr)
Emit an atomicrmw instruction, and applying relevant metadata when applicable.
void EmitOMPFuseDirective(const OMPFuseDirective &S)
void EmitOMPTargetTeamsDistributeDirective(const OMPTargetTeamsDistributeDirective &S)
void EmitOMPUseDevicePtrClause(const OMPUseDevicePtrClause &C, OMPPrivateScope &PrivateScope, const llvm::DenseMap< const ValueDecl *, llvm::Value * > CaptureDeviceAddrMap)
RValue EmitAnyExpr(const Expr *E, AggValueSlot aggSlot=AggValueSlot::ignored(), bool ignoreResult=false)
EmitAnyExpr - Emit code to compute the specified expression which can have any type.
Definition CGExpr.cpp:264
void EmitStmt(const Stmt *S, ArrayRef< const Attr * > Attrs={})
EmitStmt - Emit the code for the statement.
Definition CGStmt.cpp:61
llvm::DenseMap< const ValueDecl *, FieldDecl * > LambdaCaptureFields
void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S)
llvm::Type * ConvertTypeForMem(QualType T)
void EmitOMPInnerLoop(const OMPExecutableDirective &S, bool RequiresCleanup, const Expr *LoopCond, const Expr *IncExpr, const llvm::function_ref< void(CodeGenFunction &)> BodyGen, const llvm::function_ref< void(CodeGenFunction &)> PostIncGen)
Emit inner loop of the worksharing/simd construct.
void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S)
static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForDirective &S)
void EmitOMPTargetDirective(const OMPTargetDirective &S)
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
void EmitOMPTeamsDirective(const OMPTeamsDirective &S)
void EmitSimpleOMPExecutableDirective(const OMPExecutableDirective &D)
Emit simple code for OpenMP directives in Simd-only mode.
void EmitOMPErrorDirective(const OMPErrorDirective &S)
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
void EmitOMPParallelMaskedTaskLoopSimdDirective(const OMPParallelMaskedTaskLoopSimdDirective &S)
void EmitOMPTargetTeamsDirective(const OMPTargetTeamsDirective &S)
void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S)
Address GenerateCapturedStmtArgument(const CapturedStmt &S)
Definition CGStmt.cpp:3291
bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Emit initial code for lastprivate variables.
static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForSimdDirective &S)
Emit device code for the target teams distribute parallel for simd directive.
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block,...
Definition CGStmt.cpp:675
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S, const OMPExecutableDirective &D)
void EmitOMPSimdDirective(const OMPSimdDirective &S)
RawAddress CreateMemTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
Definition CGExpr.cpp:186
Address EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo=nullptr, TBAAAccessInfo *PointeeTBAAInfo=nullptr)
Definition CGExpr.cpp:2997
void EmitOMPParallelGenericLoopDirective(const OMPLoopDirective &S)
void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S)
void EmitOMPTeamsGenericLoopDirective(const OMPTeamsGenericLoopDirective &S)
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
Definition CGDecl.cpp:203
bool EmitOMPLinearClauseInit(const OMPLoopDirective &D)
Emit initial code for linear variables.
static void EmitOMPTargetParallelGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelGenericLoopDirective &S)
Emit device code for the target parallel loop directive.
void EmitOMPUnrollDirective(const OMPUnrollDirective &S)
void EmitOMPStripeDirective(const OMPStripeDirective &S)
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
void EmitOMPSingleDirective(const OMPSingleDirective &S)
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
llvm::function_ref< void(CodeGenFunction &, SourceLocation, const unsigned, const bool)> CodeGenOrderedTy
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
llvm::Value * EmitFromMemory(llvm::Value *Value, QualType Ty)
EmitFromMemory - Change a scalar value from its memory representation to its value representation.
Definition CGExpr.cpp:2183
static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S)
Emit device code for the target simd directive.
llvm::Function * GenerateCapturedStmtFunction(const CapturedStmt &S)
Creates the outlined function for a CapturedStmt.
Definition CGStmt.cpp:3298
static void EmitOMPTargetParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForDirective &S)
Emit device code for the target parallel for directive.
uint64_t getProfileCount(const Stmt *S)
Get the profiler's count for the given statement.
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant,...
static void EmitOMPTargetTeamsGenericLoopDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsGenericLoopDirective &S)
Emit device code for the target teams loop directive.
void EmitOMPTileDirective(const OMPTileDirective &S)
void EmitDecl(const Decl &D, bool EvaluateConditionDecl=false)
EmitDecl - Emit a declaration.
Definition CGDecl.cpp:52
void EmitOMPAtomicDirective(const OMPAtomicDirective &S)
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
ConstantEmission tryEmitAsConstant(const DeclRefExpr *RefExpr)
Try to emit a reference to the given value without producing it as an l-value.
Definition CGExpr.cpp:1864
LValue EmitLValue(const Expr *E, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
Definition CGExpr.cpp:1631
void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst)
Store of global named registers are always calls to intrinsics.
Definition CGExpr.cpp:2839
void EmitOMPParallelMasterTaskLoopDirective(const OMPParallelMasterTaskLoopDirective &S)
void EmitOMPDistributeParallelForSimdDirective(const OMPDistributeParallelForSimdDirective &S)
void EmitOMPSectionDirective(const OMPSectionDirective &S)
void EnsureInsertPoint()
EnsureInsertPoint - Ensure that an insertion point is defined so that emitted IR has a place to go.
void EmitOMPForSimdDirective(const OMPForSimdDirective &S)
llvm::LLVMContext & getLLVMContext()
void emitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty, SourceLocation Loc, SourceLocation AssumptionLoc, llvm::Value *Alignment, llvm::Value *OffsetValue=nullptr)
static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeSimdDirective &S)
Emit device code for the target teams distribute simd directive.
llvm::function_ref< void(CodeGenFunction &, const OMPLoopDirective &, JumpDest)> CodeGenLoopTy
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
bool isTrivialInitializer(const Expr *Init)
Determine whether the given initializer is trivial in the sense that it requires no code to be genera...
Definition CGDecl.cpp:1807
void EmitOMPParallelMasterDirective(const OMPParallelMasterDirective &S)
void EmitOMPTaskBasedDirective(const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion, const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen, OMPTaskDataTy &Data)
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
void EmitOMPForDirective(const OMPForDirective &S)
void EmitOMPLinearClauseFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
Emit final code for linear clauses.
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
Definition CGStmt.cpp:655
void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue, bool capturedByInit)
EmitExprAsInit - Emits the code necessary to initialize a location in memory with the given initializ...
Definition CGDecl.cpp:2092
void EmitOMPSimdFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
This class organizes the cross-function state that is used while generating LLVM code.
void SetInternalFunctionAttributes(GlobalDecl GD, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
llvm::Module & getModule() const
DiagnosticsEngine & getDiags() const
const LangOptions & getLangOpts() const
const llvm::DataLayout & getDataLayout() const
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
ASTContext & getContext() const
const CodeGenOptions & getCodeGenOpts() const
StringRef getMangledName(GlobalDecl GD)
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
Definition CGCall.cpp:1701
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
Definition CGCall.cpp:739
const CGFunctionInfo & arrangeDeviceKernelCallerDeclaration(QualType resultType, const FunctionArgList &args)
A device kernel caller function is an offload device entry point function with a target device depend...
Definition CGCall.cpp:755
FunctionArgList - Type for representing both the decl and type of parameters to a function.
Definition CGCall.h:375
LValue - This represents an lvalue references.
Definition CGValue.h:182
llvm::Value * getPointer(CodeGenFunction &CGF) const
Address getAddress() const
Definition CGValue.h:361
QualType getType() const
Definition CGValue.h:291
void setAddress(Address address)
Definition CGValue.h:363
A stack of loop information corresponding to loop nesting levels.
Definition CGLoopInfo.h:207
void setVectorizeWidth(unsigned W)
Set the vectorize width for the next loop pushed.
Definition CGLoopInfo.h:272
void setParallel(bool Enable=true)
Set the next pushed loop as parallel.
Definition CGLoopInfo.h:242
void push(llvm::BasicBlock *Header, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc)
Begin a new structured loop.
void setVectorizeEnable(bool Enable=true)
Set the next pushed loop 'vectorize.enable'.
Definition CGLoopInfo.h:245
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
virtual void Enter(CodeGenFunction &CGF)
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
Definition CGValue.h:42
bool isScalar() const
Definition CGValue.h:64
static RValue get(llvm::Value *V)
Definition CGValue.h:98
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
Definition CGValue.h:108
bool isAggregate() const
Definition CGValue.h:66
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
Definition CGValue.h:71
bool isComplex() const
Definition CGValue.h:65
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
Definition CGValue.h:78
An abstract representation of an aligned address.
Definition Address.h:42
llvm::PointerType * getType() const
Return the type of the pointer value.
Definition Address.h:72
llvm::Value * getPointer() const
Definition Address.h:66
Class intended to support codegen of all kind of the reduction clauses.
LValue getSharedLValue(unsigned N) const
Returns LValue for the reduction item.
void emitAggregateType(CodeGenFunction &CGF, unsigned N)
Emits the code for the variable-modified type, if required.
const VarDecl * getBaseDecl(unsigned N) const
Returns the base declaration of the reduction item.
void emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N)
Emits lvalue for the shared and original reduction item.
void emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr, llvm::function_ref< bool(CodeGenFunction &)> DefaultInit)
Performs initialization of the private copy for the reduction item.
Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Adjusts PrivatedAddr for using instead of the original variable address in normal operations.
Class provides a way to call simple version of codegen for OpenMP region, or an advanced with possibl...
void setAction(PrePostActionTy &Action) const
Complex values, per C99 6.2.5p11.
Definition TypeBase.h:3275
CompoundStmt - This represents a group of statements like { stmt stmt }.
Definition Stmt.h:1720
ConstStmtVisitor - This class implements a simple visitor for Stmt subclasses.
DeclContext * getParent()
getParent - Returns the containing DeclContext.
Definition DeclBase.h:2109
A reference to a declared variable, function, enum, etc.
Definition Expr.h:1270
static DeclRefExpr * Create(const ASTContext &Context, NestedNameSpecifierLoc QualifierLoc, SourceLocation TemplateKWLoc, ValueDecl *D, bool RefersToEnclosingVariableOrCapture, SourceLocation NameLoc, QualType T, ExprValueKind VK, NamedDecl *FoundD=nullptr, const TemplateArgumentListInfo *TemplateArgs=nullptr, NonOdrUseReason NOUR=NOUR_None)
Definition Expr.cpp:484
ValueDecl * getDecl()
Definition Expr.h:1338
DeclStmt - Adaptor class for mixing declarations with statements and expressions.
Definition Stmt.h:1611
decl_range decls()
Definition Stmt.h:1659
Decl - This represents one declaration (or definition), e.g.
Definition DeclBase.h:86
T * getAttr() const
Definition DeclBase.h:573
SourceLocation getBodyRBrace() const
getBodyRBrace - Gets the right brace of the body, if a body exists.
virtual bool hasBody() const
Returns true if this Decl represents a declaration for a body of code, such as a function or method d...
Definition DeclBase.h:1093
SourceLocation getLocation() const
Definition DeclBase.h:439
bool hasAttr() const
Definition DeclBase.h:577
The name of a declaration.
SourceLocation getBeginLoc() const LLVM_READONLY
Definition Decl.h:830
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
unsigned getCustomDiagID(Level L, const char(&FormatString)[N])
Return an ID for a diagnostic with the specified format string and level.
Definition Diagnostic.h:905
This represents one expression.
Definition Expr.h:112
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer,...
Expr * IgnoreParenImpCasts() LLVM_READONLY
Skip past any parentheses and implicit casts which might surround this expression until reaching a fi...
Definition Expr.cpp:3085
Expr * IgnoreImplicitAsWritten() LLVM_READONLY
Skip past any implicit AST nodes which might surround this expression until reaching a fixed point.
Definition Expr.cpp:3077
Expr * IgnoreImpCasts() LLVM_READONLY
Skip past any implicit casts which might surround this expression until reaching a fixed point.
Definition Expr.cpp:3065
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
Definition Expr.cpp:273
QualType getType() const
Definition Expr.h:144
Represents difference between two FPOptions values.
Represents a member of a struct/union/class.
Definition Decl.h:3157
Represents a function declaration or definition.
Definition Decl.h:1999
static FunctionDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation NLoc, DeclarationName N, QualType T, TypeSourceInfo *TInfo, StorageClass SC, bool UsesFPIntrin=false, bool isInlineSpecified=false, bool hasWrittenPrototype=true, ConstexprSpecKind ConstexprKind=ConstexprSpecKind::Unspecified, const AssociatedConstraint &TrailingRequiresClause={})
Definition Decl.h:2188
GlobalDecl - represents a global declaration.
Definition GlobalDecl.h:57
One of these records is kept for each identifier that is lexed.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
static ImplicitCastExpr * Create(const ASTContext &Context, QualType T, CastKind Kind, Expr *Operand, const CXXCastPath *BasePath, ExprValueKind Cat, FPOptionsOverride FPO)
Definition Expr.cpp:2068
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
Definition Decl.cpp:5474
std::vector< llvm::Triple > OMPTargetTriples
Triples of the OpenMP targets that the host code codegen should take into account in order to generat...
Represents a point when we exit a loop.
IdentifierInfo * getIdentifier() const
Get the identifier that names this declaration, if there is one.
Definition Decl.h:294
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
Definition Decl.h:300
A C++ nested-name-specifier augmented with source location information.
This represents 'acq_rel' clause in the 'pragma omp atomic|flush' directives.
This represents 'acquire' clause in the 'pragma omp atomic|flush' directives.
This represents clause 'aligned' in the 'pragma omp ...' directives.
This represents 'bind' clause in the 'pragma omp ...' directives.
static OMPClauseWithPreInit * get(OMPClause *C)
This is a basic class for representing single OpenMP clause.
This represents clause 'copyin' in the 'pragma omp ...' directives.
This represents clause 'copyprivate' in the 'pragma omp ...' directives.
This represents implicit clause 'depend' for the 'pragma omp task' directive.
This represents implicit clause 'depobj' for the 'pragma omp depobj' directive.
This represents 'destroy' clause in the 'pragma omp depobj' directive or the 'pragma omp interop' dir...
This represents 'device' clause in the 'pragma omp ...' directive.
This represents 'dist_schedule' clause in the 'pragma omp ...' directive.
This represents the 'doacross' clause for the 'pragma omp ordered' directive.
This represents 'fail' clause in the 'pragma omp atomic' directive.
This represents 'filter' clause in the 'pragma omp ...' directive.
This represents 'final' clause in the 'pragma omp ...' directive.
This represents clause 'firstprivate' in the 'pragma omp ...' directives.
This represents implicit clause 'flush' for the 'pragma omp flush' directive.
Representation of the 'full' clause of the 'pragma omp unroll' directive.
This represents 'grainsize' clause in the 'pragma omp ...' directive.
This represents 'hint' clause in the 'pragma omp ...' directive.
This represents 'if' clause in the 'pragma omp ...' directive.
This represents clause 'in_reduction' in the 'pragma omp task' directives.
This represents clause 'inclusive' in the 'pragma omp scan' directive.
This represents the 'init' clause in 'pragma omp ...' directives.
This represents clause 'lastprivate' in the 'pragma omp ...' directives.
This represents clause 'linear' in the 'pragma omp ...' directives.
This represents the 'message' clause in the 'pragma omp error' and the 'pragma omp parallel' directiv...
Expr * getMessageString() const
Returns message string of the clause.
This represents 'nogroup' clause in the 'pragma omp ...' directive.
This represents 'nowait' clause in the 'pragma omp ...' directive.
This represents 'num_tasks' clause in the 'pragma omp ...' directive.
This represents 'num_teams' clause in the 'pragma omp ...' directive.
This represents 'num_threads' clause in the 'pragma omp ...' directive.
This represents 'order' clause in the 'pragma omp ...' directive.
This represents 'ordered' clause in the 'pragma omp ...' directive.
Representation of the 'partial' clause of the 'pragma omp unroll' directive.
This represents 'priority' clause in the 'pragma omp ...' directive.
This represents clause 'private' in the 'pragma omp ...' directives.
This represents 'proc_bind' clause in the 'pragma omp ...' directive.
This represents clause 'reduction' in the 'pragma omp ...' directives.
This represents 'relaxed' clause in the 'pragma omp atomic' directives.
This represents 'release' clause in the 'pragma omp atomic|flush' directives.
This represents 'simd' clause in the 'pragma omp ...' directive.
This represents 'safelen' clause in the 'pragma omp ...' directive.
This represents 'schedule' clause in the 'pragma omp ...' directive.
This represents 'seq_cst' clause in the 'pragma omp atomic|flush' directives.
This represents the 'severity' clause in the 'pragma omp error' and the 'pragma omp parallel' directi...
OpenMPSeverityClauseKind getSeverityKind() const
Returns kind of the clause.
This represents 'simdlen' clause in the 'pragma omp ...' directive.
This represents clause 'task_reduction' in the 'pragma omp taskgroup' directives.
This represents 'thread_limit' clause in the 'pragma omp ...' directive.
This represents 'untied' clause in the 'pragma omp ...' directive.
This represents 'update' clause in the 'pragma omp atomic' directive.
This represents the 'use' clause in 'pragma omp ...' directives.
This represents clause 'use_device_addr' in the 'pragma omp ...' directives.
This represents clause 'use_device_ptr' in the 'pragma omp ...' directives.
OpaqueValueExpr - An expression referring to an opaque object of a fixed type and value class.
Definition Expr.h:1178
static ParmVarDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation IdLoc, const IdentifierInfo *Id, QualType T, TypeSourceInfo *TInfo, StorageClass S, Expr *DefArg)
Definition Decl.cpp:2946
PointerType - C99 6.7.5.1 - Pointer Declarators.
Definition TypeBase.h:3328
Represents an unpacked "presumed" location which can be presented to the user.
const char * getFilename() const
Return the presumed filename of this location.
unsigned getLine() const
Return the presumed line number of this location.
If a crash happens while one of these objects are live, the message is printed out along with the spe...
A (possibly-)qualified type.
Definition TypeBase.h:937
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
Definition TypeBase.h:8371
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
Definition TypeBase.h:8472
Represents a struct/union/class.
Definition Decl.h:4309
field_range fields() const
Definition Decl.h:4512
field_iterator field_begin() const
Definition Decl.cpp:5154
Base for LValueReferenceType and RValueReferenceType.
Definition TypeBase.h:3573
Scope - A scope is a transient data structure that is used while parsing the program.
Definition Scope.h:41
Encodes a location in the source.
A trivial tuple used to represent a source range.
SourceLocation getEnd() const
SourceLocation getBegin() const
Stmt - This represents one statement.
Definition Stmt.h:85
child_range children()
Definition Stmt.cpp:295
SourceRange getSourceRange() const LLVM_READONLY
SourceLocation tokens are not useful in isolation - they are low level value objects created/interpre...
Definition Stmt.cpp:334
Stmt * IgnoreContainers(bool IgnoreCaptured=false)
Skip no-op (attributed, compound) container stmts and skip captured stmt at the top,...
Definition Stmt.cpp:205
SourceLocation getBeginLoc() const LLVM_READONLY
Definition Stmt.cpp:346
bool isArrayType() const
Definition TypeBase.h:8623
bool isPointerType() const
Definition TypeBase.h:8524
const T * castAs() const
Member-template castAs<specific type>.
Definition TypeBase.h:9167
bool isReferenceType() const
Definition TypeBase.h:8548
bool isLValueReferenceType() const
Definition TypeBase.h:8552
bool isAnyComplexType() const
Definition TypeBase.h:8659
bool hasSignedIntegerRepresentation() const
Determine whether this type has an signed integer representation of some sort, e.g....
Definition Type.cpp:2243
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
Definition TypeBase.h:2800
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type.
Definition TypeBase.h:9153
static UnaryOperator * Create(const ASTContext &C, Expr *input, Opcode opc, QualType type, ExprValueKind VK, ExprObjectKind OK, SourceLocation l, bool CanOverflow, FPOptionsOverride FPFeatures)
Definition Expr.cpp:5036
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
Definition Decl.h:711
QualType getType() const
Definition Decl.h:722
Represents a variable declaration or definition.
Definition Decl.h:925
TLSKind getTLSKind() const
Definition Decl.cpp:2168
VarDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
Definition Decl.cpp:2257
@ CInit
C-style initialization with assignment.
Definition Decl.h:930
bool hasGlobalStorage() const
Returns true for all variables that do not have local storage.
Definition Decl.h:1225
bool isStaticLocal() const
Returns true if a variable with function scope is a static local variable.
Definition Decl.h:1207
const Expr * getInit() const
Definition Decl.h:1367
bool hasLocalStorage() const
Returns true if a variable with function scope is a non-static local variable.
Definition Decl.h:1183
@ TLS_None
Not a TLS variable.
Definition Decl.h:945
Represents a C array with a specified size that is not an integer-constant-expression.
Definition TypeBase.h:3966
Expr * getSizeExpr() const
Definition TypeBase.h:3980
Definition SPIR.cpp:35
@ Type
The l-value was considered opaque, so the alignment was determined from a type.
Definition CGValue.h:154
@ Decl
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
Definition CGValue.h:145
bool Inc(InterpState &S, CodePtr OpPC, bool CanOverflow)
1) Pops a pointer from the stack 2) Load the value from the pointer 3) Writes the value increased by ...
Definition Interp.h:865
CharSourceRange getSourceRange(const SourceRange &Range)
Returns the token CharSourceRange corresponding to Range.
Definition FixIt.h:32
The JSON file list parser is used to communicate input to InstallAPI.
bool isOpenMPWorksharingDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a worksharing directive.
CanQual< Type > CanQualType
Represents a canonical, potentially-qualified type.
bool needsTaskBasedThreadLimit(OpenMPDirectiveKind DKind)
Checks if the specified target directive, combined or not, needs task based thread_limit.
@ Ctor_Complete
Complete object ctor.
Definition ABI.h:25
bool isa(CodeGen::Address addr)
Definition Address.h:330
if(T->getSizeExpr()) TRY_TO(TraverseStmt(const_cast< Expr * >(T -> getSizeExpr())))
@ OK_Ordinary
An ordinary object is located at an address in memory.
Definition Specifiers.h:151
bool isOpenMPDistributeDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a distribute directive.
@ Tile
'tile' clause, allowed on 'loop' and Combined constructs.
OpenMPScheduleClauseModifier
OpenMP modifiers for 'schedule' clause.
Definition OpenMPKinds.h:39
@ OMPC_SCHEDULE_MODIFIER_unknown
Definition OpenMPKinds.h:40
@ CR_OpenMP
bool isOpenMPParallelDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a parallel-kind directive.
@ SC_Static
Definition Specifiers.h:252
@ SC_None
Definition Specifiers.h:250
OpenMPDistScheduleClauseKind
OpenMP attributes for 'dist_schedule' clause.
@ OMPC_DIST_SCHEDULE_unknown
Expr * Cond
};
bool isOpenMPTaskingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of tasking directives - task, taskloop,...
bool isOpenMPTargetExecutionDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target code offload directive.
@ Result
The result type of a method or function.
Definition TypeBase.h:905
bool isOpenMPTeamsDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a teams-kind directive.
bool isOpenMPGenericLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive constitutes a 'loop' directive in the outermost nest.
OpenMPBindClauseKind
OpenMP bindings for the 'bind' clause.
@ OMPC_BIND_unknown
const FunctionProtoType * T
OpenMPDependClauseKind
OpenMP attributes for 'depend' clause.
Definition OpenMPKinds.h:55
bool IsXLHSInRHSPart
True if UE has the first form and false if the second.
bool IsPostfixUpdate
True if original value of 'x' must be stored in 'v', not an updated one.
@ Dtor_Complete
Complete object dtor.
Definition ABI.h:36
OpenMPSeverityClauseKind
OpenMP attributes for 'severity' clause.
bool isOpenMPLoopBoundSharingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of the composite or combined directives that need loop ...
llvm::omp::Directive OpenMPDirectiveKind
OpenMP directives.
Definition OpenMPKinds.h:25
bool isOpenMPSimdDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a simd directive.
@ VK_PRValue
A pr-value expression (in the C++11 taxonomy) produces a temporary value.
Definition Specifiers.h:135
@ VK_LValue
An l-value expression is a reference to an object with independent storage.
Definition Specifiers.h:139
void getOpenMPCaptureRegions(llvm::SmallVectorImpl< OpenMPDirectiveKind > &CaptureRegions, OpenMPDirectiveKind DKind)
Return the captured regions of an OpenMP directive.
OpenMPNumThreadsClauseModifier
@ OMPC_NUMTHREADS_unknown
bool IsFailOnly
True if 'v' is updated only when the condition is false (compare capture only).
U cast(CodeGen::Address addr)
Definition Address.h:327
@ OMPC_DEVICE_unknown
Definition OpenMPKinds.h:51
llvm::omp::Clause OpenMPClauseKind
OpenMP clauses.
Definition OpenMPKinds.h:28
@ ThreadPrivateVar
Parameter for Thread private variable.
Definition Decl.h:1742
@ Other
Other implicit parameter.
Definition Decl.h:1745
OpenMPScheduleClauseKind
OpenMP attributes for 'schedule' clause.
Definition OpenMPKinds.h:31
@ OMPC_SCHEDULE_unknown
Definition OpenMPKinds.h:35
bool isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a taskloop directive.
#define true
Definition stdbool.h:25
Struct with the values to be passed to the static runtime function.
QualType getType() const
Definition CGCall.h:248
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
static Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc)
Returns address of the threadprivate variable for the current thread.
llvm::OpenMPIRBuilder::InsertPointTy InsertPointTy
static void EmitOMPOutlinedRegionBody(CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Twine RegionName)
Emit the body of an OMP region that will be outlined in OpenMPIRBuilder::finalize().
static Address getAddressOfLocalVariable(CodeGenFunction &CGF, const VarDecl *VD)
Gets the OpenMP-specific address of the local variable /p VD.
static void EmitCaptureStmt(CodeGenFunction &CGF, InsertPointTy CodeGenIP, llvm::BasicBlock &FiniBB, llvm::Function *Fn, ArrayRef< llvm::Value * > Args)
static std::string getNameWithSeparators(ArrayRef< StringRef > Parts, StringRef FirstSeparator=".", StringRef Separator=".")
Get the platform-specific name separator.
static void FinalizeOMPRegion(CodeGenFunction &CGF, InsertPointTy IP)
Emit the Finalization for an OMP region.
static void EmitOMPInlinedRegionBody(CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Twine RegionName)
Emit the body of an OMP region.
SmallVector< const Expr *, 4 > DepExprs
EvalResult is a struct with detailed info about an evaluated expression.
Definition Expr.h:645
Extra information about a function prototype.
Definition TypeBase.h:5351
Scheduling data for loop-based OpenMP directives.
OpenMPScheduleClauseModifier M2
OpenMPScheduleClauseModifier M1
OpenMPScheduleClauseKind Schedule