Thanks to visit codestin.com
Credit goes to clang.llvm.org

clang 22.0.0git
CGCall.cpp
Go to the documentation of this file.
1//===--- CGCall.cpp - Encapsulate calling convention details --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// These classes wrap the information about a call or function
10// definition used to handle ABI compliancy.
11//
12//===----------------------------------------------------------------------===//
13
14#include "CGCall.h"
15#include "ABIInfo.h"
16#include "ABIInfoImpl.h"
17#include "CGBlocks.h"
18#include "CGCXXABI.h"
19#include "CGCleanup.h"
20#include "CGDebugInfo.h"
21#include "CGRecordLayout.h"
22#include "CodeGenFunction.h"
23#include "CodeGenModule.h"
24#include "CodeGenPGO.h"
25#include "TargetInfo.h"
26#include "clang/AST/Attr.h"
27#include "clang/AST/Decl.h"
28#include "clang/AST/DeclCXX.h"
29#include "clang/AST/DeclObjC.h"
34#include "llvm/ADT/StringExtras.h"
35#include "llvm/Analysis/ValueTracking.h"
36#include "llvm/IR/Assumptions.h"
37#include "llvm/IR/AttributeMask.h"
38#include "llvm/IR/Attributes.h"
39#include "llvm/IR/CallingConv.h"
40#include "llvm/IR/DataLayout.h"
41#include "llvm/IR/InlineAsm.h"
42#include "llvm/IR/IntrinsicInst.h"
43#include "llvm/IR/Intrinsics.h"
44#include "llvm/IR/Type.h"
45#include "llvm/Transforms/Utils/Local.h"
46#include <optional>
47using namespace clang;
48using namespace CodeGen;
49
50/***/
51
53 switch (CC) {
54 default:
55 return llvm::CallingConv::C;
56 case CC_X86StdCall:
57 return llvm::CallingConv::X86_StdCall;
58 case CC_X86FastCall:
59 return llvm::CallingConv::X86_FastCall;
60 case CC_X86RegCall:
61 return llvm::CallingConv::X86_RegCall;
62 case CC_X86ThisCall:
63 return llvm::CallingConv::X86_ThisCall;
64 case CC_Win64:
65 return llvm::CallingConv::Win64;
66 case CC_X86_64SysV:
67 return llvm::CallingConv::X86_64_SysV;
68 case CC_AAPCS:
69 return llvm::CallingConv::ARM_AAPCS;
70 case CC_AAPCS_VFP:
71 return llvm::CallingConv::ARM_AAPCS_VFP;
72 case CC_IntelOclBicc:
73 return llvm::CallingConv::Intel_OCL_BI;
74 // TODO: Add support for __pascal to LLVM.
75 case CC_X86Pascal:
76 return llvm::CallingConv::C;
77 // TODO: Add support for __vectorcall to LLVM.
79 return llvm::CallingConv::X86_VectorCall;
81 return llvm::CallingConv::AArch64_VectorCall;
83 return llvm::CallingConv::AArch64_SVE_VectorCall;
84 case CC_SpirFunction:
85 return llvm::CallingConv::SPIR_FUNC;
86 case CC_DeviceKernel:
87 return CGM.getTargetCodeGenInfo().getDeviceKernelCallingConv();
88 case CC_PreserveMost:
89 return llvm::CallingConv::PreserveMost;
90 case CC_PreserveAll:
91 return llvm::CallingConv::PreserveAll;
92 case CC_Swift:
93 return llvm::CallingConv::Swift;
94 case CC_SwiftAsync:
95 return llvm::CallingConv::SwiftTail;
96 case CC_M68kRTD:
97 return llvm::CallingConv::M68k_RTD;
98 case CC_PreserveNone:
99 return llvm::CallingConv::PreserveNone;
100 // clang-format off
101 case CC_RISCVVectorCall: return llvm::CallingConv::RISCV_VectorCall;
102 // clang-format on
103#define CC_VLS_CASE(ABI_VLEN) \
104 case CC_RISCVVLSCall_##ABI_VLEN: \
105 return llvm::CallingConv::RISCV_VLSCall_##ABI_VLEN;
106 CC_VLS_CASE(32)
107 CC_VLS_CASE(64)
108 CC_VLS_CASE(128)
109 CC_VLS_CASE(256)
110 CC_VLS_CASE(512)
111 CC_VLS_CASE(1024)
112 CC_VLS_CASE(2048)
113 CC_VLS_CASE(4096)
114 CC_VLS_CASE(8192)
115 CC_VLS_CASE(16384)
116 CC_VLS_CASE(32768)
117 CC_VLS_CASE(65536)
118#undef CC_VLS_CASE
119 }
120}
121
122/// Derives the 'this' type for codegen purposes, i.e. ignoring method CVR
123/// qualification. Either or both of RD and MD may be null. A null RD indicates
124/// that there is no meaningful 'this' type, and a null MD can occur when
125/// calling a method pointer.
127 const CXXMethodDecl *MD) {
128 CanQualType RecTy;
129 if (RD)
130 RecTy = Context.getCanonicalTagType(RD);
131 else
132 RecTy = Context.VoidTy;
133
134 if (MD)
135 RecTy = CanQualType::CreateUnsafe(Context.getAddrSpaceQualType(
136 RecTy, MD->getMethodQualifiers().getAddressSpace()));
137 return Context.getPointerType(RecTy);
138}
139
140/// Returns the canonical formal type of the given C++ method.
146
147/// Returns the "extra-canonicalized" return type, which discards
148/// qualifiers on the return type. Codegen doesn't care about them,
149/// and it makes ABI code a little easier to be able to assume that
150/// all parameter and return types are top-level unqualified.
154
155/// Arrange the argument and result information for a value of the given
156/// unprototyped freestanding function type.
157const CGFunctionInfo &
159 // When translating an unprototyped function type, always use a
160 // variadic type.
161 return arrangeLLVMFunctionInfo(FTNP->getReturnType().getUnqualifiedType(),
162 FnInfoOpts::None, {}, FTNP->getExtInfo(), {},
163 RequiredArgs(0));
164}
165
168 const FunctionProtoType *proto, unsigned prefixArgs, unsigned totalArgs) {
169 assert(proto->hasExtParameterInfos());
170 assert(paramInfos.size() <= prefixArgs);
171 assert(proto->getNumParams() + prefixArgs <= totalArgs);
172
173 paramInfos.reserve(totalArgs);
174
175 // Add default infos for any prefix args that don't already have infos.
176 paramInfos.resize(prefixArgs);
177
178 // Add infos for the prototype.
179 for (const auto &ParamInfo : proto->getExtParameterInfos()) {
180 paramInfos.push_back(ParamInfo);
181 // pass_object_size params have no parameter info.
182 if (ParamInfo.hasPassObjectSize())
183 paramInfos.emplace_back();
184 }
185
186 assert(paramInfos.size() <= totalArgs &&
187 "Did we forget to insert pass_object_size args?");
188 // Add default infos for the variadic and/or suffix arguments.
189 paramInfos.resize(totalArgs);
190}
191
192/// Adds the formal parameters in FPT to the given prefix. If any parameter in
193/// FPT has pass_object_size attrs, then we'll add parameters for those, too.
195 const CodeGenTypes &CGT, SmallVectorImpl<CanQualType> &prefix,
198 // Fast path: don't touch param info if we don't need to.
199 if (!FPT->hasExtParameterInfos()) {
200 assert(paramInfos.empty() &&
201 "We have paramInfos, but the prototype doesn't?");
202 prefix.append(FPT->param_type_begin(), FPT->param_type_end());
203 return;
204 }
205
206 unsigned PrefixSize = prefix.size();
207 // In the vast majority of cases, we'll have precisely FPT->getNumParams()
208 // parameters; the only thing that can change this is the presence of
209 // pass_object_size. So, we preallocate for the common case.
210 prefix.reserve(prefix.size() + FPT->getNumParams());
211
212 auto ExtInfos = FPT->getExtParameterInfos();
213 assert(ExtInfos.size() == FPT->getNumParams());
214 for (unsigned I = 0, E = FPT->getNumParams(); I != E; ++I) {
215 prefix.push_back(FPT->getParamType(I));
216 if (ExtInfos[I].hasPassObjectSize())
217 prefix.push_back(CGT.getContext().getCanonicalSizeType());
218 }
219
220 addExtParameterInfosForCall(paramInfos, FPT.getTypePtr(), PrefixSize,
221 prefix.size());
222}
223
226
227/// Arrange the LLVM function layout for a value of the given function
228/// type, on top of any implicit parameters already stored.
229static const CGFunctionInfo &
230arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod,
233 ExtParameterInfoList paramInfos;
235 appendParameterTypes(CGT, prefix, paramInfos, FTP);
236 CanQualType resultType = FTP->getReturnType().getUnqualifiedType();
237
238 FnInfoOpts opts =
240 return CGT.arrangeLLVMFunctionInfo(resultType, opts, prefix,
241 FTP->getExtInfo(), paramInfos, Required);
242}
243
245
246/// Arrange the argument and result information for a value of the
247/// given freestanding function type.
248const CGFunctionInfo &
250 CanQualTypeList argTypes;
251 return ::arrangeLLVMFunctionInfo(*this, /*instanceMethod=*/false, argTypes,
252 FTP);
253}
254
256 bool IsTargetDefaultMSABI) {
257 // Set the appropriate calling convention for the Function.
258 if (D->hasAttr<StdCallAttr>())
259 return CC_X86StdCall;
260
261 if (D->hasAttr<FastCallAttr>())
262 return CC_X86FastCall;
263
264 if (D->hasAttr<RegCallAttr>())
265 return CC_X86RegCall;
266
267 if (D->hasAttr<ThisCallAttr>())
268 return CC_X86ThisCall;
269
270 if (D->hasAttr<VectorCallAttr>())
271 return CC_X86VectorCall;
272
273 if (D->hasAttr<PascalAttr>())
274 return CC_X86Pascal;
275
276 if (PcsAttr *PCS = D->getAttr<PcsAttr>())
277 return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP);
278
279 if (D->hasAttr<AArch64VectorPcsAttr>())
281
282 if (D->hasAttr<AArch64SVEPcsAttr>())
283 return CC_AArch64SVEPCS;
284
285 if (D->hasAttr<DeviceKernelAttr>())
286 return CC_DeviceKernel;
287
288 if (D->hasAttr<IntelOclBiccAttr>())
289 return CC_IntelOclBicc;
290
291 if (D->hasAttr<MSABIAttr>())
292 return IsTargetDefaultMSABI ? CC_C : CC_Win64;
293
294 if (D->hasAttr<SysVABIAttr>())
295 return IsTargetDefaultMSABI ? CC_X86_64SysV : CC_C;
296
297 if (D->hasAttr<PreserveMostAttr>())
298 return CC_PreserveMost;
299
300 if (D->hasAttr<PreserveAllAttr>())
301 return CC_PreserveAll;
302
303 if (D->hasAttr<M68kRTDAttr>())
304 return CC_M68kRTD;
305
306 if (D->hasAttr<PreserveNoneAttr>())
307 return CC_PreserveNone;
308
309 if (D->hasAttr<RISCVVectorCCAttr>())
310 return CC_RISCVVectorCall;
311
312 if (RISCVVLSCCAttr *PCS = D->getAttr<RISCVVLSCCAttr>()) {
313 switch (PCS->getVectorWidth()) {
314 default:
315 llvm_unreachable("Invalid RISC-V VLS ABI VLEN");
316#define CC_VLS_CASE(ABI_VLEN) \
317 case ABI_VLEN: \
318 return CC_RISCVVLSCall_##ABI_VLEN;
319 CC_VLS_CASE(32)
320 CC_VLS_CASE(64)
321 CC_VLS_CASE(128)
322 CC_VLS_CASE(256)
323 CC_VLS_CASE(512)
324 CC_VLS_CASE(1024)
325 CC_VLS_CASE(2048)
326 CC_VLS_CASE(4096)
327 CC_VLS_CASE(8192)
328 CC_VLS_CASE(16384)
329 CC_VLS_CASE(32768)
330 CC_VLS_CASE(65536)
331#undef CC_VLS_CASE
332 }
333 }
334
335 return CC_C;
336}
337
338/// Arrange the argument and result information for a call to an
339/// unknown C++ non-static member function of the given abstract type.
340/// (A null RD means we don't have any meaningful "this" argument type,
341/// so fall back to a generic pointer type).
342/// The member function must be an ordinary function, i.e. not a
343/// constructor or destructor.
344const CGFunctionInfo &
346 const FunctionProtoType *FTP,
347 const CXXMethodDecl *MD) {
348 CanQualTypeList argTypes;
349
350 // Add the 'this' pointer.
351 argTypes.push_back(DeriveThisType(RD, MD));
352
353 return ::arrangeLLVMFunctionInfo(
354 *this, /*instanceMethod=*/true, argTypes,
356}
357
358/// Set calling convention for CUDA/HIP kernel.
360 const FunctionDecl *FD) {
361 if (FD->hasAttr<CUDAGlobalAttr>()) {
362 const FunctionType *FT = FTy->getAs<FunctionType>();
364 FTy = FT->getCanonicalTypeUnqualified();
365 }
366}
367
368/// Arrange the argument and result information for a declaration or
369/// definition of the given C++ non-static member function. The
370/// member function must be an ordinary function, i.e. not a
371/// constructor or destructor.
372const CGFunctionInfo &
374 assert(!isa<CXXConstructorDecl>(MD) && "wrong method for constructors!");
375 assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!");
376
379 auto prototype = FT.getAs<FunctionProtoType>();
380
382 // The abstract case is perfectly fine.
383 const CXXRecordDecl *ThisType =
385 return arrangeCXXMethodType(ThisType, prototype.getTypePtr(), MD);
386 }
387
388 return arrangeFreeFunctionType(prototype);
389}
390
392 const InheritedConstructor &Inherited, CXXCtorType Type) {
393 // Parameters are unnecessary if we're constructing a base class subobject
394 // and the inherited constructor lives in a virtual base.
395 return Type == Ctor_Complete ||
396 !Inherited.getShadowDecl()->constructsVirtualBase() ||
397 !Target.getCXXABI().hasConstructorVariants();
398}
399
400const CGFunctionInfo &
402 auto *MD = cast<CXXMethodDecl>(GD.getDecl());
403
404 CanQualTypeList argTypes;
405 ExtParameterInfoList paramInfos;
406
408 argTypes.push_back(DeriveThisType(ThisType, MD));
409
410 bool PassParams = true;
411
412 if (auto *CD = dyn_cast<CXXConstructorDecl>(MD)) {
413 // A base class inheriting constructor doesn't get forwarded arguments
414 // needed to construct a virtual base (or base class thereof).
415 if (auto Inherited = CD->getInheritedConstructor())
416 PassParams = inheritingCtorHasParams(Inherited, GD.getCtorType());
417 }
418
420
421 // Add the formal parameters.
422 if (PassParams)
423 appendParameterTypes(*this, argTypes, paramInfos, FTP);
424
426 getCXXABI().buildStructorSignature(GD, argTypes);
427 if (!paramInfos.empty()) {
428 // Note: prefix implies after the first param.
429 if (AddedArgs.Prefix)
430 paramInfos.insert(paramInfos.begin() + 1, AddedArgs.Prefix,
432 if (AddedArgs.Suffix)
433 paramInfos.append(AddedArgs.Suffix,
435 }
436
437 RequiredArgs required =
438 (PassParams && MD->isVariadic() ? RequiredArgs(argTypes.size())
440
441 FunctionType::ExtInfo extInfo = FTP->getExtInfo();
442 CanQualType resultType = getCXXABI().HasThisReturn(GD) ? argTypes.front()
444 ? CGM.getContext().VoidPtrTy
445 : Context.VoidTy;
447 argTypes, extInfo, paramInfos, required);
448}
449
451 const CallArgList &args) {
452 CanQualTypeList argTypes;
453 for (auto &arg : args)
454 argTypes.push_back(ctx.getCanonicalParamType(arg.Ty));
455 return argTypes;
456}
457
459 const FunctionArgList &args) {
460 CanQualTypeList argTypes;
461 for (auto &arg : args)
462 argTypes.push_back(ctx.getCanonicalParamType(arg->getType()));
463 return argTypes;
464}
465
467getExtParameterInfosForCall(const FunctionProtoType *proto, unsigned prefixArgs,
468 unsigned totalArgs) {
470 if (proto->hasExtParameterInfos()) {
471 addExtParameterInfosForCall(result, proto, prefixArgs, totalArgs);
472 }
473 return result;
474}
475
476/// Arrange a call to a C++ method, passing the given arguments.
477///
478/// ExtraPrefixArgs is the number of ABI-specific args passed after the `this`
479/// parameter.
480/// ExtraSuffixArgs is the number of ABI-specific args passed at the end of
481/// args.
482/// PassProtoArgs indicates whether `args` has args for the parameters in the
483/// given CXXConstructorDecl.
485 const CallArgList &args, const CXXConstructorDecl *D, CXXCtorType CtorKind,
486 unsigned ExtraPrefixArgs, unsigned ExtraSuffixArgs, bool PassProtoArgs) {
487 CanQualTypeList ArgTypes;
488 for (const auto &Arg : args)
489 ArgTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
490
491 // +1 for implicit this, which should always be args[0].
492 unsigned TotalPrefixArgs = 1 + ExtraPrefixArgs;
493
495 RequiredArgs Required = PassProtoArgs
497 FPT, TotalPrefixArgs + ExtraSuffixArgs)
499
500 GlobalDecl GD(D, CtorKind);
501 CanQualType ResultType = getCXXABI().HasThisReturn(GD) ? ArgTypes.front()
503 ? CGM.getContext().VoidPtrTy
504 : Context.VoidTy;
505
506 FunctionType::ExtInfo Info = FPT->getExtInfo();
507 ExtParameterInfoList ParamInfos;
508 // If the prototype args are elided, we should only have ABI-specific args,
509 // which never have param info.
510 if (PassProtoArgs && FPT->hasExtParameterInfos()) {
511 // ABI-specific suffix arguments are treated the same as variadic arguments.
512 addExtParameterInfosForCall(ParamInfos, FPT.getTypePtr(), TotalPrefixArgs,
513 ArgTypes.size());
514 }
515
517 ArgTypes, Info, ParamInfos, Required);
518}
519
520/// Arrange the argument and result information for the declaration or
521/// definition of the given function.
522const CGFunctionInfo &
524 const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
525 if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD))
526 if (MD->isImplicitObjectMemberFunction())
528
530
531 assert(isa<FunctionType>(FTy));
532 setCUDAKernelCallingConvention(FTy, CGM, FD);
533
534 if (DeviceKernelAttr::isOpenCLSpelling(FD->getAttr<DeviceKernelAttr>()) &&
536 const FunctionType *FT = FTy->getAs<FunctionType>();
537 CGM.getTargetCodeGenInfo().setOCLKernelStubCallingConvention(FT);
538 FTy = FT->getCanonicalTypeUnqualified();
539 }
540
541 // When declaring a function without a prototype, always use a
542 // non-variadic type.
544 return arrangeLLVMFunctionInfo(noProto->getReturnType(), FnInfoOpts::None,
545 {}, noProto->getExtInfo(), {},
547 }
548
550}
551
552/// Arrange the argument and result information for the declaration or
553/// definition of an Objective-C method.
554const CGFunctionInfo &
556 // It happens that this is the same as a call with no optional
557 // arguments, except also using the formal 'self' type.
559}
560
561/// Arrange the argument and result information for the function type
562/// through which to perform a send to the given Objective-C method,
563/// using the given receiver type. The receiver type is not always
564/// the 'self' type of the method or even an Objective-C pointer type.
565/// This is *not* the right method for actually performing such a
566/// message send, due to the possibility of optional arguments.
567const CGFunctionInfo &
569 QualType receiverType) {
570 CanQualTypeList argTys;
571 ExtParameterInfoList extParamInfos(MD->isDirectMethod() ? 1 : 2);
572 argTys.push_back(Context.getCanonicalParamType(receiverType));
573 if (!MD->isDirectMethod())
574 argTys.push_back(Context.getCanonicalParamType(Context.getObjCSelType()));
575 for (const auto *I : MD->parameters()) {
576 argTys.push_back(Context.getCanonicalParamType(I->getType()));
578 I->hasAttr<NoEscapeAttr>());
579 extParamInfos.push_back(extParamInfo);
580 }
581
583 bool IsTargetDefaultMSABI =
584 getContext().getTargetInfo().getTriple().isOSWindows() ||
585 getContext().getTargetInfo().getTriple().isUEFI();
586 einfo = einfo.withCallingConv(
587 getCallingConventionForDecl(MD, IsTargetDefaultMSABI));
588
589 if (getContext().getLangOpts().ObjCAutoRefCount &&
590 MD->hasAttr<NSReturnsRetainedAttr>())
591 einfo = einfo.withProducesResult(true);
592
593 RequiredArgs required =
594 (MD->isVariadic() ? RequiredArgs(argTys.size()) : RequiredArgs::All);
595
597 FnInfoOpts::None, argTys, einfo, extParamInfos,
598 required);
599}
600
601const CGFunctionInfo &
603 const CallArgList &args) {
604 CanQualTypeList argTypes = getArgTypesForCall(Context, args);
606
608 argTypes, einfo, {}, RequiredArgs::All);
609}
610
612 // FIXME: Do we need to handle ObjCMethodDecl?
616
618}
619
620/// Arrange a thunk that takes 'this' as the first parameter followed by
621/// varargs. Return a void pointer, regardless of the actual return type.
622/// The body of the thunk will end in a musttail call to a function of the
623/// correct type, and the caller will bitcast the function to the correct
624/// prototype.
625const CGFunctionInfo &
627 assert(MD->isVirtual() && "only methods have thunks");
629 CanQualType ArgTys[] = {DeriveThisType(MD->getParent(), MD)};
630 return arrangeLLVMFunctionInfo(Context.VoidTy, FnInfoOpts::None, ArgTys,
631 FTP->getExtInfo(), {}, RequiredArgs(1));
632}
633
634const CGFunctionInfo &
636 CXXCtorType CT) {
637 assert(CT == Ctor_CopyingClosure || CT == Ctor_DefaultClosure);
638
641 const CXXRecordDecl *RD = CD->getParent();
642 ArgTys.push_back(DeriveThisType(RD, CD));
643 if (CT == Ctor_CopyingClosure)
644 ArgTys.push_back(*FTP->param_type_begin());
645 if (RD->getNumVBases() > 0)
646 ArgTys.push_back(Context.IntTy);
647 CallingConv CC = Context.getDefaultCallingConvention(
648 /*IsVariadic=*/false, /*IsCXXMethod=*/true);
650 ArgTys, FunctionType::ExtInfo(CC), {},
652}
653
654/// Arrange a call as unto a free function, except possibly with an
655/// additional number of formal parameters considered required.
656static const CGFunctionInfo &
658 const CallArgList &args, const FunctionType *fnType,
659 unsigned numExtraRequiredArgs, bool chainCall) {
660 assert(args.size() >= numExtraRequiredArgs);
661
662 ExtParameterInfoList paramInfos;
663
664 // In most cases, there are no optional arguments.
666
667 // If we have a variadic prototype, the required arguments are the
668 // extra prefix plus the arguments in the prototype.
669 if (const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(fnType)) {
670 if (proto->isVariadic())
671 required = RequiredArgs::forPrototypePlus(proto, numExtraRequiredArgs);
672
673 if (proto->hasExtParameterInfos())
674 addExtParameterInfosForCall(paramInfos, proto, numExtraRequiredArgs,
675 args.size());
676
677 // If we don't have a prototype at all, but we're supposed to
678 // explicitly use the variadic convention for unprototyped calls,
679 // treat all of the arguments as required but preserve the nominal
680 // possibility of variadics.
682 args, cast<FunctionNoProtoType>(fnType))) {
683 required = RequiredArgs(args.size());
684 }
685
686 CanQualTypeList argTypes;
687 for (const auto &arg : args)
688 argTypes.push_back(CGT.getContext().getCanonicalParamType(arg.Ty));
691 opts, argTypes, fnType->getExtInfo(),
692 paramInfos, required);
693}
694
695/// Figure out the rules for calling a function with the given formal
696/// type using the given arguments. The arguments are necessary
697/// because the function might be unprototyped, in which case it's
698/// target-dependent in crazy ways.
700 const CallArgList &args, const FunctionType *fnType, bool chainCall) {
701 return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType,
702 chainCall ? 1 : 0, chainCall);
703}
704
705/// A block function is essentially a free function with an
706/// extra implicit argument.
707const CGFunctionInfo &
709 const FunctionType *fnType) {
710 return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType, 1,
711 /*chainCall=*/false);
712}
713
714const CGFunctionInfo &
716 const FunctionArgList &params) {
717 ExtParameterInfoList paramInfos =
718 getExtParameterInfosForCall(proto, 1, params.size());
719 CanQualTypeList argTypes = getArgTypesForDeclaration(Context, params);
720
722 FnInfoOpts::None, argTypes,
723 proto->getExtInfo(), paramInfos,
725}
726
727const CGFunctionInfo &
729 const CallArgList &args) {
730 CanQualTypeList argTypes;
731 for (const auto &Arg : args)
732 argTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
734 argTypes, FunctionType::ExtInfo(),
735 /*paramInfos=*/{}, RequiredArgs::All);
736}
737
738const CGFunctionInfo &
747
754
756 QualType resultType, const FunctionArgList &args) {
757 CanQualTypeList argTypes = getArgTypesForDeclaration(Context, args);
758
760 argTypes,
762 /*paramInfos=*/{}, RequiredArgs::All);
763}
764
765/// Arrange a call to a C++ method, passing the given arguments.
766///
767/// numPrefixArgs is the number of ABI-specific prefix arguments we have. It
768/// does not count `this`.
770 const CallArgList &args, const FunctionProtoType *proto,
771 RequiredArgs required, unsigned numPrefixArgs) {
772 assert(numPrefixArgs + 1 <= args.size() &&
773 "Emitting a call with less args than the required prefix?");
774 // Add one to account for `this`. It's a bit awkward here, but we don't count
775 // `this` in similar places elsewhere.
776 ExtParameterInfoList paramInfos =
777 getExtParameterInfosForCall(proto, numPrefixArgs + 1, args.size());
778
779 CanQualTypeList argTypes = getArgTypesForCall(Context, args);
780
781 FunctionType::ExtInfo info = proto->getExtInfo();
783 FnInfoOpts::IsInstanceMethod, argTypes, info,
784 paramInfos, required);
785}
786
792
794 const CallArgList &args) {
795 assert(signature.arg_size() <= args.size());
796 if (signature.arg_size() == args.size())
797 return signature;
798
799 ExtParameterInfoList paramInfos;
800 auto sigParamInfos = signature.getExtParameterInfos();
801 if (!sigParamInfos.empty()) {
802 paramInfos.append(sigParamInfos.begin(), sigParamInfos.end());
803 paramInfos.resize(args.size());
804 }
805
806 CanQualTypeList argTypes = getArgTypesForCall(Context, args);
807
808 assert(signature.getRequiredArgs().allowsOptionalArgs());
810 if (signature.isInstanceMethod())
812 if (signature.isChainCall())
814 if (signature.isDelegateCall())
816 return arrangeLLVMFunctionInfo(signature.getReturnType(), opts, argTypes,
817 signature.getExtInfo(), paramInfos,
818 signature.getRequiredArgs());
819}
820
821namespace clang {
822namespace CodeGen {
824}
825} // namespace clang
826
827/// Arrange the argument and result information for an abstract value
828/// of a given function type. This is the method which all of the
829/// above functions ultimately defer to.
831 CanQualType resultType, FnInfoOpts opts, ArrayRef<CanQualType> argTypes,
834 RequiredArgs required) {
835 assert(llvm::all_of(argTypes,
836 [](CanQualType T) { return T.isCanonicalAsParam(); }));
837
838 // Lookup or create unique function info.
839 llvm::FoldingSetNodeID ID;
840 bool isInstanceMethod =
842 bool isChainCall =
844 bool isDelegateCall =
846 CGFunctionInfo::Profile(ID, isInstanceMethod, isChainCall, isDelegateCall,
847 info, paramInfos, required, resultType, argTypes);
848
849 void *insertPos = nullptr;
850 CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, insertPos);
851 if (FI)
852 return *FI;
853
854 unsigned CC = ClangCallConvToLLVMCallConv(info.getCC());
855
856 // Construct the function info. We co-allocate the ArgInfos.
857 FI = CGFunctionInfo::create(CC, isInstanceMethod, isChainCall, isDelegateCall,
858 info, paramInfos, resultType, argTypes, required);
859 FunctionInfos.InsertNode(FI, insertPos);
860
861 bool inserted = FunctionsBeingProcessed.insert(FI).second;
862 (void)inserted;
863 assert(inserted && "Recursively being processed?");
864
865 // Compute ABI information.
866 if (CC == llvm::CallingConv::SPIR_KERNEL) {
867 // Force target independent argument handling for the host visible
868 // kernel functions.
869 computeSPIRKernelABIInfo(CGM, *FI);
870 } else if (info.getCC() == CC_Swift || info.getCC() == CC_SwiftAsync) {
872 } else {
873 CGM.getABIInfo().computeInfo(*FI);
874 }
875
876 // Loop over all of the computed argument and return value info. If any of
877 // them are direct or extend without a specified coerce type, specify the
878 // default now.
879 ABIArgInfo &retInfo = FI->getReturnInfo();
880 if (retInfo.canHaveCoerceToType() && retInfo.getCoerceToType() == nullptr)
882
883 for (auto &I : FI->arguments())
884 if (I.info.canHaveCoerceToType() && I.info.getCoerceToType() == nullptr)
885 I.info.setCoerceToType(ConvertType(I.type));
886
887 bool erased = FunctionsBeingProcessed.erase(FI);
888 (void)erased;
889 assert(erased && "Not in set?");
890
891 return *FI;
892}
893
894CGFunctionInfo *CGFunctionInfo::create(unsigned llvmCC, bool instanceMethod,
895 bool chainCall, bool delegateCall,
896 const FunctionType::ExtInfo &info,
898 CanQualType resultType,
899 ArrayRef<CanQualType> argTypes,
900 RequiredArgs required) {
901 assert(paramInfos.empty() || paramInfos.size() == argTypes.size());
902 assert(!required.allowsOptionalArgs() ||
903 required.getNumRequiredArgs() <= argTypes.size());
904
905 void *buffer = operator new(totalSizeToAlloc<ArgInfo, ExtParameterInfo>(
906 argTypes.size() + 1, paramInfos.size()));
907
908 CGFunctionInfo *FI = new (buffer) CGFunctionInfo();
909 FI->CallingConvention = llvmCC;
910 FI->EffectiveCallingConvention = llvmCC;
911 FI->ASTCallingConvention = info.getCC();
912 FI->InstanceMethod = instanceMethod;
913 FI->ChainCall = chainCall;
914 FI->DelegateCall = delegateCall;
915 FI->CmseNSCall = info.getCmseNSCall();
916 FI->NoReturn = info.getNoReturn();
917 FI->ReturnsRetained = info.getProducesResult();
918 FI->NoCallerSavedRegs = info.getNoCallerSavedRegs();
919 FI->NoCfCheck = info.getNoCfCheck();
920 FI->Required = required;
921 FI->HasRegParm = info.getHasRegParm();
922 FI->RegParm = info.getRegParm();
923 FI->ArgStruct = nullptr;
924 FI->ArgStructAlign = 0;
925 FI->NumArgs = argTypes.size();
926 FI->HasExtParameterInfos = !paramInfos.empty();
927 FI->getArgsBuffer()[0].type = resultType;
928 FI->MaxVectorWidth = 0;
929 for (unsigned i = 0, e = argTypes.size(); i != e; ++i)
930 FI->getArgsBuffer()[i + 1].type = argTypes[i];
931 for (unsigned i = 0, e = paramInfos.size(); i != e; ++i)
932 FI->getExtParameterInfosBuffer()[i] = paramInfos[i];
933 return FI;
934}
935
936/***/
937
938namespace {
939// ABIArgInfo::Expand implementation.
940
941// Specifies the way QualType passed as ABIArgInfo::Expand is expanded.
942struct TypeExpansion {
943 enum TypeExpansionKind {
944 // Elements of constant arrays are expanded recursively.
945 TEK_ConstantArray,
946 // Record fields are expanded recursively (but if record is a union, only
947 // the field with the largest size is expanded).
948 TEK_Record,
949 // For complex types, real and imaginary parts are expanded recursively.
951 // All other types are not expandable.
952 TEK_None
953 };
954
955 const TypeExpansionKind Kind;
956
957 TypeExpansion(TypeExpansionKind K) : Kind(K) {}
958 virtual ~TypeExpansion() {}
959};
960
961struct ConstantArrayExpansion : TypeExpansion {
962 QualType EltTy;
963 uint64_t NumElts;
964
965 ConstantArrayExpansion(QualType EltTy, uint64_t NumElts)
966 : TypeExpansion(TEK_ConstantArray), EltTy(EltTy), NumElts(NumElts) {}
967 static bool classof(const TypeExpansion *TE) {
968 return TE->Kind == TEK_ConstantArray;
969 }
970};
971
972struct RecordExpansion : TypeExpansion {
973 SmallVector<const CXXBaseSpecifier *, 1> Bases;
974
975 SmallVector<const FieldDecl *, 1> Fields;
976
977 RecordExpansion(SmallVector<const CXXBaseSpecifier *, 1> &&Bases,
978 SmallVector<const FieldDecl *, 1> &&Fields)
979 : TypeExpansion(TEK_Record), Bases(std::move(Bases)),
980 Fields(std::move(Fields)) {}
981 static bool classof(const TypeExpansion *TE) {
982 return TE->Kind == TEK_Record;
983 }
984};
985
986struct ComplexExpansion : TypeExpansion {
987 QualType EltTy;
988
989 ComplexExpansion(QualType EltTy) : TypeExpansion(TEK_Complex), EltTy(EltTy) {}
990 static bool classof(const TypeExpansion *TE) {
991 return TE->Kind == TEK_Complex;
992 }
993};
994
995struct NoExpansion : TypeExpansion {
996 NoExpansion() : TypeExpansion(TEK_None) {}
997 static bool classof(const TypeExpansion *TE) { return TE->Kind == TEK_None; }
998};
999} // namespace
1000
1001static std::unique_ptr<TypeExpansion>
1003 if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) {
1004 return std::make_unique<ConstantArrayExpansion>(AT->getElementType(),
1005 AT->getZExtSize());
1006 }
1007 if (const auto *RD = Ty->getAsRecordDecl()) {
1010 assert(!RD->hasFlexibleArrayMember() &&
1011 "Cannot expand structure with flexible array.");
1012 if (RD->isUnion()) {
1013 // Unions can be here only in degenerative cases - all the fields are same
1014 // after flattening. Thus we have to use the "largest" field.
1015 const FieldDecl *LargestFD = nullptr;
1016 CharUnits UnionSize = CharUnits::Zero();
1017
1018 for (const auto *FD : RD->fields()) {
1019 if (FD->isZeroLengthBitField())
1020 continue;
1021 assert(!FD->isBitField() &&
1022 "Cannot expand structure with bit-field members.");
1023 CharUnits FieldSize = Context.getTypeSizeInChars(FD->getType());
1024 if (UnionSize < FieldSize) {
1025 UnionSize = FieldSize;
1026 LargestFD = FD;
1027 }
1028 }
1029 if (LargestFD)
1030 Fields.push_back(LargestFD);
1031 } else {
1032 if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
1033 assert(!CXXRD->isDynamicClass() &&
1034 "cannot expand vtable pointers in dynamic classes");
1035 llvm::append_range(Bases, llvm::make_pointer_range(CXXRD->bases()));
1036 }
1037
1038 for (const auto *FD : RD->fields()) {
1039 if (FD->isZeroLengthBitField())
1040 continue;
1041 assert(!FD->isBitField() &&
1042 "Cannot expand structure with bit-field members.");
1043 Fields.push_back(FD);
1044 }
1045 }
1046 return std::make_unique<RecordExpansion>(std::move(Bases),
1047 std::move(Fields));
1048 }
1049 if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
1050 return std::make_unique<ComplexExpansion>(CT->getElementType());
1051 }
1052 return std::make_unique<NoExpansion>();
1053}
1054
1055static int getExpansionSize(QualType Ty, const ASTContext &Context) {
1056 auto Exp = getTypeExpansion(Ty, Context);
1057 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
1058 return CAExp->NumElts * getExpansionSize(CAExp->EltTy, Context);
1059 }
1060 if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1061 int Res = 0;
1062 for (auto BS : RExp->Bases)
1063 Res += getExpansionSize(BS->getType(), Context);
1064 for (auto FD : RExp->Fields)
1065 Res += getExpansionSize(FD->getType(), Context);
1066 return Res;
1067 }
1068 if (isa<ComplexExpansion>(Exp.get()))
1069 return 2;
1070 assert(isa<NoExpansion>(Exp.get()));
1071 return 1;
1072}
1073
1076 auto Exp = getTypeExpansion(Ty, Context);
1077 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
1078 for (int i = 0, n = CAExp->NumElts; i < n; i++) {
1079 getExpandedTypes(CAExp->EltTy, TI);
1080 }
1081 } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1082 for (auto BS : RExp->Bases)
1083 getExpandedTypes(BS->getType(), TI);
1084 for (auto FD : RExp->Fields)
1085 getExpandedTypes(FD->getType(), TI);
1086 } else if (auto CExp = dyn_cast<ComplexExpansion>(Exp.get())) {
1087 llvm::Type *EltTy = ConvertType(CExp->EltTy);
1088 *TI++ = EltTy;
1089 *TI++ = EltTy;
1090 } else {
1091 assert(isa<NoExpansion>(Exp.get()));
1092 *TI++ = ConvertType(Ty);
1093 }
1094}
1095
1097 ConstantArrayExpansion *CAE,
1098 Address BaseAddr,
1099 llvm::function_ref<void(Address)> Fn) {
1100 for (int i = 0, n = CAE->NumElts; i < n; i++) {
1101 Address EltAddr = CGF.Builder.CreateConstGEP2_32(BaseAddr, 0, i);
1102 Fn(EltAddr);
1103 }
1104}
1105
1106void CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV,
1107 llvm::Function::arg_iterator &AI) {
1108 assert(LV.isSimple() &&
1109 "Unexpected non-simple lvalue during struct expansion.");
1110
1111 auto Exp = getTypeExpansion(Ty, getContext());
1112 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
1114 *this, CAExp, LV.getAddress(), [&](Address EltAddr) {
1115 LValue LV = MakeAddrLValue(EltAddr, CAExp->EltTy);
1116 ExpandTypeFromArgs(CAExp->EltTy, LV, AI);
1117 });
1118 } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1119 Address This = LV.getAddress();
1120 for (const CXXBaseSpecifier *BS : RExp->Bases) {
1121 // Perform a single step derived-to-base conversion.
1122 Address Base =
1123 GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1,
1124 /*NullCheckValue=*/false, SourceLocation());
1125 LValue SubLV = MakeAddrLValue(Base, BS->getType());
1126
1127 // Recurse onto bases.
1128 ExpandTypeFromArgs(BS->getType(), SubLV, AI);
1129 }
1130 for (auto FD : RExp->Fields) {
1131 // FIXME: What are the right qualifiers here?
1132 LValue SubLV = EmitLValueForFieldInitialization(LV, FD);
1133 ExpandTypeFromArgs(FD->getType(), SubLV, AI);
1134 }
1135 } else if (isa<ComplexExpansion>(Exp.get())) {
1136 auto realValue = &*AI++;
1137 auto imagValue = &*AI++;
1138 EmitStoreOfComplex(ComplexPairTy(realValue, imagValue), LV, /*init*/ true);
1139 } else {
1140 // Call EmitStoreOfScalar except when the lvalue is a bitfield to emit a
1141 // primitive store.
1142 assert(isa<NoExpansion>(Exp.get()));
1143 llvm::Value *Arg = &*AI++;
1144 if (LV.isBitField()) {
1145 EmitStoreThroughLValue(RValue::get(Arg), LV);
1146 } else {
1147 // TODO: currently there are some places are inconsistent in what LLVM
1148 // pointer type they use (see D118744). Once clang uses opaque pointers
1149 // all LLVM pointer types will be the same and we can remove this check.
1150 if (Arg->getType()->isPointerTy()) {
1151 Address Addr = LV.getAddress();
1152 Arg = Builder.CreateBitCast(Arg, Addr.getElementType());
1153 }
1154 EmitStoreOfScalar(Arg, LV);
1155 }
1156 }
1157}
1158
1159void CodeGenFunction::ExpandTypeToArgs(
1160 QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
1161 SmallVectorImpl<llvm::Value *> &IRCallArgs, unsigned &IRCallArgPos) {
1162 auto Exp = getTypeExpansion(Ty, getContext());
1163 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
1164 Address Addr = Arg.hasLValue() ? Arg.getKnownLValue().getAddress()
1166 forConstantArrayExpansion(*this, CAExp, Addr, [&](Address EltAddr) {
1167 CallArg EltArg =
1168 CallArg(convertTempToRValue(EltAddr, CAExp->EltTy, SourceLocation()),
1169 CAExp->EltTy);
1170 ExpandTypeToArgs(CAExp->EltTy, EltArg, IRFuncTy, IRCallArgs,
1171 IRCallArgPos);
1172 });
1173 } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1174 Address This = Arg.hasLValue() ? Arg.getKnownLValue().getAddress()
1176 for (const CXXBaseSpecifier *BS : RExp->Bases) {
1177 // Perform a single step derived-to-base conversion.
1178 Address Base =
1179 GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1,
1180 /*NullCheckValue=*/false, SourceLocation());
1181 CallArg BaseArg = CallArg(RValue::getAggregate(Base), BS->getType());
1182
1183 // Recurse onto bases.
1184 ExpandTypeToArgs(BS->getType(), BaseArg, IRFuncTy, IRCallArgs,
1185 IRCallArgPos);
1186 }
1187
1188 LValue LV = MakeAddrLValue(This, Ty);
1189 for (auto FD : RExp->Fields) {
1190 CallArg FldArg =
1191 CallArg(EmitRValueForField(LV, FD, SourceLocation()), FD->getType());
1192 ExpandTypeToArgs(FD->getType(), FldArg, IRFuncTy, IRCallArgs,
1193 IRCallArgPos);
1194 }
1195 } else if (isa<ComplexExpansion>(Exp.get())) {
1197 IRCallArgs[IRCallArgPos++] = CV.first;
1198 IRCallArgs[IRCallArgPos++] = CV.second;
1199 } else {
1200 assert(isa<NoExpansion>(Exp.get()));
1201 auto RV = Arg.getKnownRValue();
1202 assert(RV.isScalar() &&
1203 "Unexpected non-scalar rvalue during struct expansion.");
1204
1205 // Insert a bitcast as needed.
1206 llvm::Value *V = RV.getScalarVal();
1207 if (IRCallArgPos < IRFuncTy->getNumParams() &&
1208 V->getType() != IRFuncTy->getParamType(IRCallArgPos))
1209 V = Builder.CreateBitCast(V, IRFuncTy->getParamType(IRCallArgPos));
1210
1211 IRCallArgs[IRCallArgPos++] = V;
1212 }
1213}
1214
1215/// Create a temporary allocation for the purposes of coercion.
1217 llvm::Type *Ty,
1218 CharUnits MinAlign,
1219 const Twine &Name = "tmp") {
1220 // Don't use an alignment that's worse than what LLVM would prefer.
1221 auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlign(Ty);
1222 CharUnits Align = std::max(MinAlign, CharUnits::fromQuantity(PrefAlign));
1223
1224 return CGF.CreateTempAlloca(Ty, Align, Name + ".coerce");
1225}
1226
1227/// EnterStructPointerForCoercedAccess - Given a struct pointer that we are
1228/// accessing some number of bytes out of it, try to gep into the struct to get
1229/// at its inner goodness. Dive as deep as possible without entering an element
1230/// with an in-memory size smaller than DstSize.
1232 llvm::StructType *SrcSTy,
1233 uint64_t DstSize,
1234 CodeGenFunction &CGF) {
1235 // We can't dive into a zero-element struct.
1236 if (SrcSTy->getNumElements() == 0)
1237 return SrcPtr;
1238
1239 llvm::Type *FirstElt = SrcSTy->getElementType(0);
1240
1241 // If the first elt is at least as large as what we're looking for, or if the
1242 // first element is the same size as the whole struct, we can enter it. The
1243 // comparison must be made on the store size and not the alloca size. Using
1244 // the alloca size may overstate the size of the load.
1245 uint64_t FirstEltSize = CGF.CGM.getDataLayout().getTypeStoreSize(FirstElt);
1246 if (FirstEltSize < DstSize &&
1247 FirstEltSize < CGF.CGM.getDataLayout().getTypeStoreSize(SrcSTy))
1248 return SrcPtr;
1249
1250 // GEP into the first element.
1251 SrcPtr = CGF.Builder.CreateStructGEP(SrcPtr, 0, "coerce.dive");
1252
1253 // If the first element is a struct, recurse.
1254 llvm::Type *SrcTy = SrcPtr.getElementType();
1255 if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy))
1256 return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
1257
1258 return SrcPtr;
1259}
1260
1261/// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both
1262/// are either integers or pointers. This does a truncation of the value if it
1263/// is too large or a zero extension if it is too small.
1264///
1265/// This behaves as if the value were coerced through memory, so on big-endian
1266/// targets the high bits are preserved in a truncation, while little-endian
1267/// targets preserve the low bits.
1268static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val, llvm::Type *Ty,
1269 CodeGenFunction &CGF) {
1270 if (Val->getType() == Ty)
1271 return Val;
1272
1273 if (isa<llvm::PointerType>(Val->getType())) {
1274 // If this is Pointer->Pointer avoid conversion to and from int.
1275 if (isa<llvm::PointerType>(Ty))
1276 return CGF.Builder.CreateBitCast(Val, Ty, "coerce.val");
1277
1278 // Convert the pointer to an integer so we can play with its width.
1279 Val = CGF.Builder.CreatePtrToInt(Val, CGF.IntPtrTy, "coerce.val.pi");
1280 }
1281
1282 llvm::Type *DestIntTy = Ty;
1283 if (isa<llvm::PointerType>(DestIntTy))
1284 DestIntTy = CGF.IntPtrTy;
1285
1286 if (Val->getType() != DestIntTy) {
1287 const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
1288 if (DL.isBigEndian()) {
1289 // Preserve the high bits on big-endian targets.
1290 // That is what memory coercion does.
1291 uint64_t SrcSize = DL.getTypeSizeInBits(Val->getType());
1292 uint64_t DstSize = DL.getTypeSizeInBits(DestIntTy);
1293
1294 if (SrcSize > DstSize) {
1295 Val = CGF.Builder.CreateLShr(Val, SrcSize - DstSize, "coerce.highbits");
1296 Val = CGF.Builder.CreateTrunc(Val, DestIntTy, "coerce.val.ii");
1297 } else {
1298 Val = CGF.Builder.CreateZExt(Val, DestIntTy, "coerce.val.ii");
1299 Val = CGF.Builder.CreateShl(Val, DstSize - SrcSize, "coerce.highbits");
1300 }
1301 } else {
1302 // Little-endian targets preserve the low bits. No shifts required.
1303 Val = CGF.Builder.CreateIntCast(Val, DestIntTy, false, "coerce.val.ii");
1304 }
1305 }
1306
1307 if (isa<llvm::PointerType>(Ty))
1308 Val = CGF.Builder.CreateIntToPtr(Val, Ty, "coerce.val.ip");
1309 return Val;
1310}
1311
1312/// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
1313/// a pointer to an object of type \arg Ty, known to be aligned to
1314/// \arg SrcAlign bytes.
1315///
1316/// This safely handles the case when the src type is smaller than the
1317/// destination type; in this situation the values of bits which not
1318/// present in the src are undefined.
1319static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty,
1320 CodeGenFunction &CGF) {
1321 llvm::Type *SrcTy = Src.getElementType();
1322
1323 // If SrcTy and Ty are the same, just do a load.
1324 if (SrcTy == Ty)
1325 return CGF.Builder.CreateLoad(Src);
1326
1327 llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty);
1328
1329 if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) {
1330 Src = EnterStructPointerForCoercedAccess(Src, SrcSTy,
1331 DstSize.getFixedValue(), CGF);
1332 SrcTy = Src.getElementType();
1333 }
1334
1335 llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
1336
1337 // If the source and destination are integer or pointer types, just do an
1338 // extension or truncation to the desired type.
1341 llvm::Value *Load = CGF.Builder.CreateLoad(Src);
1342 return CoerceIntOrPtrToIntOrPtr(Load, Ty, CGF);
1343 }
1344
1345 // If load is legal, just bitcast the src pointer.
1346 if (!SrcSize.isScalable() && !DstSize.isScalable() &&
1347 SrcSize.getFixedValue() >= DstSize.getFixedValue()) {
1348 // Generally SrcSize is never greater than DstSize, since this means we are
1349 // losing bits. However, this can happen in cases where the structure has
1350 // additional padding, for example due to a user specified alignment.
1351 //
1352 // FIXME: Assert that we aren't truncating non-padding bits when have access
1353 // to that information.
1354 Src = Src.withElementType(Ty);
1355 return CGF.Builder.CreateLoad(Src);
1356 }
1357
1358 // If coercing a fixed vector to a scalable vector for ABI compatibility, and
1359 // the types match, use the llvm.vector.insert intrinsic to perform the
1360 // conversion.
1361 if (auto *ScalableDstTy = dyn_cast<llvm::ScalableVectorType>(Ty)) {
1362 if (auto *FixedSrcTy = dyn_cast<llvm::FixedVectorType>(SrcTy)) {
1363 // If we are casting a fixed i8 vector to a scalable i1 predicate
1364 // vector, use a vector insert and bitcast the result.
1365 if (ScalableDstTy->getElementType()->isIntegerTy(1) &&
1366 FixedSrcTy->getElementType()->isIntegerTy(8)) {
1367 ScalableDstTy = llvm::ScalableVectorType::get(
1368 FixedSrcTy->getElementType(),
1369 llvm::divideCeil(
1370 ScalableDstTy->getElementCount().getKnownMinValue(), 8));
1371 }
1372 if (ScalableDstTy->getElementType() == FixedSrcTy->getElementType()) {
1373 auto *Load = CGF.Builder.CreateLoad(Src);
1374 auto *PoisonVec = llvm::PoisonValue::get(ScalableDstTy);
1375 llvm::Value *Result = CGF.Builder.CreateInsertVector(
1376 ScalableDstTy, PoisonVec, Load, uint64_t(0), "cast.scalable");
1377 ScalableDstTy = cast<llvm::ScalableVectorType>(
1378 llvm::VectorType::getWithSizeAndScalar(ScalableDstTy, Ty));
1379 if (Result->getType() != ScalableDstTy)
1380 Result = CGF.Builder.CreateBitCast(Result, ScalableDstTy);
1381 if (Result->getType() != Ty)
1382 Result = CGF.Builder.CreateExtractVector(Ty, Result, uint64_t(0));
1383 return Result;
1384 }
1385 }
1386 }
1387
1388 // Otherwise do coercion through memory. This is stupid, but simple.
1389 RawAddress Tmp =
1390 CreateTempAllocaForCoercion(CGF, Ty, Src.getAlignment(), Src.getName());
1392 Tmp.getPointer(), Tmp.getAlignment().getAsAlign(),
1393 Src.emitRawPointer(CGF), Src.getAlignment().getAsAlign(),
1394 llvm::ConstantInt::get(CGF.IntPtrTy, SrcSize.getKnownMinValue()));
1395 return CGF.Builder.CreateLoad(Tmp);
1396}
1397
1399 llvm::TypeSize DstSize,
1400 bool DstIsVolatile) {
1401 if (!DstSize)
1402 return;
1403
1404 llvm::Type *SrcTy = Src->getType();
1405 llvm::TypeSize SrcSize = CGM.getDataLayout().getTypeAllocSize(SrcTy);
1406
1407 // GEP into structs to try to make types match.
1408 // FIXME: This isn't really that useful with opaque types, but it impacts a
1409 // lot of regression tests.
1410 if (SrcTy != Dst.getElementType()) {
1411 if (llvm::StructType *DstSTy =
1412 dyn_cast<llvm::StructType>(Dst.getElementType())) {
1413 assert(!SrcSize.isScalable());
1414 Dst = EnterStructPointerForCoercedAccess(Dst, DstSTy,
1415 SrcSize.getFixedValue(), *this);
1416 }
1417 }
1418
1419 if (SrcSize.isScalable() || SrcSize <= DstSize) {
1420 if (SrcTy->isIntegerTy() && Dst.getElementType()->isPointerTy() &&
1421 SrcSize == CGM.getDataLayout().getTypeAllocSize(Dst.getElementType())) {
1422 // If the value is supposed to be a pointer, convert it before storing it.
1423 Src = CoerceIntOrPtrToIntOrPtr(Src, Dst.getElementType(), *this);
1424 auto *I = Builder.CreateStore(Src, Dst, DstIsVolatile);
1426 } else if (llvm::StructType *STy =
1427 dyn_cast<llvm::StructType>(Src->getType())) {
1428 // Prefer scalar stores to first-class aggregate stores.
1429 Dst = Dst.withElementType(SrcTy);
1430 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
1431 Address EltPtr = Builder.CreateStructGEP(Dst, i);
1432 llvm::Value *Elt = Builder.CreateExtractValue(Src, i);
1433 auto *I = Builder.CreateStore(Elt, EltPtr, DstIsVolatile);
1435 }
1436 } else {
1437 auto *I =
1438 Builder.CreateStore(Src, Dst.withElementType(SrcTy), DstIsVolatile);
1440 }
1441 } else if (SrcTy->isIntegerTy()) {
1442 // If the source is a simple integer, coerce it directly.
1443 llvm::Type *DstIntTy = Builder.getIntNTy(DstSize.getFixedValue() * 8);
1444 Src = CoerceIntOrPtrToIntOrPtr(Src, DstIntTy, *this);
1445 auto *I =
1446 Builder.CreateStore(Src, Dst.withElementType(DstIntTy), DstIsVolatile);
1448 } else {
1449 // Otherwise do coercion through memory. This is stupid, but
1450 // simple.
1451
1452 // Generally SrcSize is never greater than DstSize, since this means we are
1453 // losing bits. However, this can happen in cases where the structure has
1454 // additional padding, for example due to a user specified alignment.
1455 //
1456 // FIXME: Assert that we aren't truncating non-padding bits when have access
1457 // to that information.
1458 RawAddress Tmp =
1459 CreateTempAllocaForCoercion(*this, SrcTy, Dst.getAlignment());
1460 Builder.CreateStore(Src, Tmp);
1461 auto *I = Builder.CreateMemCpy(
1462 Dst.emitRawPointer(*this), Dst.getAlignment().getAsAlign(),
1463 Tmp.getPointer(), Tmp.getAlignment().getAsAlign(),
1464 Builder.CreateTypeSize(IntPtrTy, DstSize));
1466 }
1467}
1468
1470 const ABIArgInfo &info) {
1471 if (unsigned offset = info.getDirectOffset()) {
1472 addr = addr.withElementType(CGF.Int8Ty);
1474 addr, CharUnits::fromQuantity(offset));
1475 addr = addr.withElementType(info.getCoerceToType());
1476 }
1477 return addr;
1478}
1479
1480static std::pair<llvm::Value *, bool>
1481CoerceScalableToFixed(CodeGenFunction &CGF, llvm::FixedVectorType *ToTy,
1482 llvm::ScalableVectorType *FromTy, llvm::Value *V,
1483 StringRef Name = "") {
1484 // If we are casting a scalable i1 predicate vector to a fixed i8
1485 // vector, first bitcast the source.
1486 if (FromTy->getElementType()->isIntegerTy(1) &&
1487 ToTy->getElementType() == CGF.Builder.getInt8Ty()) {
1488 if (!FromTy->getElementCount().isKnownMultipleOf(8)) {
1489 FromTy = llvm::ScalableVectorType::get(
1490 FromTy->getElementType(),
1491 llvm::alignTo<8>(FromTy->getElementCount().getKnownMinValue()));
1492 llvm::Value *ZeroVec = llvm::Constant::getNullValue(FromTy);
1493 V = CGF.Builder.CreateInsertVector(FromTy, ZeroVec, V, uint64_t(0));
1494 }
1495 FromTy = llvm::ScalableVectorType::get(
1496 ToTy->getElementType(),
1497 FromTy->getElementCount().getKnownMinValue() / 8);
1498 V = CGF.Builder.CreateBitCast(V, FromTy);
1499 }
1500 if (FromTy->getElementType() == ToTy->getElementType()) {
1501 V->setName(Name + ".coerce");
1502 V = CGF.Builder.CreateExtractVector(ToTy, V, uint64_t(0), "cast.fixed");
1503 return {V, true};
1504 }
1505 return {V, false};
1506}
1507
1508namespace {
1509
1510/// Encapsulates information about the way function arguments from
1511/// CGFunctionInfo should be passed to actual LLVM IR function.
1512class ClangToLLVMArgMapping {
1513 static const unsigned InvalidIndex = ~0U;
1514 unsigned InallocaArgNo;
1515 unsigned SRetArgNo;
1516 unsigned TotalIRArgs;
1517
1518 /// Arguments of LLVM IR function corresponding to single Clang argument.
1519 struct IRArgs {
1520 unsigned PaddingArgIndex;
1521 // Argument is expanded to IR arguments at positions
1522 // [FirstArgIndex, FirstArgIndex + NumberOfArgs).
1523 unsigned FirstArgIndex;
1524 unsigned NumberOfArgs;
1525
1526 IRArgs()
1527 : PaddingArgIndex(InvalidIndex), FirstArgIndex(InvalidIndex),
1528 NumberOfArgs(0) {}
1529 };
1530
1531 SmallVector<IRArgs, 8> ArgInfo;
1532
1533public:
1534 ClangToLLVMArgMapping(const ASTContext &Context, const CGFunctionInfo &FI,
1535 bool OnlyRequiredArgs = false)
1536 : InallocaArgNo(InvalidIndex), SRetArgNo(InvalidIndex), TotalIRArgs(0),
1537 ArgInfo(OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size()) {
1538 construct(Context, FI, OnlyRequiredArgs);
1539 }
1540
1541 bool hasInallocaArg() const { return InallocaArgNo != InvalidIndex; }
1542 unsigned getInallocaArgNo() const {
1543 assert(hasInallocaArg());
1544 return InallocaArgNo;
1545 }
1546
1547 bool hasSRetArg() const { return SRetArgNo != InvalidIndex; }
1548 unsigned getSRetArgNo() const {
1549 assert(hasSRetArg());
1550 return SRetArgNo;
1551 }
1552
1553 unsigned totalIRArgs() const { return TotalIRArgs; }
1554
1555 bool hasPaddingArg(unsigned ArgNo) const {
1556 assert(ArgNo < ArgInfo.size());
1557 return ArgInfo[ArgNo].PaddingArgIndex != InvalidIndex;
1558 }
1559 unsigned getPaddingArgNo(unsigned ArgNo) const {
1560 assert(hasPaddingArg(ArgNo));
1561 return ArgInfo[ArgNo].PaddingArgIndex;
1562 }
1563
1564 /// Returns index of first IR argument corresponding to ArgNo, and their
1565 /// quantity.
1566 std::pair<unsigned, unsigned> getIRArgs(unsigned ArgNo) const {
1567 assert(ArgNo < ArgInfo.size());
1568 return std::make_pair(ArgInfo[ArgNo].FirstArgIndex,
1569 ArgInfo[ArgNo].NumberOfArgs);
1570 }
1571
1572private:
1573 void construct(const ASTContext &Context, const CGFunctionInfo &FI,
1574 bool OnlyRequiredArgs);
1575};
1576
1577void ClangToLLVMArgMapping::construct(const ASTContext &Context,
1578 const CGFunctionInfo &FI,
1579 bool OnlyRequiredArgs) {
1580 unsigned IRArgNo = 0;
1581 bool SwapThisWithSRet = false;
1582 const ABIArgInfo &RetAI = FI.getReturnInfo();
1583
1584 if (RetAI.getKind() == ABIArgInfo::Indirect) {
1585 SwapThisWithSRet = RetAI.isSRetAfterThis();
1586 SRetArgNo = SwapThisWithSRet ? 1 : IRArgNo++;
1587 }
1588
1589 unsigned ArgNo = 0;
1590 unsigned NumArgs = OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size();
1591 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(); ArgNo < NumArgs;
1592 ++I, ++ArgNo) {
1593 assert(I != FI.arg_end());
1594 QualType ArgType = I->type;
1595 const ABIArgInfo &AI = I->info;
1596 // Collect data about IR arguments corresponding to Clang argument ArgNo.
1597 auto &IRArgs = ArgInfo[ArgNo];
1598
1599 if (AI.getPaddingType())
1600 IRArgs.PaddingArgIndex = IRArgNo++;
1601
1602 switch (AI.getKind()) {
1604 case ABIArgInfo::Extend:
1605 case ABIArgInfo::Direct: {
1606 // FIXME: handle sseregparm someday...
1607 llvm::StructType *STy = dyn_cast<llvm::StructType>(AI.getCoerceToType());
1608 if (AI.isDirect() && AI.getCanBeFlattened() && STy) {
1609 IRArgs.NumberOfArgs = STy->getNumElements();
1610 } else {
1611 IRArgs.NumberOfArgs = 1;
1612 }
1613 break;
1614 }
1617 IRArgs.NumberOfArgs = 1;
1618 break;
1619 case ABIArgInfo::Ignore:
1621 // ignore and inalloca doesn't have matching LLVM parameters.
1622 IRArgs.NumberOfArgs = 0;
1623 break;
1625 IRArgs.NumberOfArgs = AI.getCoerceAndExpandTypeSequence().size();
1626 break;
1627 case ABIArgInfo::Expand:
1628 IRArgs.NumberOfArgs = getExpansionSize(ArgType, Context);
1629 break;
1630 }
1631
1632 if (IRArgs.NumberOfArgs > 0) {
1633 IRArgs.FirstArgIndex = IRArgNo;
1634 IRArgNo += IRArgs.NumberOfArgs;
1635 }
1636
1637 // Skip over the sret parameter when it comes second. We already handled it
1638 // above.
1639 if (IRArgNo == 1 && SwapThisWithSRet)
1640 IRArgNo++;
1641 }
1642 assert(ArgNo == ArgInfo.size());
1643
1644 if (FI.usesInAlloca())
1645 InallocaArgNo = IRArgNo++;
1646
1647 TotalIRArgs = IRArgNo;
1648}
1649} // namespace
1650
1651/***/
1652
1654 const auto &RI = FI.getReturnInfo();
1655 return RI.isIndirect() || (RI.isInAlloca() && RI.getInAllocaSRet());
1656}
1657
1659 const auto &RI = FI.getReturnInfo();
1660 return RI.getInReg();
1661}
1662
1664 return ReturnTypeUsesSRet(FI) &&
1665 getTargetCodeGenInfo().doesReturnSlotInterfereWithArgs();
1666}
1667
1669 if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) {
1670 switch (BT->getKind()) {
1671 default:
1672 return false;
1673 case BuiltinType::Float:
1674 return getTarget().useObjCFPRetForRealType(FloatModeKind::Float);
1675 case BuiltinType::Double:
1676 return getTarget().useObjCFPRetForRealType(FloatModeKind::Double);
1677 case BuiltinType::LongDouble:
1678 return getTarget().useObjCFPRetForRealType(FloatModeKind::LongDouble);
1679 }
1680 }
1681
1682 return false;
1683}
1684
1686 if (const ComplexType *CT = ResultType->getAs<ComplexType>()) {
1687 if (const BuiltinType *BT = CT->getElementType()->getAs<BuiltinType>()) {
1688 if (BT->getKind() == BuiltinType::LongDouble)
1689 return getTarget().useObjCFP2RetForComplexLongDouble();
1690 }
1691 }
1692
1693 return false;
1694}
1695
1698 return GetFunctionType(FI);
1699}
1700
1701llvm::FunctionType *CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI) {
1702
1703 bool Inserted = FunctionsBeingProcessed.insert(&FI).second;
1704 (void)Inserted;
1705 assert(Inserted && "Recursively being processed?");
1706
1707 llvm::Type *resultType = nullptr;
1708 const ABIArgInfo &retAI = FI.getReturnInfo();
1709 switch (retAI.getKind()) {
1710 case ABIArgInfo::Expand:
1712 llvm_unreachable("Invalid ABI kind for return argument");
1713
1715 case ABIArgInfo::Extend:
1716 case ABIArgInfo::Direct:
1717 resultType = retAI.getCoerceToType();
1718 break;
1719
1721 if (retAI.getInAllocaSRet()) {
1722 // sret things on win32 aren't void, they return the sret pointer.
1723 QualType ret = FI.getReturnType();
1724 unsigned addressSpace = CGM.getTypes().getTargetAddressSpace(ret);
1725 resultType = llvm::PointerType::get(getLLVMContext(), addressSpace);
1726 } else {
1727 resultType = llvm::Type::getVoidTy(getLLVMContext());
1728 }
1729 break;
1730
1732 case ABIArgInfo::Ignore:
1733 resultType = llvm::Type::getVoidTy(getLLVMContext());
1734 break;
1735
1737 resultType = retAI.getUnpaddedCoerceAndExpandType();
1738 break;
1739 }
1740
1741 ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI, true);
1742 SmallVector<llvm::Type *, 8> ArgTypes(IRFunctionArgs.totalIRArgs());
1743
1744 // Add type for sret argument.
1745 if (IRFunctionArgs.hasSRetArg()) {
1746 ArgTypes[IRFunctionArgs.getSRetArgNo()] = llvm::PointerType::get(
1748 }
1749
1750 // Add type for inalloca argument.
1751 if (IRFunctionArgs.hasInallocaArg())
1752 ArgTypes[IRFunctionArgs.getInallocaArgNo()] =
1753 llvm::PointerType::getUnqual(getLLVMContext());
1754
1755 // Add in all of the required arguments.
1756 unsigned ArgNo = 0;
1758 ie = it + FI.getNumRequiredArgs();
1759 for (; it != ie; ++it, ++ArgNo) {
1760 const ABIArgInfo &ArgInfo = it->info;
1761
1762 // Insert a padding type to ensure proper alignment.
1763 if (IRFunctionArgs.hasPaddingArg(ArgNo))
1764 ArgTypes[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
1765 ArgInfo.getPaddingType();
1766
1767 unsigned FirstIRArg, NumIRArgs;
1768 std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
1769
1770 switch (ArgInfo.getKind()) {
1771 case ABIArgInfo::Ignore:
1773 assert(NumIRArgs == 0);
1774 break;
1775
1777 assert(NumIRArgs == 1);
1778 // indirect arguments are always on the stack, which is alloca addr space.
1779 ArgTypes[FirstIRArg] = llvm::PointerType::get(
1780 getLLVMContext(), CGM.getDataLayout().getAllocaAddrSpace());
1781 break;
1783 assert(NumIRArgs == 1);
1784 ArgTypes[FirstIRArg] = llvm::PointerType::get(
1786 break;
1788 case ABIArgInfo::Extend:
1789 case ABIArgInfo::Direct: {
1790 // Fast-isel and the optimizer generally like scalar values better than
1791 // FCAs, so we flatten them if this is safe to do for this argument.
1792 llvm::Type *argType = ArgInfo.getCoerceToType();
1793 llvm::StructType *st = dyn_cast<llvm::StructType>(argType);
1794 if (st && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
1795 assert(NumIRArgs == st->getNumElements());
1796 for (unsigned i = 0, e = st->getNumElements(); i != e; ++i)
1797 ArgTypes[FirstIRArg + i] = st->getElementType(i);
1798 } else {
1799 assert(NumIRArgs == 1);
1800 ArgTypes[FirstIRArg] = argType;
1801 }
1802 break;
1803 }
1804
1806 auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1807 for (auto *EltTy : ArgInfo.getCoerceAndExpandTypeSequence()) {
1808 *ArgTypesIter++ = EltTy;
1809 }
1810 assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1811 break;
1812 }
1813
1814 case ABIArgInfo::Expand:
1815 auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1816 getExpandedTypes(it->type, ArgTypesIter);
1817 assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1818 break;
1819 }
1820 }
1821
1822 bool Erased = FunctionsBeingProcessed.erase(&FI);
1823 (void)Erased;
1824 assert(Erased && "Not in set?");
1825
1826 return llvm::FunctionType::get(resultType, ArgTypes, FI.isVariadic());
1827}
1828
1830 const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
1831 const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();
1832
1833 if (!isFuncTypeConvertible(FPT))
1834 return llvm::StructType::get(getLLVMContext());
1835
1836 return GetFunctionType(GD);
1837}
1838
1840 llvm::AttrBuilder &FuncAttrs,
1841 const FunctionProtoType *FPT) {
1842 if (!FPT)
1843 return;
1844
1846 FPT->isNothrow())
1847 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1848
1849 unsigned SMEBits = FPT->getAArch64SMEAttributes();
1851 FuncAttrs.addAttribute("aarch64_pstate_sm_enabled");
1853 FuncAttrs.addAttribute("aarch64_pstate_sm_compatible");
1855 FuncAttrs.addAttribute("aarch64_za_state_agnostic");
1856
1857 // ZA
1859 FuncAttrs.addAttribute("aarch64_preserves_za");
1861 FuncAttrs.addAttribute("aarch64_in_za");
1863 FuncAttrs.addAttribute("aarch64_out_za");
1865 FuncAttrs.addAttribute("aarch64_inout_za");
1866
1867 // ZT0
1869 FuncAttrs.addAttribute("aarch64_preserves_zt0");
1871 FuncAttrs.addAttribute("aarch64_in_zt0");
1873 FuncAttrs.addAttribute("aarch64_out_zt0");
1875 FuncAttrs.addAttribute("aarch64_inout_zt0");
1876}
1877
1878static void AddAttributesFromOMPAssumes(llvm::AttrBuilder &FuncAttrs,
1879 const Decl *Callee) {
1880 if (!Callee)
1881 return;
1882
1884
1885 for (const OMPAssumeAttr *AA : Callee->specific_attrs<OMPAssumeAttr>())
1886 AA->getAssumption().split(Attrs, ",");
1887
1888 if (!Attrs.empty())
1889 FuncAttrs.addAttribute(llvm::AssumptionAttrKey,
1890 llvm::join(Attrs.begin(), Attrs.end(), ","));
1891}
1892
1894 QualType ReturnType) const {
1895 // We can't just discard the return value for a record type with a
1896 // complex destructor or a non-trivially copyable type.
1897 if (const RecordType *RT =
1898 ReturnType.getCanonicalType()->getAsCanonical<RecordType>()) {
1899 if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(RT->getOriginalDecl()))
1900 return ClassDecl->hasTrivialDestructor();
1901 }
1902 return ReturnType.isTriviallyCopyableType(Context);
1903}
1904
1906 const Decl *TargetDecl) {
1907 // As-is msan can not tolerate noundef mismatch between caller and
1908 // implementation. Mismatch is possible for e.g. indirect calls from C-caller
1909 // into C++. Such mismatches lead to confusing false reports. To avoid
1910 // expensive workaround on msan we enforce initialization event in uncommon
1911 // cases where it's allowed.
1912 if (Module.getLangOpts().Sanitize.has(SanitizerKind::Memory))
1913 return true;
1914 // C++ explicitly makes returning undefined values UB. C's rule only applies
1915 // to used values, so we never mark them noundef for now.
1916 if (!Module.getLangOpts().CPlusPlus)
1917 return false;
1918 if (TargetDecl) {
1919 if (const FunctionDecl *FDecl = dyn_cast<FunctionDecl>(TargetDecl)) {
1920 if (FDecl->isExternC())
1921 return false;
1922 } else if (const VarDecl *VDecl = dyn_cast<VarDecl>(TargetDecl)) {
1923 // Function pointer.
1924 if (VDecl->isExternC())
1925 return false;
1926 }
1927 }
1928
1929 // We don't want to be too aggressive with the return checking, unless
1930 // it's explicit in the code opts or we're using an appropriate sanitizer.
1931 // Try to respect what the programmer intended.
1932 return Module.getCodeGenOpts().StrictReturn ||
1933 !Module.MayDropFunctionReturn(Module.getContext(), RetTy) ||
1934 Module.getLangOpts().Sanitize.has(SanitizerKind::Return);
1935}
1936
1937/// Add denormal-fp-math and denormal-fp-math-f32 as appropriate for the
1938/// requested denormal behavior, accounting for the overriding behavior of the
1939/// -f32 case.
1940static void addDenormalModeAttrs(llvm::DenormalMode FPDenormalMode,
1941 llvm::DenormalMode FP32DenormalMode,
1942 llvm::AttrBuilder &FuncAttrs) {
1943 if (FPDenormalMode != llvm::DenormalMode::getDefault())
1944 FuncAttrs.addAttribute("denormal-fp-math", FPDenormalMode.str());
1945
1946 if (FP32DenormalMode != FPDenormalMode && FP32DenormalMode.isValid())
1947 FuncAttrs.addAttribute("denormal-fp-math-f32", FP32DenormalMode.str());
1948}
1949
1950/// Add default attributes to a function, which have merge semantics under
1951/// -mlink-builtin-bitcode and should not simply overwrite any existing
1952/// attributes in the linked library.
1953static void
1955 llvm::AttrBuilder &FuncAttrs) {
1956 addDenormalModeAttrs(CodeGenOpts.FPDenormalMode, CodeGenOpts.FP32DenormalMode,
1957 FuncAttrs);
1958}
1959
1961 StringRef Name, bool HasOptnone, const CodeGenOptions &CodeGenOpts,
1962 const LangOptions &LangOpts, bool AttrOnCallSite,
1963 llvm::AttrBuilder &FuncAttrs) {
1964 // OptimizeNoneAttr takes precedence over -Os or -Oz. No warning needed.
1965 if (!HasOptnone) {
1966 if (CodeGenOpts.OptimizeSize)
1967 FuncAttrs.addAttribute(llvm::Attribute::OptimizeForSize);
1968 if (CodeGenOpts.OptimizeSize == 2)
1969 FuncAttrs.addAttribute(llvm::Attribute::MinSize);
1970 }
1971
1972 if (CodeGenOpts.DisableRedZone)
1973 FuncAttrs.addAttribute(llvm::Attribute::NoRedZone);
1974 if (CodeGenOpts.IndirectTlsSegRefs)
1975 FuncAttrs.addAttribute("indirect-tls-seg-refs");
1976 if (CodeGenOpts.NoImplicitFloat)
1977 FuncAttrs.addAttribute(llvm::Attribute::NoImplicitFloat);
1978
1979 if (AttrOnCallSite) {
1980 // Attributes that should go on the call site only.
1981 // FIXME: Look for 'BuiltinAttr' on the function rather than re-checking
1982 // the -fno-builtin-foo list.
1983 if (!CodeGenOpts.SimplifyLibCalls || LangOpts.isNoBuiltinFunc(Name))
1984 FuncAttrs.addAttribute(llvm::Attribute::NoBuiltin);
1985 if (!CodeGenOpts.TrapFuncName.empty())
1986 FuncAttrs.addAttribute("trap-func-name", CodeGenOpts.TrapFuncName);
1987 } else {
1988 switch (CodeGenOpts.getFramePointer()) {
1990 // This is the default behavior.
1991 break;
1995 FuncAttrs.addAttribute("frame-pointer",
1997 CodeGenOpts.getFramePointer()));
1998 }
1999
2000 if (CodeGenOpts.LessPreciseFPMAD)
2001 FuncAttrs.addAttribute("less-precise-fpmad", "true");
2002
2003 if (CodeGenOpts.NullPointerIsValid)
2004 FuncAttrs.addAttribute(llvm::Attribute::NullPointerIsValid);
2005
2007 FuncAttrs.addAttribute("no-trapping-math", "true");
2008
2009 // TODO: Are these all needed?
2010 // unsafe/inf/nan/nsz are handled by instruction-level FastMathFlags.
2011 if (LangOpts.NoHonorInfs)
2012 FuncAttrs.addAttribute("no-infs-fp-math", "true");
2013 if (LangOpts.NoHonorNaNs)
2014 FuncAttrs.addAttribute("no-nans-fp-math", "true");
2015 if (LangOpts.AllowFPReassoc && LangOpts.AllowRecip &&
2016 LangOpts.NoSignedZero && LangOpts.ApproxFunc &&
2017 (LangOpts.getDefaultFPContractMode() ==
2019 LangOpts.getDefaultFPContractMode() ==
2021 FuncAttrs.addAttribute("unsafe-fp-math", "true");
2022 if (CodeGenOpts.SoftFloat)
2023 FuncAttrs.addAttribute("use-soft-float", "true");
2024 FuncAttrs.addAttribute("stack-protector-buffer-size",
2025 llvm::utostr(CodeGenOpts.SSPBufferSize));
2026 if (LangOpts.NoSignedZero)
2027 FuncAttrs.addAttribute("no-signed-zeros-fp-math", "true");
2028
2029 // TODO: Reciprocal estimate codegen options should apply to instructions?
2030 const std::vector<std::string> &Recips = CodeGenOpts.Reciprocals;
2031 if (!Recips.empty())
2032 FuncAttrs.addAttribute("reciprocal-estimates", llvm::join(Recips, ","));
2033
2034 if (!CodeGenOpts.PreferVectorWidth.empty() &&
2035 CodeGenOpts.PreferVectorWidth != "none")
2036 FuncAttrs.addAttribute("prefer-vector-width",
2037 CodeGenOpts.PreferVectorWidth);
2038
2039 if (CodeGenOpts.StackRealignment)
2040 FuncAttrs.addAttribute("stackrealign");
2041 if (CodeGenOpts.Backchain)
2042 FuncAttrs.addAttribute("backchain");
2043 if (CodeGenOpts.EnableSegmentedStacks)
2044 FuncAttrs.addAttribute("split-stack");
2045
2046 if (CodeGenOpts.SpeculativeLoadHardening)
2047 FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
2048
2049 // Add zero-call-used-regs attribute.
2050 switch (CodeGenOpts.getZeroCallUsedRegs()) {
2051 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Skip:
2052 FuncAttrs.removeAttribute("zero-call-used-regs");
2053 break;
2054 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPRArg:
2055 FuncAttrs.addAttribute("zero-call-used-regs", "used-gpr-arg");
2056 break;
2057 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPR:
2058 FuncAttrs.addAttribute("zero-call-used-regs", "used-gpr");
2059 break;
2060 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedArg:
2061 FuncAttrs.addAttribute("zero-call-used-regs", "used-arg");
2062 break;
2063 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Used:
2064 FuncAttrs.addAttribute("zero-call-used-regs", "used");
2065 break;
2066 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPRArg:
2067 FuncAttrs.addAttribute("zero-call-used-regs", "all-gpr-arg");
2068 break;
2069 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPR:
2070 FuncAttrs.addAttribute("zero-call-used-regs", "all-gpr");
2071 break;
2072 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllArg:
2073 FuncAttrs.addAttribute("zero-call-used-regs", "all-arg");
2074 break;
2075 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::All:
2076 FuncAttrs.addAttribute("zero-call-used-regs", "all");
2077 break;
2078 }
2079 }
2080
2081 if (LangOpts.assumeFunctionsAreConvergent()) {
2082 // Conservatively, mark all functions and calls in CUDA and OpenCL as
2083 // convergent (meaning, they may call an intrinsically convergent op, such
2084 // as __syncthreads() / barrier(), and so can't have certain optimizations
2085 // applied around them). LLVM will remove this attribute where it safely
2086 // can.
2087 FuncAttrs.addAttribute(llvm::Attribute::Convergent);
2088 }
2089
2090 // TODO: NoUnwind attribute should be added for other GPU modes HIP,
2091 // OpenMP offload. AFAIK, neither of them support exceptions in device code.
2092 if ((LangOpts.CUDA && LangOpts.CUDAIsDevice) || LangOpts.OpenCL ||
2093 LangOpts.SYCLIsDevice) {
2094 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2095 }
2096
2097 if (CodeGenOpts.SaveRegParams && !AttrOnCallSite)
2098 FuncAttrs.addAttribute("save-reg-params");
2099
2100 for (StringRef Attr : CodeGenOpts.DefaultFunctionAttrs) {
2101 StringRef Var, Value;
2102 std::tie(Var, Value) = Attr.split('=');
2103 FuncAttrs.addAttribute(Var, Value);
2104 }
2105
2108}
2109
2110/// Merges `target-features` from \TargetOpts and \F, and sets the result in
2111/// \FuncAttr
2112/// * features from \F are always kept
2113/// * a feature from \TargetOpts is kept if itself and its opposite are absent
2114/// from \F
2115static void
2117 const llvm::Function &F,
2118 const TargetOptions &TargetOpts) {
2119 auto FFeatures = F.getFnAttribute("target-features");
2120
2121 llvm::StringSet<> MergedNames;
2122 SmallVector<StringRef> MergedFeatures;
2123 MergedFeatures.reserve(TargetOpts.Features.size());
2124
2125 auto AddUnmergedFeatures = [&](auto &&FeatureRange) {
2126 for (StringRef Feature : FeatureRange) {
2127 if (Feature.empty())
2128 continue;
2129 assert(Feature[0] == '+' || Feature[0] == '-');
2130 StringRef Name = Feature.drop_front(1);
2131 bool Merged = !MergedNames.insert(Name).second;
2132 if (!Merged)
2133 MergedFeatures.push_back(Feature);
2134 }
2135 };
2136
2137 if (FFeatures.isValid())
2138 AddUnmergedFeatures(llvm::split(FFeatures.getValueAsString(), ','));
2139 AddUnmergedFeatures(TargetOpts.Features);
2140
2141 if (!MergedFeatures.empty()) {
2142 llvm::sort(MergedFeatures);
2143 FuncAttr.addAttribute("target-features", llvm::join(MergedFeatures, ","));
2144 }
2145}
2146
2148 llvm::Function &F, const CodeGenOptions &CodeGenOpts,
2149 const LangOptions &LangOpts, const TargetOptions &TargetOpts,
2150 bool WillInternalize) {
2151
2152 llvm::AttrBuilder FuncAttrs(F.getContext());
2153 // Here we only extract the options that are relevant compared to the version
2154 // from GetCPUAndFeaturesAttributes.
2155 if (!TargetOpts.CPU.empty())
2156 FuncAttrs.addAttribute("target-cpu", TargetOpts.CPU);
2157 if (!TargetOpts.TuneCPU.empty())
2158 FuncAttrs.addAttribute("tune-cpu", TargetOpts.TuneCPU);
2159
2160 ::getTrivialDefaultFunctionAttributes(F.getName(), F.hasOptNone(),
2161 CodeGenOpts, LangOpts,
2162 /*AttrOnCallSite=*/false, FuncAttrs);
2163
2164 if (!WillInternalize && F.isInterposable()) {
2165 // Do not promote "dynamic" denormal-fp-math to this translation unit's
2166 // setting for weak functions that won't be internalized. The user has no
2167 // real control for how builtin bitcode is linked, so we shouldn't assume
2168 // later copies will use a consistent mode.
2169 F.addFnAttrs(FuncAttrs);
2170 return;
2171 }
2172
2173 llvm::AttributeMask AttrsToRemove;
2174
2175 llvm::DenormalMode DenormModeToMerge = F.getDenormalModeRaw();
2176 llvm::DenormalMode DenormModeToMergeF32 = F.getDenormalModeF32Raw();
2177 llvm::DenormalMode Merged =
2178 CodeGenOpts.FPDenormalMode.mergeCalleeMode(DenormModeToMerge);
2179 llvm::DenormalMode MergedF32 = CodeGenOpts.FP32DenormalMode;
2180
2181 if (DenormModeToMergeF32.isValid()) {
2182 MergedF32 =
2183 CodeGenOpts.FP32DenormalMode.mergeCalleeMode(DenormModeToMergeF32);
2184 }
2185
2186 if (Merged == llvm::DenormalMode::getDefault()) {
2187 AttrsToRemove.addAttribute("denormal-fp-math");
2188 } else if (Merged != DenormModeToMerge) {
2189 // Overwrite existing attribute
2190 FuncAttrs.addAttribute("denormal-fp-math",
2191 CodeGenOpts.FPDenormalMode.str());
2192 }
2193
2194 if (MergedF32 == llvm::DenormalMode::getDefault()) {
2195 AttrsToRemove.addAttribute("denormal-fp-math-f32");
2196 } else if (MergedF32 != DenormModeToMergeF32) {
2197 // Overwrite existing attribute
2198 FuncAttrs.addAttribute("denormal-fp-math-f32",
2199 CodeGenOpts.FP32DenormalMode.str());
2200 }
2201
2202 F.removeFnAttrs(AttrsToRemove);
2203 addDenormalModeAttrs(Merged, MergedF32, FuncAttrs);
2204
2205 overrideFunctionFeaturesWithTargetFeatures(FuncAttrs, F, TargetOpts);
2206
2207 F.addFnAttrs(FuncAttrs);
2208}
2209
2210void CodeGenModule::getTrivialDefaultFunctionAttributes(
2211 StringRef Name, bool HasOptnone, bool AttrOnCallSite,
2212 llvm::AttrBuilder &FuncAttrs) {
2213 ::getTrivialDefaultFunctionAttributes(Name, HasOptnone, getCodeGenOpts(),
2214 getLangOpts(), AttrOnCallSite,
2215 FuncAttrs);
2216}
2217
2218void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
2219 bool HasOptnone,
2220 bool AttrOnCallSite,
2221 llvm::AttrBuilder &FuncAttrs) {
2222 getTrivialDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite,
2223 FuncAttrs);
2224
2225 if (!AttrOnCallSite)
2226 TargetCodeGenInfo::initPointerAuthFnAttributes(CodeGenOpts.PointerAuth,
2227 FuncAttrs);
2228
2229 // If we're just getting the default, get the default values for mergeable
2230 // attributes.
2231 if (!AttrOnCallSite)
2232 addMergableDefaultFunctionAttributes(CodeGenOpts, FuncAttrs);
2233}
2234
2236 llvm::AttrBuilder &attrs) {
2237 getDefaultFunctionAttributes(/*function name*/ "", /*optnone*/ false,
2238 /*for call*/ false, attrs);
2239 GetCPUAndFeaturesAttributes(GlobalDecl(), attrs);
2240}
2241
2242static void addNoBuiltinAttributes(llvm::AttrBuilder &FuncAttrs,
2243 const LangOptions &LangOpts,
2244 const NoBuiltinAttr *NBA = nullptr) {
2245 auto AddNoBuiltinAttr = [&FuncAttrs](StringRef BuiltinName) {
2246 SmallString<32> AttributeName;
2247 AttributeName += "no-builtin-";
2248 AttributeName += BuiltinName;
2249 FuncAttrs.addAttribute(AttributeName);
2250 };
2251
2252 // First, handle the language options passed through -fno-builtin.
2253 if (LangOpts.NoBuiltin) {
2254 // -fno-builtin disables them all.
2255 FuncAttrs.addAttribute("no-builtins");
2256 return;
2257 }
2258
2259 // Then, add attributes for builtins specified through -fno-builtin-<name>.
2260 llvm::for_each(LangOpts.NoBuiltinFuncs, AddNoBuiltinAttr);
2261
2262 // Now, let's check the __attribute__((no_builtin("...")) attribute added to
2263 // the source.
2264 if (!NBA)
2265 return;
2266
2267 // If there is a wildcard in the builtin names specified through the
2268 // attribute, disable them all.
2269 if (llvm::is_contained(NBA->builtinNames(), "*")) {
2270 FuncAttrs.addAttribute("no-builtins");
2271 return;
2272 }
2273
2274 // And last, add the rest of the builtin names.
2275 llvm::for_each(NBA->builtinNames(), AddNoBuiltinAttr);
2276}
2277
2279 const llvm::DataLayout &DL, const ABIArgInfo &AI,
2280 bool CheckCoerce = true) {
2281 llvm::Type *Ty = Types.ConvertTypeForMem(QTy);
2282 if (AI.getKind() == ABIArgInfo::Indirect ||
2284 return true;
2285 if (AI.getKind() == ABIArgInfo::Extend && !AI.isNoExt())
2286 return true;
2287 if (!DL.typeSizeEqualsStoreSize(Ty))
2288 // TODO: This will result in a modest amount of values not marked noundef
2289 // when they could be. We care about values that *invisibly* contain undef
2290 // bits from the perspective of LLVM IR.
2291 return false;
2292 if (CheckCoerce && AI.canHaveCoerceToType()) {
2293 llvm::Type *CoerceTy = AI.getCoerceToType();
2294 if (llvm::TypeSize::isKnownGT(DL.getTypeSizeInBits(CoerceTy),
2295 DL.getTypeSizeInBits(Ty)))
2296 // If we're coercing to a type with a greater size than the canonical one,
2297 // we're introducing new undef bits.
2298 // Coercing to a type of smaller or equal size is ok, as we know that
2299 // there's no internal padding (typeSizeEqualsStoreSize).
2300 return false;
2301 }
2302 if (QTy->isBitIntType())
2303 return true;
2304 if (QTy->isReferenceType())
2305 return true;
2306 if (QTy->isNullPtrType())
2307 return false;
2308 if (QTy->isMemberPointerType())
2309 // TODO: Some member pointers are `noundef`, but it depends on the ABI. For
2310 // now, never mark them.
2311 return false;
2312 if (QTy->isScalarType()) {
2313 if (const ComplexType *Complex = dyn_cast<ComplexType>(QTy))
2314 return DetermineNoUndef(Complex->getElementType(), Types, DL, AI, false);
2315 return true;
2316 }
2317 if (const VectorType *Vector = dyn_cast<VectorType>(QTy))
2318 return DetermineNoUndef(Vector->getElementType(), Types, DL, AI, false);
2319 if (const MatrixType *Matrix = dyn_cast<MatrixType>(QTy))
2320 return DetermineNoUndef(Matrix->getElementType(), Types, DL, AI, false);
2321 if (const ArrayType *Array = dyn_cast<ArrayType>(QTy))
2322 return DetermineNoUndef(Array->getElementType(), Types, DL, AI, false);
2323
2324 // TODO: Some structs may be `noundef`, in specific situations.
2325 return false;
2326}
2327
2328/// Check if the argument of a function has maybe_undef attribute.
2329static bool IsArgumentMaybeUndef(const Decl *TargetDecl,
2330 unsigned NumRequiredArgs, unsigned ArgNo) {
2331 const auto *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl);
2332 if (!FD)
2333 return false;
2334
2335 // Assume variadic arguments do not have maybe_undef attribute.
2336 if (ArgNo >= NumRequiredArgs)
2337 return false;
2338
2339 // Check if argument has maybe_undef attribute.
2340 if (ArgNo < FD->getNumParams()) {
2341 const ParmVarDecl *Param = FD->getParamDecl(ArgNo);
2342 if (Param && Param->hasAttr<MaybeUndefAttr>())
2343 return true;
2344 }
2345
2346 return false;
2347}
2348
2349/// Test if it's legal to apply nofpclass for the given parameter type and it's
2350/// lowered IR type.
2351static bool canApplyNoFPClass(const ABIArgInfo &AI, QualType ParamType,
2352 bool IsReturn) {
2353 // Should only apply to FP types in the source, not ABI promoted.
2354 if (!ParamType->hasFloatingRepresentation())
2355 return false;
2356
2357 // The promoted-to IR type also needs to support nofpclass.
2358 llvm::Type *IRTy = AI.getCoerceToType();
2359 if (llvm::AttributeFuncs::isNoFPClassCompatibleType(IRTy))
2360 return true;
2361
2362 if (llvm::StructType *ST = dyn_cast<llvm::StructType>(IRTy)) {
2363 return !IsReturn && AI.getCanBeFlattened() &&
2364 llvm::all_of(ST->elements(),
2365 llvm::AttributeFuncs::isNoFPClassCompatibleType);
2366 }
2367
2368 return false;
2369}
2370
2371/// Return the nofpclass mask that can be applied to floating-point parameters.
2372static llvm::FPClassTest getNoFPClassTestMask(const LangOptions &LangOpts) {
2373 llvm::FPClassTest Mask = llvm::fcNone;
2374 if (LangOpts.NoHonorInfs)
2375 Mask |= llvm::fcInf;
2376 if (LangOpts.NoHonorNaNs)
2377 Mask |= llvm::fcNan;
2378 return Mask;
2379}
2380
2382 CGCalleeInfo CalleeInfo,
2383 llvm::AttributeList &Attrs) {
2384 if (Attrs.getMemoryEffects().getModRef() == llvm::ModRefInfo::NoModRef) {
2385 Attrs = Attrs.removeFnAttribute(getLLVMContext(), llvm::Attribute::Memory);
2386 llvm::Attribute MemoryAttr = llvm::Attribute::getWithMemoryEffects(
2387 getLLVMContext(), llvm::MemoryEffects::writeOnly());
2388 Attrs = Attrs.addFnAttribute(getLLVMContext(), MemoryAttr);
2389 }
2390}
2391
2392/// Construct the IR attribute list of a function or call.
2393///
2394/// When adding an attribute, please consider where it should be handled:
2395///
2396/// - getDefaultFunctionAttributes is for attributes that are essentially
2397/// part of the global target configuration (but perhaps can be
2398/// overridden on a per-function basis). Adding attributes there
2399/// will cause them to also be set in frontends that build on Clang's
2400/// target-configuration logic, as well as for code defined in library
2401/// modules such as CUDA's libdevice.
2402///
2403/// - ConstructAttributeList builds on top of getDefaultFunctionAttributes
2404/// and adds declaration-specific, convention-specific, and
2405/// frontend-specific logic. The last is of particular importance:
2406/// attributes that restrict how the frontend generates code must be
2407/// added here rather than getDefaultFunctionAttributes.
2408///
2410 const CGFunctionInfo &FI,
2411 CGCalleeInfo CalleeInfo,
2412 llvm::AttributeList &AttrList,
2413 unsigned &CallingConv,
2414 bool AttrOnCallSite, bool IsThunk) {
2415 llvm::AttrBuilder FuncAttrs(getLLVMContext());
2416 llvm::AttrBuilder RetAttrs(getLLVMContext());
2417
2418 // Collect function IR attributes from the CC lowering.
2419 // We'll collect the paramete and result attributes later.
2421 if (FI.isNoReturn())
2422 FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2423 if (FI.isCmseNSCall())
2424 FuncAttrs.addAttribute("cmse_nonsecure_call");
2425
2426 // Collect function IR attributes from the callee prototype if we have one.
2428 CalleeInfo.getCalleeFunctionProtoType());
2429 const Decl *TargetDecl = CalleeInfo.getCalleeDecl().getDecl();
2430
2431 // Attach assumption attributes to the declaration. If this is a call
2432 // site, attach assumptions from the caller to the call as well.
2433 AddAttributesFromOMPAssumes(FuncAttrs, TargetDecl);
2434
2435 bool HasOptnone = false;
2436 // The NoBuiltinAttr attached to the target FunctionDecl.
2437 const NoBuiltinAttr *NBA = nullptr;
2438
2439 // Some ABIs may result in additional accesses to arguments that may
2440 // otherwise not be present.
2441 std::optional<llvm::Attribute::AttrKind> MemAttrForPtrArgs;
2442 bool AddedPotentialArgAccess = false;
2443 auto AddPotentialArgAccess = [&]() {
2444 AddedPotentialArgAccess = true;
2445 llvm::Attribute A = FuncAttrs.getAttribute(llvm::Attribute::Memory);
2446 if (A.isValid())
2447 FuncAttrs.addMemoryAttr(A.getMemoryEffects() |
2448 llvm::MemoryEffects::argMemOnly());
2449 };
2450
2451 // Collect function IR attributes based on declaration-specific
2452 // information.
2453 // FIXME: handle sseregparm someday...
2454 if (TargetDecl) {
2455 if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
2456 FuncAttrs.addAttribute(llvm::Attribute::ReturnsTwice);
2457 if (TargetDecl->hasAttr<NoThrowAttr>())
2458 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2459 if (TargetDecl->hasAttr<NoReturnAttr>())
2460 FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2461 if (TargetDecl->hasAttr<ColdAttr>())
2462 FuncAttrs.addAttribute(llvm::Attribute::Cold);
2463 if (TargetDecl->hasAttr<HotAttr>())
2464 FuncAttrs.addAttribute(llvm::Attribute::Hot);
2465 if (TargetDecl->hasAttr<NoDuplicateAttr>())
2466 FuncAttrs.addAttribute(llvm::Attribute::NoDuplicate);
2467 if (TargetDecl->hasAttr<ConvergentAttr>())
2468 FuncAttrs.addAttribute(llvm::Attribute::Convergent);
2469
2470 if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
2472 getContext(), FuncAttrs, Fn->getType()->getAs<FunctionProtoType>());
2473 if (AttrOnCallSite && Fn->isReplaceableGlobalAllocationFunction()) {
2474 // A sane operator new returns a non-aliasing pointer.
2475 auto Kind = Fn->getDeclName().getCXXOverloadedOperator();
2476 if (getCodeGenOpts().AssumeSaneOperatorNew &&
2477 (Kind == OO_New || Kind == OO_Array_New))
2478 RetAttrs.addAttribute(llvm::Attribute::NoAlias);
2479 }
2480 const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Fn);
2481 const bool IsVirtualCall = MD && MD->isVirtual();
2482 // Don't use [[noreturn]], _Noreturn or [[no_builtin]] for a call to a
2483 // virtual function. These attributes are not inherited by overloads.
2484 if (!(AttrOnCallSite && IsVirtualCall)) {
2485 if (Fn->isNoReturn())
2486 FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2487 NBA = Fn->getAttr<NoBuiltinAttr>();
2488 }
2489 }
2490
2491 if (isa<FunctionDecl>(TargetDecl) || isa<VarDecl>(TargetDecl)) {
2492 // Only place nomerge attribute on call sites, never functions. This
2493 // allows it to work on indirect virtual function calls.
2494 if (AttrOnCallSite && TargetDecl->hasAttr<NoMergeAttr>())
2495 FuncAttrs.addAttribute(llvm::Attribute::NoMerge);
2496 }
2497
2498 // 'const', 'pure' and 'noalias' attributed functions are also nounwind.
2499 if (TargetDecl->hasAttr<ConstAttr>()) {
2500 FuncAttrs.addMemoryAttr(llvm::MemoryEffects::none());
2501 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2502 // gcc specifies that 'const' functions have greater restrictions than
2503 // 'pure' functions, so they also cannot have infinite loops.
2504 FuncAttrs.addAttribute(llvm::Attribute::WillReturn);
2505 MemAttrForPtrArgs = llvm::Attribute::ReadNone;
2506 } else if (TargetDecl->hasAttr<PureAttr>()) {
2507 FuncAttrs.addMemoryAttr(llvm::MemoryEffects::readOnly());
2508 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2509 // gcc specifies that 'pure' functions cannot have infinite loops.
2510 FuncAttrs.addAttribute(llvm::Attribute::WillReturn);
2511 MemAttrForPtrArgs = llvm::Attribute::ReadOnly;
2512 } else if (TargetDecl->hasAttr<NoAliasAttr>()) {
2513 FuncAttrs.addMemoryAttr(llvm::MemoryEffects::inaccessibleOrArgMemOnly());
2514 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2515 }
2516 if (const auto *RA = TargetDecl->getAttr<RestrictAttr>();
2517 RA && RA->getDeallocator() == nullptr)
2518 RetAttrs.addAttribute(llvm::Attribute::NoAlias);
2519 if (TargetDecl->hasAttr<ReturnsNonNullAttr>() &&
2520 !CodeGenOpts.NullPointerIsValid)
2521 RetAttrs.addAttribute(llvm::Attribute::NonNull);
2522 if (TargetDecl->hasAttr<AnyX86NoCallerSavedRegistersAttr>())
2523 FuncAttrs.addAttribute("no_caller_saved_registers");
2524 if (TargetDecl->hasAttr<AnyX86NoCfCheckAttr>())
2525 FuncAttrs.addAttribute(llvm::Attribute::NoCfCheck);
2526 if (TargetDecl->hasAttr<LeafAttr>())
2527 FuncAttrs.addAttribute(llvm::Attribute::NoCallback);
2528 if (TargetDecl->hasAttr<BPFFastCallAttr>())
2529 FuncAttrs.addAttribute("bpf_fastcall");
2530
2531 HasOptnone = TargetDecl->hasAttr<OptimizeNoneAttr>();
2532 if (auto *AllocSize = TargetDecl->getAttr<AllocSizeAttr>()) {
2533 std::optional<unsigned> NumElemsParam;
2534 if (AllocSize->getNumElemsParam().isValid())
2535 NumElemsParam = AllocSize->getNumElemsParam().getLLVMIndex();
2536 FuncAttrs.addAllocSizeAttr(AllocSize->getElemSizeParam().getLLVMIndex(),
2537 NumElemsParam);
2538 }
2539
2540 if (DeviceKernelAttr::isOpenCLSpelling(
2541 TargetDecl->getAttr<DeviceKernelAttr>()) &&
2544 // Check CallingConv to avoid adding uniform-work-group-size attribute to
2545 // OpenCL Kernel Stub
2546 if (getLangOpts().OpenCLVersion <= 120) {
2547 // OpenCL v1.2 Work groups are always uniform
2548 FuncAttrs.addAttribute("uniform-work-group-size", "true");
2549 } else {
2550 // OpenCL v2.0 Work groups may be whether uniform or not.
2551 // '-cl-uniform-work-group-size' compile option gets a hint
2552 // to the compiler that the global work-size be a multiple of
2553 // the work-group size specified to clEnqueueNDRangeKernel
2554 // (i.e. work groups are uniform).
2555 FuncAttrs.addAttribute(
2556 "uniform-work-group-size",
2557 llvm::toStringRef(getLangOpts().OffloadUniformBlock));
2558 }
2559 }
2560
2561 if (TargetDecl->hasAttr<CUDAGlobalAttr>() &&
2562 getLangOpts().OffloadUniformBlock)
2563 FuncAttrs.addAttribute("uniform-work-group-size", "true");
2564
2565 if (TargetDecl->hasAttr<ArmLocallyStreamingAttr>())
2566 FuncAttrs.addAttribute("aarch64_pstate_sm_body");
2567 }
2568
2569 // Attach "no-builtins" attributes to:
2570 // * call sites: both `nobuiltin` and "no-builtins" or "no-builtin-<name>".
2571 // * definitions: "no-builtins" or "no-builtin-<name>" only.
2572 // The attributes can come from:
2573 // * LangOpts: -ffreestanding, -fno-builtin, -fno-builtin-<name>
2574 // * FunctionDecl attributes: __attribute__((no_builtin(...)))
2575 addNoBuiltinAttributes(FuncAttrs, getLangOpts(), NBA);
2576
2577 // Collect function IR attributes based on global settiings.
2578 getDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite, FuncAttrs);
2579
2580 // Override some default IR attributes based on declaration-specific
2581 // information.
2582 if (TargetDecl) {
2583 if (TargetDecl->hasAttr<NoSpeculativeLoadHardeningAttr>())
2584 FuncAttrs.removeAttribute(llvm::Attribute::SpeculativeLoadHardening);
2585 if (TargetDecl->hasAttr<SpeculativeLoadHardeningAttr>())
2586 FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
2587 if (TargetDecl->hasAttr<NoSplitStackAttr>())
2588 FuncAttrs.removeAttribute("split-stack");
2589 if (TargetDecl->hasAttr<ZeroCallUsedRegsAttr>()) {
2590 // A function "__attribute__((...))" overrides the command-line flag.
2591 auto Kind =
2592 TargetDecl->getAttr<ZeroCallUsedRegsAttr>()->getZeroCallUsedRegs();
2593 FuncAttrs.removeAttribute("zero-call-used-regs");
2594 FuncAttrs.addAttribute(
2595 "zero-call-used-regs",
2596 ZeroCallUsedRegsAttr::ConvertZeroCallUsedRegsKindToStr(Kind));
2597 }
2598
2599 // Add NonLazyBind attribute to function declarations when -fno-plt
2600 // is used.
2601 // FIXME: what if we just haven't processed the function definition
2602 // yet, or if it's an external definition like C99 inline?
2603 if (CodeGenOpts.NoPLT) {
2604 if (auto *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
2605 if (!Fn->isDefined() && !AttrOnCallSite) {
2606 FuncAttrs.addAttribute(llvm::Attribute::NonLazyBind);
2607 }
2608 }
2609 }
2610 // Remove 'convergent' if requested.
2611 if (TargetDecl->hasAttr<NoConvergentAttr>())
2612 FuncAttrs.removeAttribute(llvm::Attribute::Convergent);
2613 }
2614
2615 // Add "sample-profile-suffix-elision-policy" attribute for internal linkage
2616 // functions with -funique-internal-linkage-names.
2617 if (TargetDecl && CodeGenOpts.UniqueInternalLinkageNames) {
2618 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) {
2619 if (!FD->isExternallyVisible())
2620 FuncAttrs.addAttribute("sample-profile-suffix-elision-policy",
2621 "selected");
2622 }
2623 }
2624
2625 // Collect non-call-site function IR attributes from declaration-specific
2626 // information.
2627 if (!AttrOnCallSite) {
2628 if (TargetDecl && TargetDecl->hasAttr<CmseNSEntryAttr>())
2629 FuncAttrs.addAttribute("cmse_nonsecure_entry");
2630
2631 // Whether tail calls are enabled.
2632 auto shouldDisableTailCalls = [&] {
2633 // Should this be honored in getDefaultFunctionAttributes?
2634 if (CodeGenOpts.DisableTailCalls)
2635 return true;
2636
2637 if (!TargetDecl)
2638 return false;
2639
2640 if (TargetDecl->hasAttr<DisableTailCallsAttr>() ||
2641 TargetDecl->hasAttr<AnyX86InterruptAttr>())
2642 return true;
2643
2644 if (CodeGenOpts.NoEscapingBlockTailCalls) {
2645 if (const auto *BD = dyn_cast<BlockDecl>(TargetDecl))
2646 if (!BD->doesNotEscape())
2647 return true;
2648 }
2649
2650 return false;
2651 };
2652 if (shouldDisableTailCalls())
2653 FuncAttrs.addAttribute("disable-tail-calls", "true");
2654
2655 // These functions require the returns_twice attribute for correct codegen,
2656 // but the attribute may not be added if -fno-builtin is specified. We
2657 // explicitly add that attribute here.
2658 static const llvm::StringSet<> ReturnsTwiceFn{
2659 "_setjmpex", "setjmp", "_setjmp", "vfork",
2660 "sigsetjmp", "__sigsetjmp", "savectx", "getcontext"};
2661 if (ReturnsTwiceFn.contains(Name))
2662 FuncAttrs.addAttribute(llvm::Attribute::ReturnsTwice);
2663
2664 // CPU/feature overrides. addDefaultFunctionDefinitionAttributes
2665 // handles these separately to set them based on the global defaults.
2666 GetCPUAndFeaturesAttributes(CalleeInfo.getCalleeDecl(), FuncAttrs);
2667
2668 // Windows hotpatching support
2669 if (!MSHotPatchFunctions.empty()) {
2670 bool IsHotPatched = llvm::binary_search(MSHotPatchFunctions, Name);
2671 if (IsHotPatched)
2672 FuncAttrs.addAttribute("marked_for_windows_hot_patching");
2673 }
2674 }
2675
2676 // Mark functions that are replaceable by the loader.
2677 if (CodeGenOpts.isLoaderReplaceableFunctionName(Name))
2678 FuncAttrs.addAttribute("loader-replaceable");
2679
2680 // Collect attributes from arguments and return values.
2681 ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI);
2682
2683 QualType RetTy = FI.getReturnType();
2684 const ABIArgInfo &RetAI = FI.getReturnInfo();
2685 const llvm::DataLayout &DL = getDataLayout();
2686
2687 // Determine if the return type could be partially undef
2688 if (CodeGenOpts.EnableNoundefAttrs &&
2689 HasStrictReturn(*this, RetTy, TargetDecl)) {
2690 if (!RetTy->isVoidType() && RetAI.getKind() != ABIArgInfo::Indirect &&
2691 DetermineNoUndef(RetTy, getTypes(), DL, RetAI))
2692 RetAttrs.addAttribute(llvm::Attribute::NoUndef);
2693 }
2694
2695 switch (RetAI.getKind()) {
2696 case ABIArgInfo::Extend:
2697 if (RetAI.isSignExt())
2698 RetAttrs.addAttribute(llvm::Attribute::SExt);
2699 else if (RetAI.isZeroExt())
2700 RetAttrs.addAttribute(llvm::Attribute::ZExt);
2701 else
2702 RetAttrs.addAttribute(llvm::Attribute::NoExt);
2703 [[fallthrough]];
2705 case ABIArgInfo::Direct:
2706 if (RetAI.getInReg())
2707 RetAttrs.addAttribute(llvm::Attribute::InReg);
2708
2709 if (canApplyNoFPClass(RetAI, RetTy, true))
2710 RetAttrs.addNoFPClassAttr(getNoFPClassTestMask(getLangOpts()));
2711
2712 break;
2713 case ABIArgInfo::Ignore:
2714 break;
2715
2717 case ABIArgInfo::Indirect: {
2718 // inalloca and sret disable readnone and readonly
2719 AddPotentialArgAccess();
2720 break;
2721 }
2722
2724 break;
2725
2726 case ABIArgInfo::Expand:
2728 llvm_unreachable("Invalid ABI kind for return argument");
2729 }
2730
2731 if (!IsThunk) {
2732 // FIXME: fix this properly, https://reviews.llvm.org/D100388
2733 if (const auto *RefTy = RetTy->getAs<ReferenceType>()) {
2734 QualType PTy = RefTy->getPointeeType();
2735 if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
2736 RetAttrs.addDereferenceableAttr(
2737 getMinimumObjectSize(PTy).getQuantity());
2738 if (getTypes().getTargetAddressSpace(PTy) == 0 &&
2739 !CodeGenOpts.NullPointerIsValid)
2740 RetAttrs.addAttribute(llvm::Attribute::NonNull);
2741 if (PTy->isObjectType()) {
2742 llvm::Align Alignment =
2743 getNaturalPointeeTypeAlignment(RetTy).getAsAlign();
2744 RetAttrs.addAlignmentAttr(Alignment);
2745 }
2746 }
2747 }
2748
2749 bool hasUsedSRet = false;
2750 SmallVector<llvm::AttributeSet, 4> ArgAttrs(IRFunctionArgs.totalIRArgs());
2751
2752 // Attach attributes to sret.
2753 if (IRFunctionArgs.hasSRetArg()) {
2754 llvm::AttrBuilder SRETAttrs(getLLVMContext());
2755 SRETAttrs.addStructRetAttr(getTypes().ConvertTypeForMem(RetTy));
2756 SRETAttrs.addAttribute(llvm::Attribute::Writable);
2757 SRETAttrs.addAttribute(llvm::Attribute::DeadOnUnwind);
2758 hasUsedSRet = true;
2759 if (RetAI.getInReg())
2760 SRETAttrs.addAttribute(llvm::Attribute::InReg);
2761 SRETAttrs.addAlignmentAttr(RetAI.getIndirectAlign().getQuantity());
2762 ArgAttrs[IRFunctionArgs.getSRetArgNo()] =
2763 llvm::AttributeSet::get(getLLVMContext(), SRETAttrs);
2764 }
2765
2766 // Attach attributes to inalloca argument.
2767 if (IRFunctionArgs.hasInallocaArg()) {
2768 llvm::AttrBuilder Attrs(getLLVMContext());
2769 Attrs.addInAllocaAttr(FI.getArgStruct());
2770 ArgAttrs[IRFunctionArgs.getInallocaArgNo()] =
2771 llvm::AttributeSet::get(getLLVMContext(), Attrs);
2772 }
2773
2774 // Apply `nonnull`, `dereferenceable(N)` and `align N` to the `this` argument,
2775 // unless this is a thunk function.
2776 // FIXME: fix this properly, https://reviews.llvm.org/D100388
2777 if (FI.isInstanceMethod() && !IRFunctionArgs.hasInallocaArg() &&
2778 !FI.arg_begin()->type->isVoidPointerType() && !IsThunk) {
2779 auto IRArgs = IRFunctionArgs.getIRArgs(0);
2780
2781 assert(IRArgs.second == 1 && "Expected only a single `this` pointer.");
2782
2783 llvm::AttrBuilder Attrs(getLLVMContext());
2784
2785 QualType ThisTy = FI.arg_begin()->type.getTypePtr()->getPointeeType();
2786
2787 if (!CodeGenOpts.NullPointerIsValid &&
2788 getTypes().getTargetAddressSpace(FI.arg_begin()->type) == 0) {
2789 Attrs.addAttribute(llvm::Attribute::NonNull);
2790 Attrs.addDereferenceableAttr(getMinimumObjectSize(ThisTy).getQuantity());
2791 } else {
2792 // FIXME dereferenceable should be correct here, regardless of
2793 // NullPointerIsValid. However, dereferenceable currently does not always
2794 // respect NullPointerIsValid and may imply nonnull and break the program.
2795 // See https://reviews.llvm.org/D66618 for discussions.
2796 Attrs.addDereferenceableOrNullAttr(
2799 .getQuantity());
2800 }
2801
2802 llvm::Align Alignment =
2803 getNaturalTypeAlignment(ThisTy, /*BaseInfo=*/nullptr,
2804 /*TBAAInfo=*/nullptr, /*forPointeeType=*/true)
2805 .getAsAlign();
2806 Attrs.addAlignmentAttr(Alignment);
2807
2808 ArgAttrs[IRArgs.first] = llvm::AttributeSet::get(getLLVMContext(), Attrs);
2809 }
2810
2811 unsigned ArgNo = 0;
2813 I != E; ++I, ++ArgNo) {
2814 QualType ParamType = I->type;
2815 const ABIArgInfo &AI = I->info;
2816 llvm::AttrBuilder Attrs(getLLVMContext());
2817
2818 // Add attribute for padding argument, if necessary.
2819 if (IRFunctionArgs.hasPaddingArg(ArgNo)) {
2820 if (AI.getPaddingInReg()) {
2821 ArgAttrs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
2822 llvm::AttributeSet::get(getLLVMContext(),
2823 llvm::AttrBuilder(getLLVMContext())
2824 .addAttribute(llvm::Attribute::InReg));
2825 }
2826 }
2827
2828 // Decide whether the argument we're handling could be partially undef
2829 if (CodeGenOpts.EnableNoundefAttrs &&
2830 DetermineNoUndef(ParamType, getTypes(), DL, AI)) {
2831 Attrs.addAttribute(llvm::Attribute::NoUndef);
2832 }
2833
2834 // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
2835 // have the corresponding parameter variable. It doesn't make
2836 // sense to do it here because parameters are so messed up.
2837 switch (AI.getKind()) {
2838 case ABIArgInfo::Extend:
2839 if (AI.isSignExt())
2840 Attrs.addAttribute(llvm::Attribute::SExt);
2841 else if (AI.isZeroExt())
2842 Attrs.addAttribute(llvm::Attribute::ZExt);
2843 else
2844 Attrs.addAttribute(llvm::Attribute::NoExt);
2845 [[fallthrough]];
2847 case ABIArgInfo::Direct:
2848 if (ArgNo == 0 && FI.isChainCall())
2849 Attrs.addAttribute(llvm::Attribute::Nest);
2850 else if (AI.getInReg())
2851 Attrs.addAttribute(llvm::Attribute::InReg);
2852 Attrs.addStackAlignmentAttr(llvm::MaybeAlign(AI.getDirectAlign()));
2853
2854 if (canApplyNoFPClass(AI, ParamType, false))
2855 Attrs.addNoFPClassAttr(getNoFPClassTestMask(getLangOpts()));
2856 break;
2857 case ABIArgInfo::Indirect: {
2858 if (AI.getInReg())
2859 Attrs.addAttribute(llvm::Attribute::InReg);
2860
2861 // HLSL out and inout parameters must not be marked with ByVal or
2862 // DeadOnReturn attributes because stores to these parameters by the
2863 // callee are visible to the caller.
2864 if (auto ParamABI = FI.getExtParameterInfo(ArgNo).getABI();
2865 ParamABI != ParameterABI::HLSLOut &&
2866 ParamABI != ParameterABI::HLSLInOut) {
2867
2868 // Depending on the ABI, this may be either a byval or a dead_on_return
2869 // argument.
2870 if (AI.getIndirectByVal()) {
2871 Attrs.addByValAttr(getTypes().ConvertTypeForMem(ParamType));
2872 } else {
2873 // Add dead_on_return when the object's lifetime ends in the callee.
2874 // This includes trivially-destructible objects, as well as objects
2875 // whose destruction / clean-up is carried out within the callee
2876 // (e.g., Obj-C ARC-managed structs, MSVC callee-destroyed objects).
2877 if (!ParamType.isDestructedType() || !ParamType->isRecordType() ||
2879 Attrs.addAttribute(llvm::Attribute::DeadOnReturn);
2880 }
2881 }
2882
2883 auto *Decl = ParamType->getAsRecordDecl();
2884 if (CodeGenOpts.PassByValueIsNoAlias && Decl &&
2885 Decl->getArgPassingRestrictions() ==
2887 // When calling the function, the pointer passed in will be the only
2888 // reference to the underlying object. Mark it accordingly.
2889 Attrs.addAttribute(llvm::Attribute::NoAlias);
2890
2891 // TODO: We could add the byref attribute if not byval, but it would
2892 // require updating many testcases.
2893
2894 CharUnits Align = AI.getIndirectAlign();
2895
2896 // In a byval argument, it is important that the required
2897 // alignment of the type is honored, as LLVM might be creating a
2898 // *new* stack object, and needs to know what alignment to give
2899 // it. (Sometimes it can deduce a sensible alignment on its own,
2900 // but not if clang decides it must emit a packed struct, or the
2901 // user specifies increased alignment requirements.)
2902 //
2903 // This is different from indirect *not* byval, where the object
2904 // exists already, and the align attribute is purely
2905 // informative.
2906 assert(!Align.isZero());
2907
2908 // For now, only add this when we have a byval argument.
2909 // TODO: be less lazy about updating test cases.
2910 if (AI.getIndirectByVal())
2911 Attrs.addAlignmentAttr(Align.getQuantity());
2912
2913 // byval disables readnone and readonly.
2914 AddPotentialArgAccess();
2915 break;
2916 }
2918 CharUnits Align = AI.getIndirectAlign();
2919 Attrs.addByRefAttr(getTypes().ConvertTypeForMem(ParamType));
2920 Attrs.addAlignmentAttr(Align.getQuantity());
2921 break;
2922 }
2923 case ABIArgInfo::Ignore:
2924 case ABIArgInfo::Expand:
2926 break;
2927
2929 // inalloca disables readnone and readonly.
2930 AddPotentialArgAccess();
2931 continue;
2932 }
2933
2934 if (const auto *RefTy = ParamType->getAs<ReferenceType>()) {
2935 QualType PTy = RefTy->getPointeeType();
2936 if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
2937 Attrs.addDereferenceableAttr(getMinimumObjectSize(PTy).getQuantity());
2938 if (getTypes().getTargetAddressSpace(PTy) == 0 &&
2939 !CodeGenOpts.NullPointerIsValid)
2940 Attrs.addAttribute(llvm::Attribute::NonNull);
2941 if (PTy->isObjectType()) {
2942 llvm::Align Alignment =
2943 getNaturalPointeeTypeAlignment(ParamType).getAsAlign();
2944 Attrs.addAlignmentAttr(Alignment);
2945 }
2946 }
2947
2948 // From OpenCL spec v3.0.10 section 6.3.5 Alignment of Types:
2949 // > For arguments to a __kernel function declared to be a pointer to a
2950 // > data type, the OpenCL compiler can assume that the pointee is always
2951 // > appropriately aligned as required by the data type.
2952 if (TargetDecl &&
2953 DeviceKernelAttr::isOpenCLSpelling(
2954 TargetDecl->getAttr<DeviceKernelAttr>()) &&
2955 ParamType->isPointerType()) {
2956 QualType PTy = ParamType->getPointeeType();
2957 if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
2958 llvm::Align Alignment =
2959 getNaturalPointeeTypeAlignment(ParamType).getAsAlign();
2960 Attrs.addAlignmentAttr(Alignment);
2961 }
2962 }
2963
2964 switch (FI.getExtParameterInfo(ArgNo).getABI()) {
2967 Attrs.addAttribute(llvm::Attribute::NoAlias);
2968 break;
2970 break;
2971
2973 // Add 'sret' if we haven't already used it for something, but
2974 // only if the result is void.
2975 if (!hasUsedSRet && RetTy->isVoidType()) {
2976 Attrs.addStructRetAttr(getTypes().ConvertTypeForMem(ParamType));
2977 hasUsedSRet = true;
2978 }
2979
2980 // Add 'noalias' in either case.
2981 Attrs.addAttribute(llvm::Attribute::NoAlias);
2982
2983 // Add 'dereferenceable' and 'alignment'.
2984 auto PTy = ParamType->getPointeeType();
2985 if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
2986 auto info = getContext().getTypeInfoInChars(PTy);
2987 Attrs.addDereferenceableAttr(info.Width.getQuantity());
2988 Attrs.addAlignmentAttr(info.Align.getAsAlign());
2989 }
2990 break;
2991 }
2992
2994 Attrs.addAttribute(llvm::Attribute::SwiftError);
2995 break;
2996
2998 Attrs.addAttribute(llvm::Attribute::SwiftSelf);
2999 break;
3000
3002 Attrs.addAttribute(llvm::Attribute::SwiftAsync);
3003 break;
3004 }
3005
3006 if (FI.getExtParameterInfo(ArgNo).isNoEscape())
3007 Attrs.addCapturesAttr(llvm::CaptureInfo::none());
3008
3009 if (Attrs.hasAttributes()) {
3010 unsigned FirstIRArg, NumIRArgs;
3011 std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
3012 for (unsigned i = 0; i < NumIRArgs; i++)
3013 ArgAttrs[FirstIRArg + i] = ArgAttrs[FirstIRArg + i].addAttributes(
3014 getLLVMContext(), llvm::AttributeSet::get(getLLVMContext(), Attrs));
3015 }
3016 }
3017 assert(ArgNo == FI.arg_size());
3018
3019 ArgNo = 0;
3020 if (AddedPotentialArgAccess && MemAttrForPtrArgs) {
3021 llvm::FunctionType *FunctionType = FunctionType =
3022 getTypes().GetFunctionType(FI);
3024 E = FI.arg_end();
3025 I != E; ++I, ++ArgNo) {
3026 if (I->info.isDirect() || I->info.isExpand() ||
3027 I->info.isCoerceAndExpand()) {
3028 unsigned FirstIRArg, NumIRArgs;
3029 std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
3030 for (unsigned i = FirstIRArg; i < FirstIRArg + NumIRArgs; ++i) {
3031 if (FunctionType->getParamType(i)->isPointerTy()) {
3032 ArgAttrs[i] =
3033 ArgAttrs[i].addAttribute(getLLVMContext(), *MemAttrForPtrArgs);
3034 }
3035 }
3036 }
3037 }
3038 }
3039
3040 AttrList = llvm::AttributeList::get(
3041 getLLVMContext(), llvm::AttributeSet::get(getLLVMContext(), FuncAttrs),
3042 llvm::AttributeSet::get(getLLVMContext(), RetAttrs), ArgAttrs);
3043}
3044
3045/// An argument came in as a promoted argument; demote it back to its
3046/// declared type.
3047static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF,
3048 const VarDecl *var,
3049 llvm::Value *value) {
3050 llvm::Type *varType = CGF.ConvertType(var->getType());
3051
3052 // This can happen with promotions that actually don't change the
3053 // underlying type, like the enum promotions.
3054 if (value->getType() == varType)
3055 return value;
3056
3057 assert((varType->isIntegerTy() || varType->isFloatingPointTy()) &&
3058 "unexpected promotion type");
3059
3060 if (isa<llvm::IntegerType>(varType))
3061 return CGF.Builder.CreateTrunc(value, varType, "arg.unpromote");
3062
3063 return CGF.Builder.CreateFPCast(value, varType, "arg.unpromote");
3064}
3065
3066/// Returns the attribute (either parameter attribute, or function
3067/// attribute), which declares argument ArgNo to be non-null.
3068static const NonNullAttr *getNonNullAttr(const Decl *FD, const ParmVarDecl *PVD,
3069 QualType ArgType, unsigned ArgNo) {
3070 // FIXME: __attribute__((nonnull)) can also be applied to:
3071 // - references to pointers, where the pointee is known to be
3072 // nonnull (apparently a Clang extension)
3073 // - transparent unions containing pointers
3074 // In the former case, LLVM IR cannot represent the constraint. In
3075 // the latter case, we have no guarantee that the transparent union
3076 // is in fact passed as a pointer.
3077 if (!ArgType->isAnyPointerType() && !ArgType->isBlockPointerType())
3078 return nullptr;
3079 // First, check attribute on parameter itself.
3080 if (PVD) {
3081 if (auto ParmNNAttr = PVD->getAttr<NonNullAttr>())
3082 return ParmNNAttr;
3083 }
3084 // Check function attributes.
3085 if (!FD)
3086 return nullptr;
3087 for (const auto *NNAttr : FD->specific_attrs<NonNullAttr>()) {
3088 if (NNAttr->isNonNull(ArgNo))
3089 return NNAttr;
3090 }
3091 return nullptr;
3092}
3093
3094namespace {
3095struct CopyBackSwiftError final : EHScopeStack::Cleanup {
3096 Address Temp;
3097 Address Arg;
3098 CopyBackSwiftError(Address temp, Address arg) : Temp(temp), Arg(arg) {}
3099 void Emit(CodeGenFunction &CGF, Flags flags) override {
3100 llvm::Value *errorValue = CGF.Builder.CreateLoad(Temp);
3101 CGF.Builder.CreateStore(errorValue, Arg);
3102 }
3103};
3104} // namespace
3105
3107 llvm::Function *Fn,
3108 const FunctionArgList &Args) {
3109 if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>())
3110 // Naked functions don't have prologues.
3111 return;
3112
3113 // If this is an implicit-return-zero function, go ahead and
3114 // initialize the return value. TODO: it might be nice to have
3115 // a more general mechanism for this that didn't require synthesized
3116 // return statements.
3117 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurCodeDecl)) {
3118 if (FD->hasImplicitReturnZero()) {
3119 QualType RetTy = FD->getReturnType().getUnqualifiedType();
3120 llvm::Type *LLVMTy = CGM.getTypes().ConvertType(RetTy);
3121 llvm::Constant *Zero = llvm::Constant::getNullValue(LLVMTy);
3122 Builder.CreateStore(Zero, ReturnValue);
3123 }
3124 }
3125
3126 // FIXME: We no longer need the types from FunctionArgList; lift up and
3127 // simplify.
3128
3129 ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), FI);
3130 assert(Fn->arg_size() == IRFunctionArgs.totalIRArgs());
3131
3132 // If we're using inalloca, all the memory arguments are GEPs off of the last
3133 // parameter, which is a pointer to the complete memory area.
3134 Address ArgStruct = Address::invalid();
3135 if (IRFunctionArgs.hasInallocaArg())
3136 ArgStruct = Address(Fn->getArg(IRFunctionArgs.getInallocaArgNo()),
3138
3139 // Name the struct return parameter.
3140 if (IRFunctionArgs.hasSRetArg()) {
3141 auto AI = Fn->getArg(IRFunctionArgs.getSRetArgNo());
3142 AI->setName("agg.result");
3143 AI->addAttr(llvm::Attribute::NoAlias);
3144 }
3145
3146 // Track if we received the parameter as a pointer (indirect, byval, or
3147 // inalloca). If already have a pointer, EmitParmDecl doesn't need to copy it
3148 // into a local alloca for us.
3150 ArgVals.reserve(Args.size());
3151
3152 // Create a pointer value for every parameter declaration. This usually
3153 // entails copying one or more LLVM IR arguments into an alloca. Don't push
3154 // any cleanups or do anything that might unwind. We do that separately, so
3155 // we can push the cleanups in the correct order for the ABI.
3156 assert(FI.arg_size() == Args.size() &&
3157 "Mismatch between function signature & arguments.");
3158 unsigned ArgNo = 0;
3160 for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end(); i != e;
3161 ++i, ++info_it, ++ArgNo) {
3162 const VarDecl *Arg = *i;
3163 const ABIArgInfo &ArgI = info_it->info;
3164
3165 bool isPromoted =
3166 isa<ParmVarDecl>(Arg) && cast<ParmVarDecl>(Arg)->isKNRPromoted();
3167 // We are converting from ABIArgInfo type to VarDecl type directly, unless
3168 // the parameter is promoted. In this case we convert to
3169 // CGFunctionInfo::ArgInfo type with subsequent argument demotion.
3170 QualType Ty = isPromoted ? info_it->type : Arg->getType();
3171 assert(hasScalarEvaluationKind(Ty) ==
3173
3174 unsigned FirstIRArg, NumIRArgs;
3175 std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
3176
3177 switch (ArgI.getKind()) {
3178 case ABIArgInfo::InAlloca: {
3179 assert(NumIRArgs == 0);
3180 auto FieldIndex = ArgI.getInAllocaFieldIndex();
3181 Address V =
3182 Builder.CreateStructGEP(ArgStruct, FieldIndex, Arg->getName());
3183 if (ArgI.getInAllocaIndirect())
3184 V = Address(Builder.CreateLoad(V), ConvertTypeForMem(Ty),
3185 getContext().getTypeAlignInChars(Ty));
3186 ArgVals.push_back(ParamValue::forIndirect(V));
3187 break;
3188 }
3189
3192 assert(NumIRArgs == 1);
3194 Fn->getArg(FirstIRArg), Ty, ArgI.getIndirectAlign(), false, nullptr,
3195 nullptr, KnownNonNull);
3196
3197 if (!hasScalarEvaluationKind(Ty)) {
3198 // Aggregates and complex variables are accessed by reference. All we
3199 // need to do is realign the value, if requested. Also, if the address
3200 // may be aliased, copy it to ensure that the parameter variable is
3201 // mutable and has a unique adress, as C requires.
3202 if (ArgI.getIndirectRealign() || ArgI.isIndirectAliased()) {
3203 RawAddress AlignedTemp = CreateMemTemp(Ty, "coerce");
3204
3205 // Copy from the incoming argument pointer to the temporary with the
3206 // appropriate alignment.
3207 //
3208 // FIXME: We should have a common utility for generating an aggregate
3209 // copy.
3210 CharUnits Size = getContext().getTypeSizeInChars(Ty);
3211 Builder.CreateMemCpy(
3212 AlignedTemp.getPointer(), AlignedTemp.getAlignment().getAsAlign(),
3213 ParamAddr.emitRawPointer(*this),
3214 ParamAddr.getAlignment().getAsAlign(),
3215 llvm::ConstantInt::get(IntPtrTy, Size.getQuantity()));
3216 ParamAddr = AlignedTemp;
3217 }
3218 ArgVals.push_back(ParamValue::forIndirect(ParamAddr));
3219 } else {
3220 // Load scalar value from indirect argument.
3221 llvm::Value *V =
3222 EmitLoadOfScalar(ParamAddr, false, Ty, Arg->getBeginLoc());
3223
3224 if (isPromoted)
3225 V = emitArgumentDemotion(*this, Arg, V);
3226 ArgVals.push_back(ParamValue::forDirect(V));
3227 }
3228 break;
3229 }
3230
3231 case ABIArgInfo::Extend:
3232 case ABIArgInfo::Direct: {
3233 auto AI = Fn->getArg(FirstIRArg);
3234 llvm::Type *LTy = ConvertType(Arg->getType());
3235
3236 // Prepare parameter attributes. So far, only attributes for pointer
3237 // parameters are prepared. See
3238 // http://llvm.org/docs/LangRef.html#paramattrs.
3239 if (ArgI.getDirectOffset() == 0 && LTy->isPointerTy() &&
3240 ArgI.getCoerceToType()->isPointerTy()) {
3241 assert(NumIRArgs == 1);
3242
3243 if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Arg)) {
3244 // Set `nonnull` attribute if any.
3245 if (getNonNullAttr(CurCodeDecl, PVD, PVD->getType(),
3246 PVD->getFunctionScopeIndex()) &&
3247 !CGM.getCodeGenOpts().NullPointerIsValid)
3248 AI->addAttr(llvm::Attribute::NonNull);
3249
3250 QualType OTy = PVD->getOriginalType();
3251 if (const auto *ArrTy = getContext().getAsConstantArrayType(OTy)) {
3252 // A C99 array parameter declaration with the static keyword also
3253 // indicates dereferenceability, and if the size is constant we can
3254 // use the dereferenceable attribute (which requires the size in
3255 // bytes).
3256 if (ArrTy->getSizeModifier() == ArraySizeModifier::Static) {
3257 QualType ETy = ArrTy->getElementType();
3258 llvm::Align Alignment =
3259 CGM.getNaturalTypeAlignment(ETy).getAsAlign();
3260 AI->addAttrs(llvm::AttrBuilder(getLLVMContext())
3261 .addAlignmentAttr(Alignment));
3262 uint64_t ArrSize = ArrTy->getZExtSize();
3263 if (!ETy->isIncompleteType() && ETy->isConstantSizeType() &&
3264 ArrSize) {
3265 llvm::AttrBuilder Attrs(getLLVMContext());
3266 Attrs.addDereferenceableAttr(
3267 getContext().getTypeSizeInChars(ETy).getQuantity() *
3268 ArrSize);
3269 AI->addAttrs(Attrs);
3270 } else if (getContext().getTargetInfo().getNullPointerValue(
3271 ETy.getAddressSpace()) == 0 &&
3272 !CGM.getCodeGenOpts().NullPointerIsValid) {
3273 AI->addAttr(llvm::Attribute::NonNull);
3274 }
3275 }
3276 } else if (const auto *ArrTy =
3277 getContext().getAsVariableArrayType(OTy)) {
3278 // For C99 VLAs with the static keyword, we don't know the size so
3279 // we can't use the dereferenceable attribute, but in addrspace(0)
3280 // we know that it must be nonnull.
3281 if (ArrTy->getSizeModifier() == ArraySizeModifier::Static) {
3282 QualType ETy = ArrTy->getElementType();
3283 llvm::Align Alignment =
3284 CGM.getNaturalTypeAlignment(ETy).getAsAlign();
3285 AI->addAttrs(llvm::AttrBuilder(getLLVMContext())
3286 .addAlignmentAttr(Alignment));
3287 if (!getTypes().getTargetAddressSpace(ETy) &&
3288 !CGM.getCodeGenOpts().NullPointerIsValid)
3289 AI->addAttr(llvm::Attribute::NonNull);
3290 }
3291 }
3292
3293 // Set `align` attribute if any.
3294 const auto *AVAttr = PVD->getAttr<AlignValueAttr>();
3295 if (!AVAttr)
3296 if (const auto *TOTy = OTy->getAs<TypedefType>())
3297 AVAttr = TOTy->getDecl()->getAttr<AlignValueAttr>();
3298 if (AVAttr && !SanOpts.has(SanitizerKind::Alignment)) {
3299 // If alignment-assumption sanitizer is enabled, we do *not* add
3300 // alignment attribute here, but emit normal alignment assumption,
3301 // so the UBSAN check could function.
3302 llvm::ConstantInt *AlignmentCI =
3303 cast<llvm::ConstantInt>(EmitScalarExpr(AVAttr->getAlignment()));
3304 uint64_t AlignmentInt =
3305 AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment);
3306 if (AI->getParamAlign().valueOrOne() < AlignmentInt) {
3307 AI->removeAttr(llvm::Attribute::AttrKind::Alignment);
3308 AI->addAttrs(llvm::AttrBuilder(getLLVMContext())
3309 .addAlignmentAttr(llvm::Align(AlignmentInt)));
3310 }
3311 }
3312 }
3313
3314 // Set 'noalias' if an argument type has the `restrict` qualifier.
3315 if (Arg->getType().isRestrictQualified())
3316 AI->addAttr(llvm::Attribute::NoAlias);
3317 }
3318
3319 // Prepare the argument value. If we have the trivial case, handle it
3320 // with no muss and fuss.
3322 ArgI.getCoerceToType() == ConvertType(Ty) &&
3323 ArgI.getDirectOffset() == 0) {
3324 assert(NumIRArgs == 1);
3325
3326 // LLVM expects swifterror parameters to be used in very restricted
3327 // ways. Copy the value into a less-restricted temporary.
3328 llvm::Value *V = AI;
3329 if (FI.getExtParameterInfo(ArgNo).getABI() ==
3331 QualType pointeeTy = Ty->getPointeeType();
3332 assert(pointeeTy->isPointerType());
3333 RawAddress temp =
3334 CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
3336 V, pointeeTy, getContext().getTypeAlignInChars(pointeeTy));
3337 llvm::Value *incomingErrorValue = Builder.CreateLoad(arg);
3338 Builder.CreateStore(incomingErrorValue, temp);
3339 V = temp.getPointer();
3340
3341 // Push a cleanup to copy the value back at the end of the function.
3342 // The convention does not guarantee that the value will be written
3343 // back if the function exits with an unwind exception.
3344 EHStack.pushCleanup<CopyBackSwiftError>(NormalCleanup, temp, arg);
3345 }
3346
3347 // Ensure the argument is the correct type.
3348 if (V->getType() != ArgI.getCoerceToType())
3349 V = Builder.CreateBitCast(V, ArgI.getCoerceToType());
3350
3351 if (isPromoted)
3352 V = emitArgumentDemotion(*this, Arg, V);
3353
3354 // Because of merging of function types from multiple decls it is
3355 // possible for the type of an argument to not match the corresponding
3356 // type in the function type. Since we are codegening the callee
3357 // in here, add a cast to the argument type.
3358 llvm::Type *LTy = ConvertType(Arg->getType());
3359 if (V->getType() != LTy)
3360 V = Builder.CreateBitCast(V, LTy);
3361
3362 ArgVals.push_back(ParamValue::forDirect(V));
3363 break;
3364 }
3365
3366 // VLST arguments are coerced to VLATs at the function boundary for
3367 // ABI consistency. If this is a VLST that was coerced to
3368 // a VLAT at the function boundary and the types match up, use
3369 // llvm.vector.extract to convert back to the original VLST.
3370 if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(ConvertType(Ty))) {
3371 llvm::Value *ArgVal = Fn->getArg(FirstIRArg);
3372 if (auto *VecTyFrom =
3373 dyn_cast<llvm::ScalableVectorType>(ArgVal->getType())) {
3374 auto [Coerced, Extracted] = CoerceScalableToFixed(
3375 *this, VecTyTo, VecTyFrom, ArgVal, Arg->getName());
3376 if (Extracted) {
3377 assert(NumIRArgs == 1);
3378 ArgVals.push_back(ParamValue::forDirect(Coerced));
3379 break;
3380 }
3381 }
3382 }
3383
3384 llvm::StructType *STy =
3385 dyn_cast<llvm::StructType>(ArgI.getCoerceToType());
3386 Address Alloca =
3387 CreateMemTemp(Ty, getContext().getDeclAlign(Arg), Arg->getName());
3388
3389 // Pointer to store into.
3390 Address Ptr = emitAddressAtOffset(*this, Alloca, ArgI);
3391
3392 // Fast-isel and the optimizer generally like scalar values better than
3393 // FCAs, so we flatten them if this is safe to do for this argument.
3394 if (ArgI.isDirect() && ArgI.getCanBeFlattened() && STy &&
3395 STy->getNumElements() > 1) {
3396 llvm::TypeSize StructSize = CGM.getDataLayout().getTypeAllocSize(STy);
3397 llvm::TypeSize PtrElementSize =
3398 CGM.getDataLayout().getTypeAllocSize(Ptr.getElementType());
3399 if (StructSize.isScalable()) {
3400 assert(STy->containsHomogeneousScalableVectorTypes() &&
3401 "ABI only supports structure with homogeneous scalable vector "
3402 "type");
3403 assert(StructSize == PtrElementSize &&
3404 "Only allow non-fractional movement of structure with"
3405 "homogeneous scalable vector type");
3406 assert(STy->getNumElements() == NumIRArgs);
3407
3408 llvm::Value *LoadedStructValue = llvm::PoisonValue::get(STy);
3409 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
3410 auto *AI = Fn->getArg(FirstIRArg + i);
3411 AI->setName(Arg->getName() + ".coerce" + Twine(i));
3412 LoadedStructValue =
3413 Builder.CreateInsertValue(LoadedStructValue, AI, i);
3414 }
3415
3416 Builder.CreateStore(LoadedStructValue, Ptr);
3417 } else {
3418 uint64_t SrcSize = StructSize.getFixedValue();
3419 uint64_t DstSize = PtrElementSize.getFixedValue();
3420
3421 Address AddrToStoreInto = Address::invalid();
3422 if (SrcSize <= DstSize) {
3423 AddrToStoreInto = Ptr.withElementType(STy);
3424 } else {
3425 AddrToStoreInto =
3426 CreateTempAlloca(STy, Alloca.getAlignment(), "coerce");
3427 }
3428
3429 assert(STy->getNumElements() == NumIRArgs);
3430 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
3431 auto AI = Fn->getArg(FirstIRArg + i);
3432 AI->setName(Arg->getName() + ".coerce" + Twine(i));
3433 Address EltPtr = Builder.CreateStructGEP(AddrToStoreInto, i);
3434 Builder.CreateStore(AI, EltPtr);
3435 }
3436
3437 if (SrcSize > DstSize) {
3438 Builder.CreateMemCpy(Ptr, AddrToStoreInto, DstSize);
3439 }
3440 }
3441 } else {
3442 // Simple case, just do a coerced store of the argument into the alloca.
3443 assert(NumIRArgs == 1);
3444 auto AI = Fn->getArg(FirstIRArg);
3445 AI->setName(Arg->getName() + ".coerce");
3447 AI, Ptr,
3448 llvm::TypeSize::getFixed(
3449 getContext().getTypeSizeInChars(Ty).getQuantity() -
3450 ArgI.getDirectOffset()),
3451 /*DstIsVolatile=*/false);
3452 }
3453
3454 // Match to what EmitParmDecl is expecting for this type.
3456 llvm::Value *V =
3457 EmitLoadOfScalar(Alloca, false, Ty, Arg->getBeginLoc());
3458 if (isPromoted)
3459 V = emitArgumentDemotion(*this, Arg, V);
3460 ArgVals.push_back(ParamValue::forDirect(V));
3461 } else {
3462 ArgVals.push_back(ParamValue::forIndirect(Alloca));
3463 }
3464 break;
3465 }
3466
3468 // Reconstruct into a temporary.
3469 Address alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg));
3470 ArgVals.push_back(ParamValue::forIndirect(alloca));
3471
3472 auto coercionType = ArgI.getCoerceAndExpandType();
3473 auto unpaddedCoercionType = ArgI.getUnpaddedCoerceAndExpandType();
3474 auto *unpaddedStruct = dyn_cast<llvm::StructType>(unpaddedCoercionType);
3475
3476 alloca = alloca.withElementType(coercionType);
3477
3478 unsigned argIndex = FirstIRArg;
3479 unsigned unpaddedIndex = 0;
3480 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
3481 llvm::Type *eltType = coercionType->getElementType(i);
3483 continue;
3484
3485 auto eltAddr = Builder.CreateStructGEP(alloca, i);
3486 llvm::Value *elt = Fn->getArg(argIndex++);
3487
3488 auto paramType = unpaddedStruct
3489 ? unpaddedStruct->getElementType(unpaddedIndex++)
3490 : unpaddedCoercionType;
3491
3492 if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(eltType)) {
3493 if (auto *VecTyFrom = dyn_cast<llvm::ScalableVectorType>(paramType)) {
3494 bool Extracted;
3495 std::tie(elt, Extracted) = CoerceScalableToFixed(
3496 *this, VecTyTo, VecTyFrom, elt, elt->getName());
3497 assert(Extracted && "Unexpected scalable to fixed vector coercion");
3498 }
3499 }
3500 Builder.CreateStore(elt, eltAddr);
3501 }
3502 assert(argIndex == FirstIRArg + NumIRArgs);
3503 break;
3504 }
3505
3506 case ABIArgInfo::Expand: {
3507 // If this structure was expanded into multiple arguments then
3508 // we need to create a temporary and reconstruct it from the
3509 // arguments.
3510 Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg));
3511 LValue LV = MakeAddrLValue(Alloca, Ty);
3512 ArgVals.push_back(ParamValue::forIndirect(Alloca));
3513
3514 auto FnArgIter = Fn->arg_begin() + FirstIRArg;
3515 ExpandTypeFromArgs(Ty, LV, FnArgIter);
3516 assert(FnArgIter == Fn->arg_begin() + FirstIRArg + NumIRArgs);
3517 for (unsigned i = 0, e = NumIRArgs; i != e; ++i) {
3518 auto AI = Fn->getArg(FirstIRArg + i);
3519 AI->setName(Arg->getName() + "." + Twine(i));
3520 }
3521 break;
3522 }
3523
3525 auto *AI = Fn->getArg(FirstIRArg);
3526 AI->setName(Arg->getName() + ".target_coerce");
3527 Address Alloca =
3528 CreateMemTemp(Ty, getContext().getDeclAlign(Arg), Arg->getName());
3529 Address Ptr = emitAddressAtOffset(*this, Alloca, ArgI);
3530 CGM.getABIInfo().createCoercedStore(AI, Ptr, ArgI, false, *this);
3532 llvm::Value *V =
3533 EmitLoadOfScalar(Alloca, false, Ty, Arg->getBeginLoc());
3534 if (isPromoted) {
3535 V = emitArgumentDemotion(*this, Arg, V);
3536 }
3537 ArgVals.push_back(ParamValue::forDirect(V));
3538 } else {
3539 ArgVals.push_back(ParamValue::forIndirect(Alloca));
3540 }
3541 break;
3542 }
3543 case ABIArgInfo::Ignore:
3544 assert(NumIRArgs == 0);
3545 // Initialize the local variable appropriately.
3546 if (!hasScalarEvaluationKind(Ty)) {
3547 ArgVals.push_back(ParamValue::forIndirect(CreateMemTemp(Ty)));
3548 } else {
3549 llvm::Value *U = llvm::UndefValue::get(ConvertType(Arg->getType()));
3550 ArgVals.push_back(ParamValue::forDirect(U));
3551 }
3552 break;
3553 }
3554 }
3555
3556 if (getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
3557 for (int I = Args.size() - 1; I >= 0; --I)
3558 EmitParmDecl(*Args[I], ArgVals[I], I + 1);
3559 } else {
3560 for (unsigned I = 0, E = Args.size(); I != E; ++I)
3561 EmitParmDecl(*Args[I], ArgVals[I], I + 1);
3562 }
3563}
3564
3565static void eraseUnusedBitCasts(llvm::Instruction *insn) {
3566 while (insn->use_empty()) {
3567 llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(insn);
3568 if (!bitcast)
3569 return;
3570
3571 // This is "safe" because we would have used a ConstantExpr otherwise.
3572 insn = cast<llvm::Instruction>(bitcast->getOperand(0));
3573 bitcast->eraseFromParent();
3574 }
3575}
3576
3577/// Try to emit a fused autorelease of a return result.
3579 llvm::Value *result) {
3580 // We must be immediately followed the cast.
3581 llvm::BasicBlock *BB = CGF.Builder.GetInsertBlock();
3582 if (BB->empty())
3583 return nullptr;
3584 if (&BB->back() != result)
3585 return nullptr;
3586
3587 llvm::Type *resultType = result->getType();
3588
3589 // result is in a BasicBlock and is therefore an Instruction.
3590 llvm::Instruction *generator = cast<llvm::Instruction>(result);
3591
3593
3594 // Look for:
3595 // %generator = bitcast %type1* %generator2 to %type2*
3596 while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(generator)) {
3597 // We would have emitted this as a constant if the operand weren't
3598 // an Instruction.
3599 generator = cast<llvm::Instruction>(bitcast->getOperand(0));
3600
3601 // Require the generator to be immediately followed by the cast.
3602 if (generator->getNextNode() != bitcast)
3603 return nullptr;
3604
3605 InstsToKill.push_back(bitcast);
3606 }
3607
3608 // Look for:
3609 // %generator = call i8* @objc_retain(i8* %originalResult)
3610 // or
3611 // %generator = call i8* @objc_retainAutoreleasedReturnValue(i8* %originalResult)
3612 llvm::CallInst *call = dyn_cast<llvm::CallInst>(generator);
3613 if (!call)
3614 return nullptr;
3615
3616 bool doRetainAutorelease;
3617
3618 if (call->getCalledOperand() == CGF.CGM.getObjCEntrypoints().objc_retain) {
3619 doRetainAutorelease = true;
3620 } else if (call->getCalledOperand() ==
3622 doRetainAutorelease = false;
3623
3624 // If we emitted an assembly marker for this call (and the
3625 // ARCEntrypoints field should have been set if so), go looking
3626 // for that call. If we can't find it, we can't do this
3627 // optimization. But it should always be the immediately previous
3628 // instruction, unless we needed bitcasts around the call.
3630 llvm::Instruction *prev = call->getPrevNode();
3631 assert(prev);
3632 if (isa<llvm::BitCastInst>(prev)) {
3633 prev = prev->getPrevNode();
3634 assert(prev);
3635 }
3636 assert(isa<llvm::CallInst>(prev));
3637 assert(cast<llvm::CallInst>(prev)->getCalledOperand() ==
3639 InstsToKill.push_back(prev);
3640 }
3641 } else {
3642 return nullptr;
3643 }
3644
3645 result = call->getArgOperand(0);
3646 InstsToKill.push_back(call);
3647
3648 // Keep killing bitcasts, for sanity. Note that we no longer care
3649 // about precise ordering as long as there's exactly one use.
3650 while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(result)) {
3651 if (!bitcast->hasOneUse())
3652 break;
3653 InstsToKill.push_back(bitcast);
3654 result = bitcast->getOperand(0);
3655 }
3656
3657 // Delete all the unnecessary instructions, from latest to earliest.
3658 for (auto *I : InstsToKill)
3659 I->eraseFromParent();
3660
3661 // Do the fused retain/autorelease if we were asked to.
3662 if (doRetainAutorelease)
3663 result = CGF.EmitARCRetainAutoreleaseReturnValue(result);
3664
3665 // Cast back to the result type.
3666 return CGF.Builder.CreateBitCast(result, resultType);
3667}
3668
3669/// If this is a +1 of the value of an immutable 'self', remove it.
3671 llvm::Value *result) {
3672 // This is only applicable to a method with an immutable 'self'.
3673 const ObjCMethodDecl *method =
3674 dyn_cast_or_null<ObjCMethodDecl>(CGF.CurCodeDecl);
3675 if (!method)
3676 return nullptr;
3677 const VarDecl *self = method->getSelfDecl();
3678 if (!self->getType().isConstQualified())
3679 return nullptr;
3680
3681 // Look for a retain call. Note: stripPointerCasts looks through returned arg
3682 // functions, which would cause us to miss the retain.
3683 llvm::CallInst *retainCall = dyn_cast<llvm::CallInst>(result);
3684 if (!retainCall || retainCall->getCalledOperand() !=
3686 return nullptr;
3687
3688 // Look for an ordinary load of 'self'.
3689 llvm::Value *retainedValue = retainCall->getArgOperand(0);
3690 llvm::LoadInst *load =
3691 dyn_cast<llvm::LoadInst>(retainedValue->stripPointerCasts());
3692 if (!load || load->isAtomic() || load->isVolatile() ||
3693 load->getPointerOperand() != CGF.GetAddrOfLocalVar(self).getBasePointer())
3694 return nullptr;
3695
3696 // Okay! Burn it all down. This relies for correctness on the
3697 // assumption that the retain is emitted as part of the return and
3698 // that thereafter everything is used "linearly".
3699 llvm::Type *resultType = result->getType();
3701 assert(retainCall->use_empty());
3702 retainCall->eraseFromParent();
3704
3705 return CGF.Builder.CreateBitCast(load, resultType);
3706}
3707
3708/// Emit an ARC autorelease of the result of a function.
3709///
3710/// \return the value to actually return from the function
3712 llvm::Value *result) {
3713 // If we're returning 'self', kill the initial retain. This is a
3714 // heuristic attempt to "encourage correctness" in the really unfortunate
3715 // case where we have a return of self during a dealloc and we desperately
3716 // need to avoid the possible autorelease.
3717 if (llvm::Value *self = tryRemoveRetainOfSelf(CGF, result))
3718 return self;
3719
3720 // At -O0, try to emit a fused retain/autorelease.
3721 if (CGF.shouldUseFusedARCCalls())
3722 if (llvm::Value *fused = tryEmitFusedAutoreleaseOfResult(CGF, result))
3723 return fused;
3724
3725 return CGF.EmitARCAutoreleaseReturnValue(result);
3726}
3727
3728/// Heuristically search for a dominating store to the return-value slot.
3730 llvm::Value *ReturnValuePtr = CGF.ReturnValue.getBasePointer();
3731
3732 // Check if a User is a store which pointerOperand is the ReturnValue.
3733 // We are looking for stores to the ReturnValue, not for stores of the
3734 // ReturnValue to some other location.
3735 auto GetStoreIfValid = [&CGF,
3736 ReturnValuePtr](llvm::User *U) -> llvm::StoreInst * {
3737 auto *SI = dyn_cast<llvm::StoreInst>(U);
3738 if (!SI || SI->getPointerOperand() != ReturnValuePtr ||
3739 SI->getValueOperand()->getType() != CGF.ReturnValue.getElementType())
3740 return nullptr;
3741 // These aren't actually possible for non-coerced returns, and we
3742 // only care about non-coerced returns on this code path.
3743 // All memory instructions inside __try block are volatile.
3744 assert(!SI->isAtomic() &&
3745 (!SI->isVolatile() || CGF.currentFunctionUsesSEHTry()));
3746 return SI;
3747 };
3748 // If there are multiple uses of the return-value slot, just check
3749 // for something immediately preceding the IP. Sometimes this can
3750 // happen with how we generate implicit-returns; it can also happen
3751 // with noreturn cleanups.
3752 if (!ReturnValuePtr->hasOneUse()) {
3753 llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
3754 if (IP->empty())
3755 return nullptr;
3756
3757 // Look at directly preceding instruction, skipping bitcasts, lifetime
3758 // markers, and fake uses and their operands.
3759 const llvm::Instruction *LoadIntoFakeUse = nullptr;
3760 for (llvm::Instruction &I : llvm::reverse(*IP)) {
3761 // Ignore instructions that are just loads for fake uses; the load should
3762 // immediately precede the fake use, so we only need to remember the
3763 // operand for the last fake use seen.
3764 if (LoadIntoFakeUse == &I)
3765 continue;
3766 if (isa<llvm::BitCastInst>(&I))
3767 continue;
3768 if (auto *II = dyn_cast<llvm::IntrinsicInst>(&I)) {
3769 if (II->getIntrinsicID() == llvm::Intrinsic::lifetime_end)
3770 continue;
3771
3772 if (II->getIntrinsicID() == llvm::Intrinsic::fake_use) {
3773 LoadIntoFakeUse = dyn_cast<llvm::Instruction>(II->getArgOperand(0));
3774 continue;
3775 }
3776 }
3777 return GetStoreIfValid(&I);
3778 }
3779 return nullptr;
3780 }
3781
3782 llvm::StoreInst *store = GetStoreIfValid(ReturnValuePtr->user_back());
3783 if (!store)
3784 return nullptr;
3785
3786 // Now do a first-and-dirty dominance check: just walk up the
3787 // single-predecessors chain from the current insertion point.
3788 llvm::BasicBlock *StoreBB = store->getParent();
3789 llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
3791 while (IP != StoreBB) {
3792 if (!SeenBBs.insert(IP).second || !(IP = IP->getSinglePredecessor()))
3793 return nullptr;
3794 }
3795
3796 // Okay, the store's basic block dominates the insertion point; we
3797 // can do our thing.
3798 return store;
3799}
3800
3801// Helper functions for EmitCMSEClearRecord
3802
3803// Set the bits corresponding to a field having width `BitWidth` and located at
3804// offset `BitOffset` (from the least significant bit) within a storage unit of
3805// `Bits.size()` bytes. Each element of `Bits` corresponds to one target byte.
3806// Use little-endian layout, i.e.`Bits[0]` is the LSB.
3807static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int BitOffset,
3808 int BitWidth, int CharWidth) {
3809 assert(CharWidth <= 64);
3810 assert(static_cast<unsigned>(BitWidth) <= Bits.size() * CharWidth);
3811
3812 int Pos = 0;
3813 if (BitOffset >= CharWidth) {
3814 Pos += BitOffset / CharWidth;
3815 BitOffset = BitOffset % CharWidth;
3816 }
3817
3818 const uint64_t Used = (uint64_t(1) << CharWidth) - 1;
3819 if (BitOffset + BitWidth >= CharWidth) {
3820 Bits[Pos++] |= (Used << BitOffset) & Used;
3821 BitWidth -= CharWidth - BitOffset;
3822 BitOffset = 0;
3823 }
3824
3825 while (BitWidth >= CharWidth) {
3826 Bits[Pos++] = Used;
3827 BitWidth -= CharWidth;
3828 }
3829
3830 if (BitWidth > 0)
3831 Bits[Pos++] |= (Used >> (CharWidth - BitWidth)) << BitOffset;
3832}
3833
3834// Set the bits corresponding to a field having width `BitWidth` and located at
3835// offset `BitOffset` (from the least significant bit) within a storage unit of
3836// `StorageSize` bytes, located at `StorageOffset` in `Bits`. Each element of
3837// `Bits` corresponds to one target byte. Use target endian layout.
3838static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int StorageOffset,
3839 int StorageSize, int BitOffset, int BitWidth,
3840 int CharWidth, bool BigEndian) {
3841
3842 SmallVector<uint64_t, 8> TmpBits(StorageSize);
3843 setBitRange(TmpBits, BitOffset, BitWidth, CharWidth);
3844
3845 if (BigEndian)
3846 std::reverse(TmpBits.begin(), TmpBits.end());
3847
3848 for (uint64_t V : TmpBits)
3849 Bits[StorageOffset++] |= V;
3850}
3851
3852static void setUsedBits(CodeGenModule &, QualType, int,
3853 SmallVectorImpl<uint64_t> &);
3854
3855// Set the bits in `Bits`, which correspond to the value representations of
3856// the actual members of the record type `RTy`. Note that this function does
3857// not handle base classes, virtual tables, etc, since they cannot happen in
3858// CMSE function arguments or return. The bit mask corresponds to the target
3859// memory layout, i.e. it's endian dependent.
3860static void setUsedBits(CodeGenModule &CGM, const RecordType *RTy, int Offset,
3862 ASTContext &Context = CGM.getContext();
3863 int CharWidth = Context.getCharWidth();
3864 const RecordDecl *RD = RTy->getOriginalDecl()->getDefinition();
3865 const ASTRecordLayout &ASTLayout = Context.getASTRecordLayout(RD);
3866 const CGRecordLayout &Layout = CGM.getTypes().getCGRecordLayout(RD);
3867
3868 int Idx = 0;
3869 for (auto I = RD->field_begin(), E = RD->field_end(); I != E; ++I, ++Idx) {
3870 const FieldDecl *F = *I;
3871
3872 if (F->isUnnamedBitField() || F->isZeroLengthBitField() ||
3874 continue;
3875
3876 if (F->isBitField()) {
3877 const CGBitFieldInfo &BFI = Layout.getBitFieldInfo(F);
3878 setBitRange(Bits, Offset + BFI.StorageOffset.getQuantity(),
3879 BFI.StorageSize / CharWidth, BFI.Offset, BFI.Size, CharWidth,
3880 CGM.getDataLayout().isBigEndian());
3881 continue;
3882 }
3883
3884 setUsedBits(CGM, F->getType(),
3885 Offset + ASTLayout.getFieldOffset(Idx) / CharWidth, Bits);
3886 }
3887}
3888
3889// Set the bits in `Bits`, which correspond to the value representations of
3890// the elements of an array type `ATy`.
3891static void setUsedBits(CodeGenModule &CGM, const ConstantArrayType *ATy,
3892 int Offset, SmallVectorImpl<uint64_t> &Bits) {
3893 const ASTContext &Context = CGM.getContext();
3894
3895 QualType ETy = Context.getBaseElementType(ATy);
3896 int Size = Context.getTypeSizeInChars(ETy).getQuantity();
3897 SmallVector<uint64_t, 4> TmpBits(Size);
3898 setUsedBits(CGM, ETy, 0, TmpBits);
3899
3900 for (int I = 0, N = Context.getConstantArrayElementCount(ATy); I < N; ++I) {
3901 auto Src = TmpBits.begin();
3902 auto Dst = Bits.begin() + Offset + I * Size;
3903 for (int J = 0; J < Size; ++J)
3904 *Dst++ |= *Src++;
3905 }
3906}
3907
3908// Set the bits in `Bits`, which correspond to the value representations of
3909// the type `QTy`.
3910static void setUsedBits(CodeGenModule &CGM, QualType QTy, int Offset,
3912 if (const auto *RTy = QTy->getAsCanonical<RecordType>())
3913 return setUsedBits(CGM, RTy, Offset, Bits);
3914
3915 ASTContext &Context = CGM.getContext();
3916 if (const auto *ATy = Context.getAsConstantArrayType(QTy))
3917 return setUsedBits(CGM, ATy, Offset, Bits);
3918
3919 int Size = Context.getTypeSizeInChars(QTy).getQuantity();
3920 if (Size <= 0)
3921 return;
3922
3923 std::fill_n(Bits.begin() + Offset, Size,
3924 (uint64_t(1) << Context.getCharWidth()) - 1);
3925}
3926
3928 int Pos, int Size, int CharWidth,
3929 bool BigEndian) {
3930 assert(Size > 0);
3931 uint64_t Mask = 0;
3932 if (BigEndian) {
3933 for (auto P = Bits.begin() + Pos, E = Bits.begin() + Pos + Size; P != E;
3934 ++P)
3935 Mask = (Mask << CharWidth) | *P;
3936 } else {
3937 auto P = Bits.begin() + Pos + Size, End = Bits.begin() + Pos;
3938 do
3939 Mask = (Mask << CharWidth) | *--P;
3940 while (P != End);
3941 }
3942 return Mask;
3943}
3944
3945// Emit code to clear the bits in a record, which aren't a part of any user
3946// declared member, when the record is a function return.
3947llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
3948 llvm::IntegerType *ITy,
3949 QualType QTy) {
3950 assert(Src->getType() == ITy);
3951 assert(ITy->getScalarSizeInBits() <= 64);
3952
3953 const llvm::DataLayout &DataLayout = CGM.getDataLayout();
3954 int Size = DataLayout.getTypeStoreSize(ITy);
3955 SmallVector<uint64_t, 4> Bits(Size);
3956 setUsedBits(CGM, QTy->castAsCanonical<RecordType>(), 0, Bits);
3957
3958 int CharWidth = CGM.getContext().getCharWidth();
3959 uint64_t Mask =
3960 buildMultiCharMask(Bits, 0, Size, CharWidth, DataLayout.isBigEndian());
3961
3962 return Builder.CreateAnd(Src, Mask, "cmse.clear");
3963}
3964
3965// Emit code to clear the bits in a record, which aren't a part of any user
3966// declared member, when the record is a function argument.
3967llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
3968 llvm::ArrayType *ATy,
3969 QualType QTy) {
3970 const llvm::DataLayout &DataLayout = CGM.getDataLayout();
3971 int Size = DataLayout.getTypeStoreSize(ATy);
3972 SmallVector<uint64_t, 16> Bits(Size);
3973 setUsedBits(CGM, QTy->castAsCanonical<RecordType>(), 0, Bits);
3974
3975 // Clear each element of the LLVM array.
3976 int CharWidth = CGM.getContext().getCharWidth();
3977 int CharsPerElt =
3978 ATy->getArrayElementType()->getScalarSizeInBits() / CharWidth;
3979 int MaskIndex = 0;
3980 llvm::Value *R = llvm::PoisonValue::get(ATy);
3981 for (int I = 0, N = ATy->getArrayNumElements(); I != N; ++I) {
3982 uint64_t Mask = buildMultiCharMask(Bits, MaskIndex, CharsPerElt, CharWidth,
3983 DataLayout.isBigEndian());
3984 MaskIndex += CharsPerElt;
3985 llvm::Value *T0 = Builder.CreateExtractValue(Src, I);
3986 llvm::Value *T1 = Builder.CreateAnd(T0, Mask, "cmse.clear");
3987 R = Builder.CreateInsertValue(R, T1, I);
3988 }
3989
3990 return R;
3991}
3992
3994 const CGFunctionInfo &FI, bool EmitRetDbgLoc, SourceLocation EndLoc,
3995 uint64_t RetKeyInstructionsSourceAtom) {
3996 if (FI.isNoReturn()) {
3997 // Noreturn functions don't return.
3998 EmitUnreachable(EndLoc);
3999 return;
4000 }
4001
4002 if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>()) {
4003 // Naked functions don't have epilogues.
4004 Builder.CreateUnreachable();
4005 return;
4006 }
4007
4008 // Functions with no result always return void.
4009 if (!ReturnValue.isValid()) {
4010 auto *I = Builder.CreateRetVoid();
4011 if (RetKeyInstructionsSourceAtom)
4012 addInstToSpecificSourceAtom(I, nullptr, RetKeyInstructionsSourceAtom);
4013 else
4014 addInstToNewSourceAtom(I, nullptr);
4015 return;
4016 }
4017
4018 llvm::DebugLoc RetDbgLoc;
4019 llvm::Value *RV = nullptr;
4020 QualType RetTy = FI.getReturnType();
4021 const ABIArgInfo &RetAI = FI.getReturnInfo();
4022
4023 switch (RetAI.getKind()) {
4025 // Aggregates get evaluated directly into the destination. Sometimes we
4026 // need to return the sret value in a register, though.
4027 assert(hasAggregateEvaluationKind(RetTy));
4028 if (RetAI.getInAllocaSRet()) {
4029 llvm::Function::arg_iterator EI = CurFn->arg_end();
4030 --EI;
4031 llvm::Value *ArgStruct = &*EI;
4032 llvm::Value *SRet = Builder.CreateStructGEP(
4033 FI.getArgStruct(), ArgStruct, RetAI.getInAllocaFieldIndex());
4034 llvm::Type *Ty =
4035 cast<llvm::GetElementPtrInst>(SRet)->getResultElementType();
4036 RV = Builder.CreateAlignedLoad(Ty, SRet, getPointerAlign(), "sret");
4037 }
4038 break;
4039
4040 case ABIArgInfo::Indirect: {
4041 auto AI = CurFn->arg_begin();
4042 if (RetAI.isSRetAfterThis())
4043 ++AI;
4044 switch (getEvaluationKind(RetTy)) {
4045 case TEK_Complex: {
4046 ComplexPairTy RT =
4049 /*isInit*/ true);
4050 break;
4051 }
4052 case TEK_Aggregate:
4053 // Do nothing; aggregates get evaluated directly into the destination.
4054 break;
4055 case TEK_Scalar: {
4056 LValueBaseInfo BaseInfo;
4057 TBAAAccessInfo TBAAInfo;
4058 CharUnits Alignment =
4059 CGM.getNaturalTypeAlignment(RetTy, &BaseInfo, &TBAAInfo);
4060 Address ArgAddr(&*AI, ConvertType(RetTy), Alignment);
4061 LValue ArgVal =
4062 LValue::MakeAddr(ArgAddr, RetTy, getContext(), BaseInfo, TBAAInfo);
4064 EmitLoadOfScalar(MakeAddrLValue(ReturnValue, RetTy), EndLoc), ArgVal,
4065 /*isInit*/ true);
4066 break;
4067 }
4068 }
4069 break;
4070 }
4071
4072 case ABIArgInfo::Extend:
4073 case ABIArgInfo::Direct:
4074 if (RetAI.getCoerceToType() == ConvertType(RetTy) &&
4075 RetAI.getDirectOffset() == 0) {
4076 // The internal return value temp always will have pointer-to-return-type
4077 // type, just do a load.
4078
4079 // If there is a dominating store to ReturnValue, we can elide
4080 // the load, zap the store, and usually zap the alloca.
4081 if (llvm::StoreInst *SI = findDominatingStoreToReturnValue(*this)) {
4082 // Reuse the debug location from the store unless there is
4083 // cleanup code to be emitted between the store and return
4084 // instruction.
4085 if (EmitRetDbgLoc && !AutoreleaseResult)
4086 RetDbgLoc = SI->getDebugLoc();
4087 // Get the stored value and nuke the now-dead store.
4088 RV = SI->getValueOperand();
4089 SI->eraseFromParent();
4090
4091 // Otherwise, we have to do a simple load.
4092 } else {
4093 RV = Builder.CreateLoad(ReturnValue);
4094 }
4095 } else {
4096 // If the value is offset in memory, apply the offset now.
4097 Address V = emitAddressAtOffset(*this, ReturnValue, RetAI);
4098
4099 RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), *this);
4100 }
4101
4102 // In ARC, end functions that return a retainable type with a call
4103 // to objc_autoreleaseReturnValue.
4104 if (AutoreleaseResult) {
4105#ifndef NDEBUG
4106 // Type::isObjCRetainabletype has to be called on a QualType that hasn't
4107 // been stripped of the typedefs, so we cannot use RetTy here. Get the
4108 // original return type of FunctionDecl, CurCodeDecl, and BlockDecl from
4109 // CurCodeDecl or BlockInfo.
4110 QualType RT;
4111
4112 if (auto *FD = dyn_cast<FunctionDecl>(CurCodeDecl))
4113 RT = FD->getReturnType();
4114 else if (auto *MD = dyn_cast<ObjCMethodDecl>(CurCodeDecl))
4115 RT = MD->getReturnType();
4116 else if (isa<BlockDecl>(CurCodeDecl))
4117 RT = BlockInfo->BlockExpression->getFunctionType()->getReturnType();
4118 else
4119 llvm_unreachable("Unexpected function/method type");
4120
4121 assert(getLangOpts().ObjCAutoRefCount && !FI.isReturnsRetained() &&
4122 RT->isObjCRetainableType());
4123#endif
4124 RV = emitAutoreleaseOfResult(*this, RV);
4125 }
4126
4127 break;
4128
4129 case ABIArgInfo::Ignore:
4130 break;
4131
4133 auto coercionType = RetAI.getCoerceAndExpandType();
4134 auto unpaddedCoercionType = RetAI.getUnpaddedCoerceAndExpandType();
4135 auto *unpaddedStruct = dyn_cast<llvm::StructType>(unpaddedCoercionType);
4136
4137 // Load all of the coerced elements out into results.
4139 Address addr = ReturnValue.withElementType(coercionType);
4140 unsigned unpaddedIndex = 0;
4141 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
4142 auto coercedEltType = coercionType->getElementType(i);
4143 if (ABIArgInfo::isPaddingForCoerceAndExpand(coercedEltType))
4144 continue;
4145
4146 auto eltAddr = Builder.CreateStructGEP(addr, i);
4147 llvm::Value *elt = CreateCoercedLoad(
4148 eltAddr,
4149 unpaddedStruct ? unpaddedStruct->getElementType(unpaddedIndex++)
4150 : unpaddedCoercionType,
4151 *this);
4152 results.push_back(elt);
4153 }
4154
4155 // If we have one result, it's the single direct result type.
4156 if (results.size() == 1) {
4157 RV = results[0];
4158
4159 // Otherwise, we need to make a first-class aggregate.
4160 } else {
4161 // Construct a return type that lacks padding elements.
4162 llvm::Type *returnType = RetAI.getUnpaddedCoerceAndExpandType();
4163
4164 RV = llvm::PoisonValue::get(returnType);
4165 for (unsigned i = 0, e = results.size(); i != e; ++i) {
4166 RV = Builder.CreateInsertValue(RV, results[i], i);
4167 }
4168 }
4169 break;
4170 }
4172 Address V = emitAddressAtOffset(*this, ReturnValue, RetAI);
4173 RV = CGM.getABIInfo().createCoercedLoad(V, RetAI, *this);
4174 break;
4175 }
4176 case ABIArgInfo::Expand:
4178 llvm_unreachable("Invalid ABI kind for return argument");
4179 }
4180
4181 llvm::Instruction *Ret;
4182 if (RV) {
4183 if (CurFuncDecl && CurFuncDecl->hasAttr<CmseNSEntryAttr>()) {
4184 // For certain return types, clear padding bits, as they may reveal
4185 // sensitive information.
4186 // Small struct/union types are passed as integers.
4187 auto *ITy = dyn_cast<llvm::IntegerType>(RV->getType());
4188 if (ITy != nullptr && isa<RecordType>(RetTy.getCanonicalType()))
4189 RV = EmitCMSEClearRecord(RV, ITy, RetTy);
4190 }
4192 Ret = Builder.CreateRet(RV);
4193 } else {
4194 Ret = Builder.CreateRetVoid();
4195 }
4196
4197 if (RetDbgLoc)
4198 Ret->setDebugLoc(std::move(RetDbgLoc));
4199
4200 llvm::Value *Backup = RV ? Ret->getOperand(0) : nullptr;
4201 if (RetKeyInstructionsSourceAtom)
4202 addInstToSpecificSourceAtom(Ret, Backup, RetKeyInstructionsSourceAtom);
4203 else
4204 addInstToNewSourceAtom(Ret, Backup);
4205}
4206
4208 // A current decl may not be available when emitting vtable thunks.
4209 if (!CurCodeDecl)
4210 return;
4211
4212 // If the return block isn't reachable, neither is this check, so don't emit
4213 // it.
4214 if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty())
4215 return;
4216
4217 ReturnsNonNullAttr *RetNNAttr = nullptr;
4218 if (SanOpts.has(SanitizerKind::ReturnsNonnullAttribute))
4219 RetNNAttr = CurCodeDecl->getAttr<ReturnsNonNullAttr>();
4220
4221 if (!RetNNAttr && !requiresReturnValueNullabilityCheck())
4222 return;
4223
4224 // Prefer the returns_nonnull attribute if it's present.
4225 SourceLocation AttrLoc;
4227 SanitizerHandler Handler;
4228 if (RetNNAttr) {
4229 assert(!requiresReturnValueNullabilityCheck() &&
4230 "Cannot check nullability and the nonnull attribute");
4231 AttrLoc = RetNNAttr->getLocation();
4232 CheckKind = SanitizerKind::SO_ReturnsNonnullAttribute;
4233 Handler = SanitizerHandler::NonnullReturn;
4234 } else {
4235 if (auto *DD = dyn_cast<DeclaratorDecl>(CurCodeDecl))
4236 if (auto *TSI = DD->getTypeSourceInfo())
4237 if (auto FTL = TSI->getTypeLoc().getAsAdjusted<FunctionTypeLoc>())
4238 AttrLoc = FTL.getReturnLoc().findNullabilityLoc();
4239 CheckKind = SanitizerKind::SO_NullabilityReturn;
4240 Handler = SanitizerHandler::NullabilityReturn;
4241 }
4242
4243 SanitizerDebugLocation SanScope(this, {CheckKind}, Handler);
4244
4245 // Make sure the "return" source location is valid. If we're checking a
4246 // nullability annotation, make sure the preconditions for the check are met.
4247 llvm::BasicBlock *Check = createBasicBlock("nullcheck");
4248 llvm::BasicBlock *NoCheck = createBasicBlock("no.nullcheck");
4249 llvm::Value *SLocPtr = Builder.CreateLoad(ReturnLocation, "return.sloc.load");
4250 llvm::Value *CanNullCheck = Builder.CreateIsNotNull(SLocPtr);
4251 if (requiresReturnValueNullabilityCheck())
4252 CanNullCheck =
4253 Builder.CreateAnd(CanNullCheck, RetValNullabilityPrecondition);
4254 Builder.CreateCondBr(CanNullCheck, Check, NoCheck);
4255 EmitBlock(Check);
4256
4257 // Now do the null check.
4258 llvm::Value *Cond = Builder.CreateIsNotNull(RV);
4259 llvm::Constant *StaticData[] = {EmitCheckSourceLocation(AttrLoc)};
4260 llvm::Value *DynamicData[] = {SLocPtr};
4261 EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, DynamicData);
4262
4263 EmitBlock(NoCheck);
4264
4265#ifndef NDEBUG
4266 // The return location should not be used after the check has been emitted.
4267 ReturnLocation = Address::invalid();
4268#endif
4269}
4270
4272 const CXXRecordDecl *RD = type->getAsCXXRecordDecl();
4273 return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory;
4274}
4275
4277 // FIXME: Generate IR in one pass, rather than going back and fixing up these
4278 // placeholders.
4279 llvm::Type *IRTy = CGF.ConvertTypeForMem(Ty);
4280 llvm::Type *IRPtrTy = llvm::PointerType::getUnqual(CGF.getLLVMContext());
4281 llvm::Value *Placeholder = llvm::PoisonValue::get(IRPtrTy);
4282
4283 // FIXME: When we generate this IR in one pass, we shouldn't need
4284 // this win32-specific alignment hack.
4286 Placeholder = CGF.Builder.CreateAlignedLoad(IRPtrTy, Placeholder, Align);
4287
4288 return AggValueSlot::forAddr(
4289 Address(Placeholder, IRTy, Align), Ty.getQualifiers(),
4292}
4293
4295 const VarDecl *param,
4296 SourceLocation loc) {
4297 // StartFunction converted the ABI-lowered parameter(s) into a
4298 // local alloca. We need to turn that into an r-value suitable
4299 // for EmitCall.
4300 Address local = GetAddrOfLocalVar(param);
4301
4302 QualType type = param->getType();
4303
4304 // GetAddrOfLocalVar returns a pointer-to-pointer for references,
4305 // but the argument needs to be the original pointer.
4306 if (type->isReferenceType()) {
4307 args.add(RValue::get(Builder.CreateLoad(local)), type);
4308
4309 // In ARC, move out of consumed arguments so that the release cleanup
4310 // entered by StartFunction doesn't cause an over-release. This isn't
4311 // optimal -O0 code generation, but it should get cleaned up when
4312 // optimization is enabled. This also assumes that delegate calls are
4313 // performed exactly once for a set of arguments, but that should be safe.
4314 } else if (getLangOpts().ObjCAutoRefCount &&
4315 param->hasAttr<NSConsumedAttr>() && type->isObjCRetainableType()) {
4316 llvm::Value *ptr = Builder.CreateLoad(local);
4317 auto null =
4318 llvm::ConstantPointerNull::get(cast<llvm::PointerType>(ptr->getType()));
4319 Builder.CreateStore(null, local);
4320 args.add(RValue::get(ptr), type);
4321
4322 // For the most part, we just need to load the alloca, except that
4323 // aggregate r-values are actually pointers to temporaries.
4324 } else {
4325 args.add(convertTempToRValue(local, type, loc), type);
4326 }
4327
4328 // Deactivate the cleanup for the callee-destructed param that was pushed.
4329 if (type->isRecordType() && !CurFuncIsThunk &&
4330 type->castAsRecordDecl()->isParamDestroyedInCallee() &&
4331 param->needsDestruction(getContext())) {
4333 CalleeDestructedParamCleanups.lookup(cast<ParmVarDecl>(param));
4334 assert(cleanup.isValid() &&
4335 "cleanup for callee-destructed param not recorded");
4336 // This unreachable is a temporary marker which will be removed later.
4337 llvm::Instruction *isActive = Builder.CreateUnreachable();
4338 args.addArgCleanupDeactivation(cleanup, isActive);
4339 }
4340}
4341
4342static bool isProvablyNull(llvm::Value *addr) {
4343 return llvm::isa_and_nonnull<llvm::ConstantPointerNull>(addr);
4344}
4345
4347 return llvm::isKnownNonZero(Addr.getBasePointer(), CGF.CGM.getDataLayout());
4348}
4349
4350/// Emit the actual writing-back of a writeback.
4352 const CallArgList::Writeback &writeback) {
4353 const LValue &srcLV = writeback.Source;
4354 Address srcAddr = srcLV.getAddress();
4355 assert(!isProvablyNull(srcAddr.getBasePointer()) &&
4356 "shouldn't have writeback for provably null argument");
4357
4358 if (writeback.WritebackExpr) {
4359 CGF.EmitIgnoredExpr(writeback.WritebackExpr);
4360 CGF.EmitLifetimeEnd(writeback.Temporary.getBasePointer());
4361 return;
4362 }
4363
4364 llvm::BasicBlock *contBB = nullptr;
4365
4366 // If the argument wasn't provably non-null, we need to null check
4367 // before doing the store.
4368 bool provablyNonNull = isProvablyNonNull(srcAddr, CGF);
4369
4370 if (!provablyNonNull) {
4371 llvm::BasicBlock *writebackBB = CGF.createBasicBlock("icr.writeback");
4372 contBB = CGF.createBasicBlock("icr.done");
4373
4374 llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull");
4375 CGF.Builder.CreateCondBr(isNull, contBB, writebackBB);
4376 CGF.EmitBlock(writebackBB);
4377 }
4378
4379 // Load the value to writeback.
4380 llvm::Value *value = CGF.Builder.CreateLoad(writeback.Temporary);
4381
4382 // Cast it back, in case we're writing an id to a Foo* or something.
4383 value = CGF.Builder.CreateBitCast(value, srcAddr.getElementType(),
4384 "icr.writeback-cast");
4385
4386 // Perform the writeback.
4387
4388 // If we have a "to use" value, it's something we need to emit a use
4389 // of. This has to be carefully threaded in: if it's done after the
4390 // release it's potentially undefined behavior (and the optimizer
4391 // will ignore it), and if it happens before the retain then the
4392 // optimizer could move the release there.
4393 if (writeback.ToUse) {
4394 assert(srcLV.getObjCLifetime() == Qualifiers::OCL_Strong);
4395
4396 // Retain the new value. No need to block-copy here: the block's
4397 // being passed up the stack.
4398 value = CGF.EmitARCRetainNonBlock(value);
4399
4400 // Emit the intrinsic use here.
4401 CGF.EmitARCIntrinsicUse(writeback.ToUse);
4402
4403 // Load the old value (primitively).
4404 llvm::Value *oldValue = CGF.EmitLoadOfScalar(srcLV, SourceLocation());
4405
4406 // Put the new value in place (primitively).
4407 CGF.EmitStoreOfScalar(value, srcLV, /*init*/ false);
4408
4409 // Release the old value.
4410 CGF.EmitARCRelease(oldValue, srcLV.isARCPreciseLifetime());
4411
4412 // Otherwise, we can just do a normal lvalue store.
4413 } else {
4414 CGF.EmitStoreThroughLValue(RValue::get(value), srcLV);
4415 }
4416
4417 // Jump to the continuation block.
4418 if (!provablyNonNull)
4419 CGF.EmitBlock(contBB);
4420}
4421
4423 const CallArgList &CallArgs) {
4425 CallArgs.getCleanupsToDeactivate();
4426 // Iterate in reverse to increase the likelihood of popping the cleanup.
4427 for (const auto &I : llvm::reverse(Cleanups)) {
4428 CGF.DeactivateCleanupBlock(I.Cleanup, I.IsActiveIP);
4429 I.IsActiveIP->eraseFromParent();
4430 }
4431}
4432
4433static const Expr *maybeGetUnaryAddrOfOperand(const Expr *E) {
4434 if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(E->IgnoreParens()))
4435 if (uop->getOpcode() == UO_AddrOf)
4436 return uop->getSubExpr();
4437 return nullptr;
4438}
4439
4440/// Emit an argument that's being passed call-by-writeback. That is,
4441/// we are passing the address of an __autoreleased temporary; it
4442/// might be copy-initialized with the current value of the given
4443/// address, but it will definitely be copied out of after the call.
4445 const ObjCIndirectCopyRestoreExpr *CRE) {
4446 LValue srcLV;
4447
4448 // Make an optimistic effort to emit the address as an l-value.
4449 // This can fail if the argument expression is more complicated.
4450 if (const Expr *lvExpr = maybeGetUnaryAddrOfOperand(CRE->getSubExpr())) {
4451 srcLV = CGF.EmitLValue(lvExpr);
4452
4453 // Otherwise, just emit it as a scalar.
4454 } else {
4455 Address srcAddr = CGF.EmitPointerWithAlignment(CRE->getSubExpr());
4456
4457 QualType srcAddrType =
4459 srcLV = CGF.MakeAddrLValue(srcAddr, srcAddrType);
4460 }
4461 Address srcAddr = srcLV.getAddress();
4462
4463 // The dest and src types don't necessarily match in LLVM terms
4464 // because of the crazy ObjC compatibility rules.
4465
4466 llvm::PointerType *destType =
4468 llvm::Type *destElemType =
4470
4471 // If the address is a constant null, just pass the appropriate null.
4472 if (isProvablyNull(srcAddr.getBasePointer())) {
4473 args.add(RValue::get(llvm::ConstantPointerNull::get(destType)),
4474 CRE->getType());
4475 return;
4476 }
4477
4478 // Create the temporary.
4479 Address temp =
4480 CGF.CreateTempAlloca(destElemType, CGF.getPointerAlign(), "icr.temp");
4481 // Loading an l-value can introduce a cleanup if the l-value is __weak,
4482 // and that cleanup will be conditional if we can't prove that the l-value
4483 // isn't null, so we need to register a dominating point so that the cleanups
4484 // system will make valid IR.
4486
4487 // Zero-initialize it if we're not doing a copy-initialization.
4488 bool shouldCopy = CRE->shouldCopy();
4489 if (!shouldCopy) {
4490 llvm::Value *null =
4491 llvm::ConstantPointerNull::get(cast<llvm::PointerType>(destElemType));
4492 CGF.Builder.CreateStore(null, temp);
4493 }
4494
4495 llvm::BasicBlock *contBB = nullptr;
4496 llvm::BasicBlock *originBB = nullptr;
4497
4498 // If the address is *not* known to be non-null, we need to switch.
4499 llvm::Value *finalArgument;
4500
4501 bool provablyNonNull = isProvablyNonNull(srcAddr, CGF);
4502
4503 if (provablyNonNull) {
4504 finalArgument = temp.emitRawPointer(CGF);
4505 } else {
4506 llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull");
4507
4508 finalArgument = CGF.Builder.CreateSelect(
4509 isNull, llvm::ConstantPointerNull::get(destType),
4510 temp.emitRawPointer(CGF), "icr.argument");
4511
4512 // If we need to copy, then the load has to be conditional, which
4513 // means we need control flow.
4514 if (shouldCopy) {
4515 originBB = CGF.Builder.GetInsertBlock();
4516 contBB = CGF.createBasicBlock("icr.cont");
4517 llvm::BasicBlock *copyBB = CGF.createBasicBlock("icr.copy");
4518 CGF.Builder.CreateCondBr(isNull, contBB, copyBB);
4519 CGF.EmitBlock(copyBB);
4520 condEval.begin(CGF);
4521 }
4522 }
4523
4524 llvm::Value *valueToUse = nullptr;
4525
4526 // Perform a copy if necessary.
4527 if (shouldCopy) {
4528 RValue srcRV = CGF.EmitLoadOfLValue(srcLV, SourceLocation());
4529 assert(srcRV.isScalar());
4530
4531 llvm::Value *src = srcRV.getScalarVal();
4532 src = CGF.Builder.CreateBitCast(src, destElemType, "icr.cast");
4533
4534 // Use an ordinary store, not a store-to-lvalue.
4535 CGF.Builder.CreateStore(src, temp);
4536
4537 // If optimization is enabled, and the value was held in a
4538 // __strong variable, we need to tell the optimizer that this
4539 // value has to stay alive until we're doing the store back.
4540 // This is because the temporary is effectively unretained,
4541 // and so otherwise we can violate the high-level semantics.
4542 if (CGF.CGM.getCodeGenOpts().OptimizationLevel != 0 &&
4543 srcLV.getObjCLifetime() == Qualifiers::OCL_Strong) {
4544 valueToUse = src;
4545 }
4546 }
4547
4548 // Finish the control flow if we needed it.
4549 if (shouldCopy && !provablyNonNull) {
4550 llvm::BasicBlock *copyBB = CGF.Builder.GetInsertBlock();
4551 CGF.EmitBlock(contBB);
4552
4553 // Make a phi for the value to intrinsically use.
4554 if (valueToUse) {
4555 llvm::PHINode *phiToUse =
4556 CGF.Builder.CreatePHI(valueToUse->getType(), 2, "icr.to-use");
4557 phiToUse->addIncoming(valueToUse, copyBB);
4558 phiToUse->addIncoming(llvm::PoisonValue::get(valueToUse->getType()),
4559 originBB);
4560 valueToUse = phiToUse;
4561 }
4562
4563 condEval.end(CGF);
4564 }
4565
4566 args.addWriteback(srcLV, temp, valueToUse);
4567 args.add(RValue::get(finalArgument), CRE->getType());
4568}
4569
4571 assert(!StackBase);
4572
4573 // Save the stack.
4574 StackBase = CGF.Builder.CreateStackSave("inalloca.save");
4575}
4576
4578 if (StackBase) {
4579 // Restore the stack after the call.
4580 CGF.Builder.CreateStackRestore(StackBase);
4581 }
4582}
4583
4585 SourceLocation ArgLoc,
4586 AbstractCallee AC, unsigned ParmNum) {
4587 if (!AC.getDecl() || !(SanOpts.has(SanitizerKind::NonnullAttribute) ||
4588 SanOpts.has(SanitizerKind::NullabilityArg)))
4589 return;
4590
4591 // The param decl may be missing in a variadic function.
4592 auto PVD = ParmNum < AC.getNumParams() ? AC.getParamDecl(ParmNum) : nullptr;
4593 unsigned ArgNo = PVD ? PVD->getFunctionScopeIndex() : ParmNum;
4594
4595 // Prefer the nonnull attribute if it's present.
4596 const NonNullAttr *NNAttr = nullptr;
4597 if (SanOpts.has(SanitizerKind::NonnullAttribute))
4598 NNAttr = getNonNullAttr(AC.getDecl(), PVD, ArgType, ArgNo);
4599
4600 bool CanCheckNullability = false;
4601 if (SanOpts.has(SanitizerKind::NullabilityArg) && !NNAttr && PVD &&
4602 !PVD->getType()->isRecordType()) {
4603 auto Nullability = PVD->getType()->getNullability();
4604 CanCheckNullability = Nullability &&
4605 *Nullability == NullabilityKind::NonNull &&
4606 PVD->getTypeSourceInfo();
4607 }
4608
4609 if (!NNAttr && !CanCheckNullability)
4610 return;
4611
4612 SourceLocation AttrLoc;
4614 SanitizerHandler Handler;
4615 if (NNAttr) {
4616 AttrLoc = NNAttr->getLocation();
4617 CheckKind = SanitizerKind::SO_NonnullAttribute;
4618 Handler = SanitizerHandler::NonnullArg;
4619 } else {
4620 AttrLoc = PVD->getTypeSourceInfo()->getTypeLoc().findNullabilityLoc();
4621 CheckKind = SanitizerKind::SO_NullabilityArg;
4622 Handler = SanitizerHandler::NullabilityArg;
4623 }
4624
4625 SanitizerDebugLocation SanScope(this, {CheckKind}, Handler);
4626 llvm::Value *Cond = EmitNonNullRValueCheck(RV, ArgType);
4627 llvm::Constant *StaticData[] = {
4629 EmitCheckSourceLocation(AttrLoc),
4630 llvm::ConstantInt::get(Int32Ty, ArgNo + 1),
4631 };
4632 EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, {});
4633}
4634
4636 SourceLocation ArgLoc,
4637 AbstractCallee AC, unsigned ParmNum) {
4638 if (!AC.getDecl() || !(SanOpts.has(SanitizerKind::NonnullAttribute) ||
4639 SanOpts.has(SanitizerKind::NullabilityArg)))
4640 return;
4641
4642 EmitNonNullArgCheck(RValue::get(Addr, *this), ArgType, ArgLoc, AC, ParmNum);
4643}
4644
4645// Check if the call is going to use the inalloca convention. This needs to
4646// agree with CGFunctionInfo::usesInAlloca. The CGFunctionInfo is arranged
4647// later, so we can't check it directly.
4648static bool hasInAllocaArgs(CodeGenModule &CGM, CallingConv ExplicitCC,
4649 ArrayRef<QualType> ArgTypes) {
4650 // The Swift calling conventions don't go through the target-specific
4651 // argument classification, they never use inalloca.
4652 // TODO: Consider limiting inalloca use to only calling conventions supported
4653 // by MSVC.
4654 if (ExplicitCC == CC_Swift || ExplicitCC == CC_SwiftAsync)
4655 return false;
4656 if (!CGM.getTarget().getCXXABI().isMicrosoft())
4657 return false;
4658 return llvm::any_of(ArgTypes, [&](QualType Ty) {
4659 return isInAllocaArgument(CGM.getCXXABI(), Ty);
4660 });
4661}
4662
4663#ifndef NDEBUG
4664// Determine whether the given argument is an Objective-C method
4665// that may have type parameters in its signature.
4666static bool isObjCMethodWithTypeParams(const ObjCMethodDecl *method) {
4667 const DeclContext *dc = method->getDeclContext();
4668 if (const ObjCInterfaceDecl *classDecl = dyn_cast<ObjCInterfaceDecl>(dc)) {
4669 return classDecl->getTypeParamListAsWritten();
4670 }
4671
4672 if (const ObjCCategoryDecl *catDecl = dyn_cast<ObjCCategoryDecl>(dc)) {
4673 return catDecl->getTypeParamList();
4674 }
4675
4676 return false;
4677}
4678#endif
4679
4680/// EmitCallArgs - Emit call arguments for a function.
4683 llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
4684 AbstractCallee AC, unsigned ParamsToSkip, EvaluationOrder Order) {
4686
4687 assert((ParamsToSkip == 0 || Prototype.P) &&
4688 "Can't skip parameters if type info is not provided");
4689
4690 // This variable only captures *explicitly* written conventions, not those
4691 // applied by default via command line flags or target defaults, such as
4692 // thiscall, aapcs, stdcall via -mrtd, etc. Computing that correctly would
4693 // require knowing if this is a C++ instance method or being able to see
4694 // unprototyped FunctionTypes.
4695 CallingConv ExplicitCC = CC_C;
4696
4697 // First, if a prototype was provided, use those argument types.
4698 bool IsVariadic = false;
4699 if (Prototype.P) {
4700 const auto *MD = dyn_cast<const ObjCMethodDecl *>(Prototype.P);
4701 if (MD) {
4702 IsVariadic = MD->isVariadic();
4703 ExplicitCC = getCallingConventionForDecl(
4704 MD, CGM.getTarget().getTriple().isOSWindows());
4705 ArgTypes.assign(MD->param_type_begin() + ParamsToSkip,
4706 MD->param_type_end());
4707 } else {
4708 const auto *FPT = cast<const FunctionProtoType *>(Prototype.P);
4709 IsVariadic = FPT->isVariadic();
4710 ExplicitCC = FPT->getExtInfo().getCC();
4711 ArgTypes.assign(FPT->param_type_begin() + ParamsToSkip,
4712 FPT->param_type_end());
4713 }
4714
4715#ifndef NDEBUG
4716 // Check that the prototyped types match the argument expression types.
4717 bool isGenericMethod = MD && isObjCMethodWithTypeParams(MD);
4718 CallExpr::const_arg_iterator Arg = ArgRange.begin();
4719 for (QualType Ty : ArgTypes) {
4720 assert(Arg != ArgRange.end() && "Running over edge of argument list!");
4721 assert(
4722 (isGenericMethod || Ty->isVariablyModifiedType() ||
4724 getContext()
4725 .getCanonicalType(Ty.getNonReferenceType())
4726 .getTypePtr() ==
4727 getContext().getCanonicalType((*Arg)->getType()).getTypePtr()) &&
4728 "type mismatch in call argument!");
4729 ++Arg;
4730 }
4731
4732 // Either we've emitted all the call args, or we have a call to variadic
4733 // function.
4734 assert((Arg == ArgRange.end() || IsVariadic) &&
4735 "Extra arguments in non-variadic function!");
4736#endif
4737 }
4738
4739 // If we still have any arguments, emit them using the type of the argument.
4740 for (auto *A : llvm::drop_begin(ArgRange, ArgTypes.size()))
4741 ArgTypes.push_back(IsVariadic ? getVarArgType(A) : A->getType());
4742 assert((int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin()));
4743
4744 // We must evaluate arguments from right to left in the MS C++ ABI,
4745 // because arguments are destroyed left to right in the callee. As a special
4746 // case, there are certain language constructs that require left-to-right
4747 // evaluation, and in those cases we consider the evaluation order requirement
4748 // to trump the "destruction order is reverse construction order" guarantee.
4749 bool LeftToRight =
4750 CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()
4753
4754 auto MaybeEmitImplicitObjectSize = [&](unsigned I, const Expr *Arg,
4755 RValue EmittedArg) {
4756 if (!AC.hasFunctionDecl() || I >= AC.getNumParams())
4757 return;
4758 auto *PS = AC.getParamDecl(I)->getAttr<PassObjectSizeAttr>();
4759 if (PS == nullptr)
4760 return;
4761
4762 const auto &Context = getContext();
4763 auto SizeTy = Context.getSizeType();
4764 auto T = Builder.getIntNTy(Context.getTypeSize(SizeTy));
4765 assert(EmittedArg.getScalarVal() && "We emitted nothing for the arg?");
4766 llvm::Value *V = evaluateOrEmitBuiltinObjectSize(
4767 Arg, PS->getType(), T, EmittedArg.getScalarVal(), PS->isDynamic());
4768 Args.add(RValue::get(V), SizeTy);
4769 // If we're emitting args in reverse, be sure to do so with
4770 // pass_object_size, as well.
4771 if (!LeftToRight)
4772 std::swap(Args.back(), *(&Args.back() - 1));
4773 };
4774
4775 // Insert a stack save if we're going to need any inalloca args.
4776 if (hasInAllocaArgs(CGM, ExplicitCC, ArgTypes)) {
4777 assert(getTarget().getTriple().getArch() == llvm::Triple::x86 &&
4778 "inalloca only supported on x86");
4779 Args.allocateArgumentMemory(*this);
4780 }
4781
4782 // Evaluate each argument in the appropriate order.
4783 size_t CallArgsStart = Args.size();
4784 for (unsigned I = 0, E = ArgTypes.size(); I != E; ++I) {
4785 unsigned Idx = LeftToRight ? I : E - I - 1;
4786 CallExpr::const_arg_iterator Arg = ArgRange.begin() + Idx;
4787 unsigned InitialArgSize = Args.size();
4788 // If *Arg is an ObjCIndirectCopyRestoreExpr, check that either the types of
4789 // the argument and parameter match or the objc method is parameterized.
4790 assert((!isa<ObjCIndirectCopyRestoreExpr>(*Arg) ||
4791 getContext().hasSameUnqualifiedType((*Arg)->getType(),
4792 ArgTypes[Idx]) ||
4795 "Argument and parameter types don't match");
4796 EmitCallArg(Args, *Arg, ArgTypes[Idx]);
4797 // In particular, we depend on it being the last arg in Args, and the
4798 // objectsize bits depend on there only being one arg if !LeftToRight.
4799 assert(InitialArgSize + 1 == Args.size() &&
4800 "The code below depends on only adding one arg per EmitCallArg");
4801 (void)InitialArgSize;
4802 // Since pointer argument are never emitted as LValue, it is safe to emit
4803 // non-null argument check for r-value only.
4804 if (!Args.back().hasLValue()) {
4805 RValue RVArg = Args.back().getKnownRValue();
4806 EmitNonNullArgCheck(RVArg, ArgTypes[Idx], (*Arg)->getExprLoc(), AC,
4807 ParamsToSkip + Idx);
4808 // @llvm.objectsize should never have side-effects and shouldn't need
4809 // destruction/cleanups, so we can safely "emit" it after its arg,
4810 // regardless of right-to-leftness
4811 MaybeEmitImplicitObjectSize(Idx, *Arg, RVArg);
4812 }
4813 }
4814
4815 if (!LeftToRight) {
4816 // Un-reverse the arguments we just evaluated so they match up with the LLVM
4817 // IR function.
4818 std::reverse(Args.begin() + CallArgsStart, Args.end());
4819
4820 // Reverse the writebacks to match the MSVC ABI.
4821 Args.reverseWritebacks();
4822 }
4823}
4824
4825namespace {
4826
4827struct DestroyUnpassedArg final : EHScopeStack::Cleanup {
4828 DestroyUnpassedArg(Address Addr, QualType Ty) : Addr(Addr), Ty(Ty) {}
4829
4830 Address Addr;
4831 QualType Ty;
4832
4833 void Emit(CodeGenFunction &CGF, Flags flags) override {
4835 if (DtorKind == QualType::DK_cxx_destructor) {
4836 const CXXDestructorDecl *Dtor = Ty->getAsCXXRecordDecl()->getDestructor();
4837 assert(!Dtor->isTrivial());
4838 CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete, /*for vbase*/ false,
4839 /*Delegating=*/false, Addr, Ty);
4840 } else {
4842 }
4843 }
4844};
4845
4846} // end anonymous namespace
4847
4849 if (!HasLV)
4850 return RV;
4853 LV.isVolatile());
4854 IsUsed = true;
4855 return RValue::getAggregate(Copy.getAddress());
4856}
4857
4859 LValue Dst = CGF.MakeAddrLValue(Addr, Ty);
4860 if (!HasLV && RV.isScalar())
4861 CGF.EmitStoreOfScalar(RV.getScalarVal(), Dst, /*isInit=*/true);
4862 else if (!HasLV && RV.isComplex())
4863 CGF.EmitStoreOfComplex(RV.getComplexVal(), Dst, /*init=*/true);
4864 else {
4865 auto Addr = HasLV ? LV.getAddress() : RV.getAggregateAddress();
4866 LValue SrcLV = CGF.MakeAddrLValue(Addr, Ty);
4867 // We assume that call args are never copied into subobjects.
4869 HasLV ? LV.isVolatileQualified()
4870 : RV.isVolatileQualified());
4871 }
4872 IsUsed = true;
4873}
4874
4876 for (const auto &I : args.writebacks())
4877 emitWriteback(*this, I);
4878}
4879
4881 QualType type) {
4882 std::optional<DisableDebugLocationUpdates> Dis;
4884 Dis.emplace(*this);
4885 if (const ObjCIndirectCopyRestoreExpr *CRE =
4886 dyn_cast<ObjCIndirectCopyRestoreExpr>(E)) {
4887 assert(getLangOpts().ObjCAutoRefCount);
4888 return emitWritebackArg(*this, args, CRE);
4889 }
4890
4891 // Add writeback for HLSLOutParamExpr.
4892 // Needs to be before the assert below because HLSLOutArgExpr is an LValue
4893 // and is not a reference.
4894 if (const HLSLOutArgExpr *OE = dyn_cast<HLSLOutArgExpr>(E)) {
4895 EmitHLSLOutArgExpr(OE, args, type);
4896 return;
4897 }
4898
4899 assert(type->isReferenceType() == E->isGLValue() &&
4900 "reference binding to unmaterialized r-value!");
4901
4902 if (E->isGLValue()) {
4903 assert(E->getObjectKind() == OK_Ordinary);
4904 return args.add(EmitReferenceBindingToExpr(E), type);
4905 }
4906
4907 bool HasAggregateEvalKind = hasAggregateEvaluationKind(type);
4908
4909 // In the Microsoft C++ ABI, aggregate arguments are destructed by the callee.
4910 // However, we still have to push an EH-only cleanup in case we unwind before
4911 // we make it to the call.
4912 if (type->isRecordType() &&
4913 type->castAsRecordDecl()->isParamDestroyedInCallee()) {
4914 // If we're using inalloca, use the argument memory. Otherwise, use a
4915 // temporary.
4916 AggValueSlot Slot = args.isUsingInAlloca()
4917 ? createPlaceholderSlot(*this, type)
4918 : CreateAggTemp(type, "agg.tmp");
4919
4920 bool DestroyedInCallee = true, NeedsCleanup = true;
4921 if (const auto *RD = type->getAsCXXRecordDecl())
4922 DestroyedInCallee = RD->hasNonTrivialDestructor();
4923 else
4924 NeedsCleanup = type.isDestructedType();
4925
4926 if (DestroyedInCallee)
4928
4929 EmitAggExpr(E, Slot);
4930 RValue RV = Slot.asRValue();
4931 args.add(RV, type);
4932
4933 if (DestroyedInCallee && NeedsCleanup) {
4934 // Create a no-op GEP between the placeholder and the cleanup so we can
4935 // RAUW it successfully. It also serves as a marker of the first
4936 // instruction where the cleanup is active.
4938 Slot.getAddress(), type);
4939 // This unreachable is a temporary marker which will be removed later.
4940 llvm::Instruction *IsActive =
4941 Builder.CreateFlagLoad(llvm::Constant::getNullValue(Int8PtrTy));
4942 args.addArgCleanupDeactivation(EHStack.stable_begin(), IsActive);
4943 }
4944 return;
4945 }
4946
4947 if (HasAggregateEvalKind && isa<ImplicitCastExpr>(E) &&
4948 cast<CastExpr>(E)->getCastKind() == CK_LValueToRValue &&
4949 !type->isArrayParameterType() && !type.isNonTrivialToPrimitiveCopy()) {
4950 LValue L = EmitLValue(cast<CastExpr>(E)->getSubExpr());
4951 assert(L.isSimple());
4952 args.addUncopiedAggregate(L, type);
4953 return;
4954 }
4955
4956 args.add(EmitAnyExprToTemp(E), type);
4957}
4958
4959QualType CodeGenFunction::getVarArgType(const Expr *Arg) {
4960 // System headers on Windows define NULL to 0 instead of 0LL on Win64. MSVC
4961 // implicitly widens null pointer constants that are arguments to varargs
4962 // functions to pointer-sized ints.
4963 if (!getTarget().getTriple().isOSWindows())
4964 return Arg->getType();
4965
4966 if (Arg->getType()->isIntegerType() &&
4967 getContext().getTypeSize(Arg->getType()) <
4968 getContext().getTargetInfo().getPointerWidth(LangAS::Default) &&
4969 Arg->isNullPointerConstant(getContext(),
4971 return getContext().getIntPtrType();
4972 }
4973
4974 return Arg->getType();
4975}
4976
4977// In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
4978// optimizer it can aggressively ignore unwind edges.
4979void CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) {
4980 if (CGM.getCodeGenOpts().OptimizationLevel != 0 &&
4981 !CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)
4982 Inst->setMetadata("clang.arc.no_objc_arc_exceptions",
4983 CGM.getNoObjCARCExceptionsMetadata());
4984}
4985
4986/// Emits a call to the given no-arguments nounwind runtime function.
4987llvm::CallInst *
4988CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4989 const llvm::Twine &name) {
4990 return EmitNounwindRuntimeCall(callee, ArrayRef<llvm::Value *>(), name);
4991}
4992
4993/// Emits a call to the given nounwind runtime function.
4994llvm::CallInst *
4995CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4996 ArrayRef<Address> args,
4997 const llvm::Twine &name) {
4998 SmallVector<llvm::Value *, 3> values;
4999 for (auto arg : args)
5000 values.push_back(arg.emitRawPointer(*this));
5001 return EmitNounwindRuntimeCall(callee, values, name);
5002}
5003
5004llvm::CallInst *
5005CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
5006 ArrayRef<llvm::Value *> args,
5007 const llvm::Twine &name) {
5008 llvm::CallInst *call = EmitRuntimeCall(callee, args, name);
5009 call->setDoesNotThrow();
5010 return call;
5011}
5012
5013/// Emits a simple call (never an invoke) to the given no-arguments
5014/// runtime function.
5015llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
5016 const llvm::Twine &name) {
5017 return EmitRuntimeCall(callee, {}, name);
5018}
5019
5020// Calls which may throw must have operand bundles indicating which funclet
5021// they are nested within.
5022SmallVector<llvm::OperandBundleDef, 1>
5024 // There is no need for a funclet operand bundle if we aren't inside a
5025 // funclet.
5026 if (!CurrentFuncletPad)
5028
5029 // Skip intrinsics which cannot throw (as long as they don't lower into
5030 // regular function calls in the course of IR transformations).
5031 if (auto *CalleeFn = dyn_cast<llvm::Function>(Callee->stripPointerCasts())) {
5032 if (CalleeFn->isIntrinsic() && CalleeFn->doesNotThrow()) {
5033 auto IID = CalleeFn->getIntrinsicID();
5034 if (!llvm::IntrinsicInst::mayLowerToFunctionCall(IID))
5036 }
5037 }
5038
5040 BundleList.emplace_back("funclet", CurrentFuncletPad);
5041 return BundleList;
5042}
5043
5044/// Emits a simple call (never an invoke) to the given runtime function.
5045llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
5047 const llvm::Twine &name) {
5048 llvm::CallInst *call = Builder.CreateCall(
5049 callee, args, getBundlesForFunclet(callee.getCallee()), name);
5050 call->setCallingConv(getRuntimeCC());
5051
5052 if (CGM.shouldEmitConvergenceTokens() && call->isConvergent())
5053 return cast<llvm::CallInst>(addConvergenceControlToken(call));
5054 return call;
5055}
5056
5057/// Emits a call or invoke to the given noreturn runtime function.
5059 llvm::FunctionCallee callee, ArrayRef<llvm::Value *> args) {
5061 getBundlesForFunclet(callee.getCallee());
5062
5063 if (getInvokeDest()) {
5064 llvm::InvokeInst *invoke = Builder.CreateInvoke(
5065 callee, getUnreachableBlock(), getInvokeDest(), args, BundleList);
5066 invoke->setDoesNotReturn();
5067 invoke->setCallingConv(getRuntimeCC());
5068 } else {
5069 llvm::CallInst *call = Builder.CreateCall(callee, args, BundleList);
5070 call->setDoesNotReturn();
5071 call->setCallingConv(getRuntimeCC());
5072 Builder.CreateUnreachable();
5073 }
5074}
5075
5076/// Emits a call or invoke instruction to the given nullary runtime function.
5077llvm::CallBase *
5079 const Twine &name) {
5080 return EmitRuntimeCallOrInvoke(callee, {}, name);
5081}
5082
5083/// Emits a call or invoke instruction to the given runtime function.
5084llvm::CallBase *
5087 const Twine &name) {
5088 llvm::CallBase *call = EmitCallOrInvoke(callee, args, name);
5089 call->setCallingConv(getRuntimeCC());
5090 return call;
5091}
5092
5093/// Emits a call or invoke instruction to the given function, depending
5094/// on the current state of the EH stack.
5095llvm::CallBase *CodeGenFunction::EmitCallOrInvoke(llvm::FunctionCallee Callee,
5097 const Twine &Name) {
5098 llvm::BasicBlock *InvokeDest = getInvokeDest();
5100 getBundlesForFunclet(Callee.getCallee());
5101
5102 llvm::CallBase *Inst;
5103 if (!InvokeDest)
5104 Inst = Builder.CreateCall(Callee, Args, BundleList, Name);
5105 else {
5106 llvm::BasicBlock *ContBB = createBasicBlock("invoke.cont");
5107 Inst = Builder.CreateInvoke(Callee, ContBB, InvokeDest, Args, BundleList,
5108 Name);
5109 EmitBlock(ContBB);
5110 }
5111
5112 // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
5113 // optimizer it can aggressively ignore unwind edges.
5114 if (CGM.getLangOpts().ObjCAutoRefCount)
5115 AddObjCARCExceptionMetadata(Inst);
5116
5117 return Inst;
5118}
5119
5120void CodeGenFunction::deferPlaceholderReplacement(llvm::Instruction *Old,
5121 llvm::Value *New) {
5122 DeferredReplacements.push_back(
5123 std::make_pair(llvm::WeakTrackingVH(Old), New));
5124}
5125
5126namespace {
5127
5128/// Specify given \p NewAlign as the alignment of return value attribute. If
5129/// such attribute already exists, re-set it to the maximal one of two options.
5130[[nodiscard]] llvm::AttributeList
5131maybeRaiseRetAlignmentAttribute(llvm::LLVMContext &Ctx,
5132 const llvm::AttributeList &Attrs,
5133 llvm::Align NewAlign) {
5134 llvm::Align CurAlign = Attrs.getRetAlignment().valueOrOne();
5135 if (CurAlign >= NewAlign)
5136 return Attrs;
5137 llvm::Attribute AlignAttr = llvm::Attribute::getWithAlignment(Ctx, NewAlign);
5138 return Attrs.removeRetAttribute(Ctx, llvm::Attribute::AttrKind::Alignment)
5139 .addRetAttribute(Ctx, AlignAttr);
5140}
5141
5142template <typename AlignedAttrTy> class AbstractAssumeAlignedAttrEmitter {
5143protected:
5144 CodeGenFunction &CGF;
5145
5146 /// We do nothing if this is, or becomes, nullptr.
5147 const AlignedAttrTy *AA = nullptr;
5148
5149 llvm::Value *Alignment = nullptr; // May or may not be a constant.
5150 llvm::ConstantInt *OffsetCI = nullptr; // Constant, hopefully zero.
5151
5152 AbstractAssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
5153 : CGF(CGF_) {
5154 if (!FuncDecl)
5155 return;
5156 AA = FuncDecl->getAttr<AlignedAttrTy>();
5157 }
5158
5159public:
5160 /// If we can, materialize the alignment as an attribute on return value.
5161 [[nodiscard]] llvm::AttributeList
5162 TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
5163 if (!AA || OffsetCI || CGF.SanOpts.has(SanitizerKind::Alignment))
5164 return Attrs;
5165 const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Alignment);
5166 if (!AlignmentCI)
5167 return Attrs;
5168 // We may legitimately have non-power-of-2 alignment here.
5169 // If so, this is UB land, emit it via `@llvm.assume` instead.
5170 if (!AlignmentCI->getValue().isPowerOf2())
5171 return Attrs;
5172 llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
5173 CGF.getLLVMContext(), Attrs,
5174 llvm::Align(
5175 AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment)));
5176 AA = nullptr; // We're done. Disallow doing anything else.
5177 return NewAttrs;
5178 }
5179
5180 /// Emit alignment assumption.
5181 /// This is a general fallback that we take if either there is an offset,
5182 /// or the alignment is variable or we are sanitizing for alignment.
5183 void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
5184 if (!AA)
5185 return;
5186 CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
5187 AA->getLocation(), Alignment, OffsetCI);
5188 AA = nullptr; // We're done. Disallow doing anything else.
5189 }
5190};
5191
5192/// Helper data structure to emit `AssumeAlignedAttr`.
5193class AssumeAlignedAttrEmitter final
5194 : public AbstractAssumeAlignedAttrEmitter<AssumeAlignedAttr> {
5195public:
5196 AssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
5197 : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
5198 if (!AA)
5199 return;
5200 // It is guaranteed that the alignment/offset are constants.
5201 Alignment = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AA->getAlignment()));
5202 if (Expr *Offset = AA->getOffset()) {
5203 OffsetCI = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(Offset));
5204 if (OffsetCI->isNullValue()) // Canonicalize zero offset to no offset.
5205 OffsetCI = nullptr;
5206 }
5207 }
5208};
5209
5210/// Helper data structure to emit `AllocAlignAttr`.
5211class AllocAlignAttrEmitter final
5212 : public AbstractAssumeAlignedAttrEmitter<AllocAlignAttr> {
5213public:
5214 AllocAlignAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl,
5215 const CallArgList &CallArgs)
5216 : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
5217 if (!AA)
5218 return;
5219 // Alignment may or may not be a constant, and that is okay.
5220 Alignment = CallArgs[AA->getParamIndex().getLLVMIndex()]
5221 .getRValue(CGF)
5222 .getScalarVal();
5223 }
5224};
5225
5226} // namespace
5227
5228static unsigned getMaxVectorWidth(const llvm::Type *Ty) {
5229 if (auto *VT = dyn_cast<llvm::VectorType>(Ty))
5230 return VT->getPrimitiveSizeInBits().getKnownMinValue();
5231 if (auto *AT = dyn_cast<llvm::ArrayType>(Ty))
5232 return getMaxVectorWidth(AT->getElementType());
5233
5234 unsigned MaxVectorWidth = 0;
5235 if (auto *ST = dyn_cast<llvm::StructType>(Ty))
5236 for (auto *I : ST->elements())
5237 MaxVectorWidth = std::max(MaxVectorWidth, getMaxVectorWidth(I));
5238 return MaxVectorWidth;
5239}
5240
5242 const CGCallee &Callee,
5244 const CallArgList &CallArgs,
5245 llvm::CallBase **callOrInvoke, bool IsMustTail,
5246 SourceLocation Loc,
5247 bool IsVirtualFunctionPointerThunk) {
5248 // FIXME: We no longer need the types from CallArgs; lift up and simplify.
5249
5250 assert(Callee.isOrdinary() || Callee.isVirtual());
5251
5252 // Handle struct-return functions by passing a pointer to the
5253 // location that we would like to return into.
5254 QualType RetTy = CallInfo.getReturnType();
5255 const ABIArgInfo &RetAI = CallInfo.getReturnInfo();
5256
5257 llvm::FunctionType *IRFuncTy = getTypes().GetFunctionType(CallInfo);
5258
5259 const Decl *TargetDecl = Callee.getAbstractInfo().getCalleeDecl().getDecl();
5260 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) {
5261 // We can only guarantee that a function is called from the correct
5262 // context/function based on the appropriate target attributes,
5263 // so only check in the case where we have both always_inline and target
5264 // since otherwise we could be making a conditional call after a check for
5265 // the proper cpu features (and it won't cause code generation issues due to
5266 // function based code generation).
5267 if ((TargetDecl->hasAttr<AlwaysInlineAttr>() &&
5268 (TargetDecl->hasAttr<TargetAttr>() ||
5269 (CurFuncDecl && CurFuncDecl->hasAttr<TargetAttr>()))) ||
5270 (CurFuncDecl && CurFuncDecl->hasAttr<FlattenAttr>() &&
5271 (CurFuncDecl->hasAttr<TargetAttr>() ||
5272 TargetDecl->hasAttr<TargetAttr>())))
5273 checkTargetFeatures(Loc, FD);
5274 }
5275
5276 // Some architectures (such as x86-64) have the ABI changed based on
5277 // attribute-target/features. Give them a chance to diagnose.
5278 const FunctionDecl *CallerDecl = dyn_cast_or_null<FunctionDecl>(CurCodeDecl);
5279 const FunctionDecl *CalleeDecl = dyn_cast_or_null<FunctionDecl>(TargetDecl);
5280 CGM.getTargetCodeGenInfo().checkFunctionCallABI(CGM, Loc, CallerDecl,
5281 CalleeDecl, CallArgs, RetTy);
5282
5283 // 1. Set up the arguments.
5284
5285 // If we're using inalloca, insert the allocation after the stack save.
5286 // FIXME: Do this earlier rather than hacking it in here!
5287 RawAddress ArgMemory = RawAddress::invalid();
5288 if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) {
5289 const llvm::DataLayout &DL = CGM.getDataLayout();
5290 llvm::Instruction *IP = CallArgs.getStackBase();
5291 llvm::AllocaInst *AI;
5292 if (IP) {
5293 IP = IP->getNextNode();
5294 AI = new llvm::AllocaInst(ArgStruct, DL.getAllocaAddrSpace(), "argmem",
5295 IP->getIterator());
5296 } else {
5297 AI = CreateTempAlloca(ArgStruct, "argmem");
5298 }
5299 auto Align = CallInfo.getArgStructAlignment();
5300 AI->setAlignment(Align.getAsAlign());
5301 AI->setUsedWithInAlloca(true);
5302 assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca());
5303 ArgMemory = RawAddress(AI, ArgStruct, Align);
5304 }
5305
5306 ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo);
5307 SmallVector<llvm::Value *, 16> IRCallArgs(IRFunctionArgs.totalIRArgs());
5308
5309 // If the call returns a temporary with struct return, create a temporary
5310 // alloca to hold the result, unless one is given to us.
5311 Address SRetPtr = Address::invalid();
5312 bool NeedSRetLifetimeEnd = false;
5313 if (RetAI.isIndirect() || RetAI.isInAlloca() || RetAI.isCoerceAndExpand()) {
5314 // For virtual function pointer thunks and musttail calls, we must always
5315 // forward an incoming SRet pointer to the callee, because a local alloca
5316 // would be de-allocated before the call. These cases both guarantee that
5317 // there will be an incoming SRet argument of the correct type.
5318 if ((IsVirtualFunctionPointerThunk || IsMustTail) && RetAI.isIndirect()) {
5319 SRetPtr = makeNaturalAddressForPointer(CurFn->arg_begin() +
5320 IRFunctionArgs.getSRetArgNo(),
5321 RetTy, CharUnits::fromQuantity(1));
5322 } else if (!ReturnValue.isNull()) {
5323 SRetPtr = ReturnValue.getAddress();
5324 } else {
5325 SRetPtr = CreateMemTempWithoutCast(RetTy, "tmp");
5326 if (HaveInsertPoint() && ReturnValue.isUnused())
5327 NeedSRetLifetimeEnd = EmitLifetimeStart(SRetPtr.getBasePointer());
5328 }
5329 if (IRFunctionArgs.hasSRetArg()) {
5330 // A mismatch between the allocated return value's AS and the target's
5331 // chosen IndirectAS can happen e.g. when passing the this pointer through
5332 // a chain involving stores to / loads from the DefaultAS; we address this
5333 // here, symmetrically with the handling we have for normal pointer args.
5334 if (SRetPtr.getAddressSpace() != RetAI.getIndirectAddrSpace()) {
5335 llvm::Value *V = SRetPtr.getBasePointer();
5337 llvm::Type *Ty = llvm::PointerType::get(getLLVMContext(),
5338 RetAI.getIndirectAddrSpace());
5339
5340 SRetPtr = SRetPtr.withPointer(
5341 getTargetHooks().performAddrSpaceCast(*this, V, SAS, Ty, true),
5342 SRetPtr.isKnownNonNull());
5343 }
5344 IRCallArgs[IRFunctionArgs.getSRetArgNo()] =
5345 getAsNaturalPointerTo(SRetPtr, RetTy);
5346 } else if (RetAI.isInAlloca()) {
5347 Address Addr =
5348 Builder.CreateStructGEP(ArgMemory, RetAI.getInAllocaFieldIndex());
5349 Builder.CreateStore(getAsNaturalPointerTo(SRetPtr, RetTy), Addr);
5350 }
5351 }
5352
5353 RawAddress swiftErrorTemp = RawAddress::invalid();
5354 Address swiftErrorArg = Address::invalid();
5355
5356 // When passing arguments using temporary allocas, we need to add the
5357 // appropriate lifetime markers. This vector keeps track of all the lifetime
5358 // markers that need to be ended right after the call.
5359 SmallVector<CallLifetimeEnd, 2> CallLifetimeEndAfterCall;
5360
5361 // Translate all of the arguments as necessary to match the IR lowering.
5362 assert(CallInfo.arg_size() == CallArgs.size() &&
5363 "Mismatch between function signature & arguments.");
5364 unsigned ArgNo = 0;
5365 CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin();
5366 for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
5367 I != E; ++I, ++info_it, ++ArgNo) {
5368 const ABIArgInfo &ArgInfo = info_it->info;
5369
5370 // Insert a padding argument to ensure proper alignment.
5371 if (IRFunctionArgs.hasPaddingArg(ArgNo))
5372 IRCallArgs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
5373 llvm::UndefValue::get(ArgInfo.getPaddingType());
5374
5375 unsigned FirstIRArg, NumIRArgs;
5376 std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
5377
5378 bool ArgHasMaybeUndefAttr =
5379 IsArgumentMaybeUndef(TargetDecl, CallInfo.getNumRequiredArgs(), ArgNo);
5380
5381 switch (ArgInfo.getKind()) {
5382 case ABIArgInfo::InAlloca: {
5383 assert(NumIRArgs == 0);
5384 assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
5385 if (I->isAggregate()) {
5386 RawAddress Addr = I->hasLValue()
5387 ? I->getKnownLValue().getAddress()
5388 : I->getKnownRValue().getAggregateAddress();
5389 llvm::Instruction *Placeholder =
5390 cast<llvm::Instruction>(Addr.getPointer());
5391
5392 if (!ArgInfo.getInAllocaIndirect()) {
5393 // Replace the placeholder with the appropriate argument slot GEP.
5394 CGBuilderTy::InsertPoint IP = Builder.saveIP();
5395 Builder.SetInsertPoint(Placeholder);
5396 Addr = Builder.CreateStructGEP(ArgMemory,
5397 ArgInfo.getInAllocaFieldIndex());
5398 Builder.restoreIP(IP);
5399 } else {
5400 // For indirect things such as overaligned structs, replace the
5401 // placeholder with a regular aggregate temporary alloca. Store the
5402 // address of this alloca into the struct.
5403 Addr = CreateMemTemp(info_it->type, "inalloca.indirect.tmp");
5404 Address ArgSlot = Builder.CreateStructGEP(
5405 ArgMemory, ArgInfo.getInAllocaFieldIndex());
5406 Builder.CreateStore(Addr.getPointer(), ArgSlot);
5407 }
5408 deferPlaceholderReplacement(Placeholder, Addr.getPointer());
5409 } else if (ArgInfo.getInAllocaIndirect()) {
5410 // Make a temporary alloca and store the address of it into the argument
5411 // struct.
5413 I->Ty, getContext().getTypeAlignInChars(I->Ty),
5414 "indirect-arg-temp");
5415 I->copyInto(*this, Addr);
5416 Address ArgSlot =
5417 Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());
5418 Builder.CreateStore(Addr.getPointer(), ArgSlot);
5419 } else {
5420 // Store the RValue into the argument struct.
5421 Address Addr =
5422 Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());
5423 Addr = Addr.withElementType(ConvertTypeForMem(I->Ty));
5424 I->copyInto(*this, Addr);
5425 }
5426 break;
5427 }
5428
5431 assert(NumIRArgs == 1);
5432 if (I->isAggregate()) {
5433 // We want to avoid creating an unnecessary temporary+copy here;
5434 // however, we need one in three cases:
5435 // 1. If the argument is not byval, and we are required to copy the
5436 // source. (This case doesn't occur on any common architecture.)
5437 // 2. If the argument is byval, RV is not sufficiently aligned, and
5438 // we cannot force it to be sufficiently aligned.
5439 // 3. If the argument is byval, but RV is not located in default
5440 // or alloca address space.
5441 Address Addr = I->hasLValue()
5442 ? I->getKnownLValue().getAddress()
5443 : I->getKnownRValue().getAggregateAddress();
5444 CharUnits Align = ArgInfo.getIndirectAlign();
5445 const llvm::DataLayout *TD = &CGM.getDataLayout();
5446
5447 assert((FirstIRArg >= IRFuncTy->getNumParams() ||
5448 IRFuncTy->getParamType(FirstIRArg)->getPointerAddressSpace() ==
5449 TD->getAllocaAddrSpace()) &&
5450 "indirect argument must be in alloca address space");
5451
5452 bool NeedCopy = false;
5453 if (Addr.getAlignment() < Align &&
5454 llvm::getOrEnforceKnownAlignment(Addr.emitRawPointer(*this),
5455 Align.getAsAlign(),
5456 *TD) < Align.getAsAlign()) {
5457 NeedCopy = true;
5458 } else if (I->hasLValue()) {
5459 auto LV = I->getKnownLValue();
5460
5461 bool isByValOrRef =
5462 ArgInfo.isIndirectAliased() || ArgInfo.getIndirectByVal();
5463
5464 if (!isByValOrRef ||
5465 (LV.getAlignment() < getContext().getTypeAlignInChars(I->Ty))) {
5466 NeedCopy = true;
5467 }
5468
5469 if (isByValOrRef && Addr.getType()->getAddressSpace() !=
5470 ArgInfo.getIndirectAddrSpace()) {
5471 NeedCopy = true;
5472 }
5473 }
5474
5475 if (!NeedCopy) {
5476 // Skip the extra memcpy call.
5477 llvm::Value *V = getAsNaturalPointerTo(Addr, I->Ty);
5478 auto *T = llvm::PointerType::get(CGM.getLLVMContext(),
5479 ArgInfo.getIndirectAddrSpace());
5480
5481 // FIXME: This should not depend on the language address spaces, and
5482 // only the contextual values. If the address space mismatches, see if
5483 // we can look through a cast to a compatible address space value,
5484 // otherwise emit a copy.
5485 llvm::Value *Val = getTargetHooks().performAddrSpaceCast(
5486 *this, V, I->Ty.getAddressSpace(), T, true);
5487 if (ArgHasMaybeUndefAttr)
5488 Val = Builder.CreateFreeze(Val);
5489 IRCallArgs[FirstIRArg] = Val;
5490 break;
5491 }
5492 } else if (I->getType()->isArrayParameterType()) {
5493 // Don't produce a temporary for ArrayParameterType arguments.
5494 // ArrayParameterType arguments are only created from
5495 // HLSL_ArrayRValue casts and HLSLOutArgExpr expressions, both
5496 // of which create temporaries already. This allows us to just use the
5497 // scalar for the decayed array pointer as the argument directly.
5498 IRCallArgs[FirstIRArg] = I->getKnownRValue().getScalarVal();
5499 break;
5500 }
5501
5502 // For non-aggregate args and aggregate args meeting conditions above
5503 // we need to create an aligned temporary, and copy to it.
5505 I->Ty, ArgInfo.getIndirectAlign(), "byval-temp");
5506 llvm::Value *Val = getAsNaturalPointerTo(AI, I->Ty);
5507 if (ArgHasMaybeUndefAttr)
5508 Val = Builder.CreateFreeze(Val);
5509 IRCallArgs[FirstIRArg] = Val;
5510
5511 // Emit lifetime markers for the temporary alloca and add cleanup code to
5512 // emit the end lifetime marker after the call.
5513 if (EmitLifetimeStart(AI.getPointer()))
5514 CallLifetimeEndAfterCall.emplace_back(AI);
5515
5516 // Generate the copy.
5517 I->copyInto(*this, AI);
5518 break;
5519 }
5520
5521 case ABIArgInfo::Ignore:
5522 assert(NumIRArgs == 0);
5523 break;
5524
5525 case ABIArgInfo::Extend:
5526 case ABIArgInfo::Direct: {
5527 if (!isa<llvm::StructType>(ArgInfo.getCoerceToType()) &&
5528 ArgInfo.getCoerceToType() == ConvertType(info_it->type) &&
5529 ArgInfo.getDirectOffset() == 0) {
5530 assert(NumIRArgs == 1);
5531 llvm::Value *V;
5532 if (!I->isAggregate())
5533 V = I->getKnownRValue().getScalarVal();
5534 else
5535 V = Builder.CreateLoad(
5536 I->hasLValue() ? I->getKnownLValue().getAddress()
5537 : I->getKnownRValue().getAggregateAddress());
5538
5539 // Implement swifterror by copying into a new swifterror argument.
5540 // We'll write back in the normal path out of the call.
5541 if (CallInfo.getExtParameterInfo(ArgNo).getABI() ==
5543 assert(!swiftErrorTemp.isValid() && "multiple swifterror args");
5544
5545 QualType pointeeTy = I->Ty->getPointeeType();
5546 swiftErrorArg = makeNaturalAddressForPointer(
5547 V, pointeeTy, getContext().getTypeAlignInChars(pointeeTy));
5548
5549 swiftErrorTemp =
5550 CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
5551 V = swiftErrorTemp.getPointer();
5552 cast<llvm::AllocaInst>(V)->setSwiftError(true);
5553
5554 llvm::Value *errorValue = Builder.CreateLoad(swiftErrorArg);
5555 Builder.CreateStore(errorValue, swiftErrorTemp);
5556 }
5557
5558 // We might have to widen integers, but we should never truncate.
5559 if (ArgInfo.getCoerceToType() != V->getType() &&
5560 V->getType()->isIntegerTy())
5561 V = Builder.CreateZExt(V, ArgInfo.getCoerceToType());
5562
5563 // The only plausible mismatch here would be for pointer address spaces.
5564 // We assume that the target has a reasonable mapping for the DefaultAS
5565 // (it can be casted to from incoming specific ASes), and insert an AS
5566 // cast to address the mismatch.
5567 if (FirstIRArg < IRFuncTy->getNumParams() &&
5568 V->getType() != IRFuncTy->getParamType(FirstIRArg)) {
5569 assert(V->getType()->isPointerTy() && "Only pointers can mismatch!");
5570 auto ActualAS = I->Ty.getAddressSpace();
5571 V = getTargetHooks().performAddrSpaceCast(
5572 *this, V, ActualAS, IRFuncTy->getParamType(FirstIRArg));
5573 }
5574
5575 if (ArgHasMaybeUndefAttr)
5576 V = Builder.CreateFreeze(V);
5577 IRCallArgs[FirstIRArg] = V;
5578 break;
5579 }
5580
5581 llvm::StructType *STy =
5582 dyn_cast<llvm::StructType>(ArgInfo.getCoerceToType());
5583
5584 // FIXME: Avoid the conversion through memory if possible.
5585 Address Src = Address::invalid();
5586 if (!I->isAggregate()) {
5587 Src = CreateMemTemp(I->Ty, "coerce");
5588 I->copyInto(*this, Src);
5589 } else {
5590 Src = I->hasLValue() ? I->getKnownLValue().getAddress()
5591 : I->getKnownRValue().getAggregateAddress();
5592 }
5593
5594 // If the value is offset in memory, apply the offset now.
5595 Src = emitAddressAtOffset(*this, Src, ArgInfo);
5596
5597 // Fast-isel and the optimizer generally like scalar values better than
5598 // FCAs, so we flatten them if this is safe to do for this argument.
5599 if (STy && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
5600 llvm::Type *SrcTy = Src.getElementType();
5601 llvm::TypeSize SrcTypeSize =
5602 CGM.getDataLayout().getTypeAllocSize(SrcTy);
5603 llvm::TypeSize DstTypeSize = CGM.getDataLayout().getTypeAllocSize(STy);
5604 if (SrcTypeSize.isScalable()) {
5605 assert(STy->containsHomogeneousScalableVectorTypes() &&
5606 "ABI only supports structure with homogeneous scalable vector "
5607 "type");
5608 assert(SrcTypeSize == DstTypeSize &&
5609 "Only allow non-fractional movement of structure with "
5610 "homogeneous scalable vector type");
5611 assert(NumIRArgs == STy->getNumElements());
5612
5613 llvm::Value *StoredStructValue =
5614 Builder.CreateLoad(Src, Src.getName() + ".tuple");
5615 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
5616 llvm::Value *Extract = Builder.CreateExtractValue(
5617 StoredStructValue, i, Src.getName() + ".extract" + Twine(i));
5618 IRCallArgs[FirstIRArg + i] = Extract;
5619 }
5620 } else {
5621 uint64_t SrcSize = SrcTypeSize.getFixedValue();
5622 uint64_t DstSize = DstTypeSize.getFixedValue();
5623
5624 // If the source type is smaller than the destination type of the
5625 // coerce-to logic, copy the source value into a temp alloca the size
5626 // of the destination type to allow loading all of it. The bits past
5627 // the source value are left undef.
5628 if (SrcSize < DstSize) {
5629 Address TempAlloca = CreateTempAlloca(STy, Src.getAlignment(),
5630 Src.getName() + ".coerce");
5631 Builder.CreateMemCpy(TempAlloca, Src, SrcSize);
5632 Src = TempAlloca;
5633 } else {
5634 Src = Src.withElementType(STy);
5635 }
5636
5637 assert(NumIRArgs == STy->getNumElements());
5638 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
5639 Address EltPtr = Builder.CreateStructGEP(Src, i);
5640 llvm::Value *LI = Builder.CreateLoad(EltPtr);
5641 if (ArgHasMaybeUndefAttr)
5642 LI = Builder.CreateFreeze(LI);
5643 IRCallArgs[FirstIRArg + i] = LI;
5644 }
5645 }
5646 } else {
5647 // In the simple case, just pass the coerced loaded value.
5648 assert(NumIRArgs == 1);
5649 llvm::Value *Load =
5650 CreateCoercedLoad(Src, ArgInfo.getCoerceToType(), *this);
5651
5652 if (CallInfo.isCmseNSCall()) {
5653 // For certain parameter types, clear padding bits, as they may reveal
5654 // sensitive information.
5655 // Small struct/union types are passed as integer arrays.
5656 auto *ATy = dyn_cast<llvm::ArrayType>(Load->getType());
5657 if (ATy != nullptr && isa<RecordType>(I->Ty.getCanonicalType()))
5658 Load = EmitCMSEClearRecord(Load, ATy, I->Ty);
5659 }
5660
5661 if (ArgHasMaybeUndefAttr)
5662 Load = Builder.CreateFreeze(Load);
5663 IRCallArgs[FirstIRArg] = Load;
5664 }
5665
5666 break;
5667 }
5668
5670 auto coercionType = ArgInfo.getCoerceAndExpandType();
5671 auto layout = CGM.getDataLayout().getStructLayout(coercionType);
5672 auto unpaddedCoercionType = ArgInfo.getUnpaddedCoerceAndExpandType();
5673 auto *unpaddedStruct = dyn_cast<llvm::StructType>(unpaddedCoercionType);
5674
5675 Address addr = Address::invalid();
5676 RawAddress AllocaAddr = RawAddress::invalid();
5677 bool NeedLifetimeEnd = false;
5678 if (I->isAggregate()) {
5679 addr = I->hasLValue() ? I->getKnownLValue().getAddress()
5680 : I->getKnownRValue().getAggregateAddress();
5681
5682 } else {
5683 RValue RV = I->getKnownRValue();
5684 assert(RV.isScalar()); // complex should always just be direct
5685
5686 llvm::Type *scalarType = RV.getScalarVal()->getType();
5687 auto scalarAlign = CGM.getDataLayout().getPrefTypeAlign(scalarType);
5688
5689 // Materialize to a temporary.
5690 addr = CreateTempAlloca(RV.getScalarVal()->getType(),
5691 CharUnits::fromQuantity(std::max(
5692 layout->getAlignment(), scalarAlign)),
5693 "tmp",
5694 /*ArraySize=*/nullptr, &AllocaAddr);
5695 NeedLifetimeEnd = EmitLifetimeStart(AllocaAddr.getPointer());
5696
5697 Builder.CreateStore(RV.getScalarVal(), addr);
5698 }
5699
5700 addr = addr.withElementType(coercionType);
5701
5702 unsigned IRArgPos = FirstIRArg;
5703 unsigned unpaddedIndex = 0;
5704 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
5705 llvm::Type *eltType = coercionType->getElementType(i);
5707 continue;
5708 Address eltAddr = Builder.CreateStructGEP(addr, i);
5709 llvm::Value *elt = CreateCoercedLoad(
5710 eltAddr,
5711 unpaddedStruct ? unpaddedStruct->getElementType(unpaddedIndex++)
5712 : unpaddedCoercionType,
5713 *this);
5714 if (ArgHasMaybeUndefAttr)
5715 elt = Builder.CreateFreeze(elt);
5716 IRCallArgs[IRArgPos++] = elt;
5717 }
5718 assert(IRArgPos == FirstIRArg + NumIRArgs);
5719
5720 if (NeedLifetimeEnd)
5721 EmitLifetimeEnd(AllocaAddr.getPointer());
5722 break;
5723 }
5724
5725 case ABIArgInfo::Expand: {
5726 unsigned IRArgPos = FirstIRArg;
5727 ExpandTypeToArgs(I->Ty, *I, IRFuncTy, IRCallArgs, IRArgPos);
5728 assert(IRArgPos == FirstIRArg + NumIRArgs);
5729 break;
5730 }
5731
5733 Address Src = Address::invalid();
5734 if (!I->isAggregate()) {
5735 Src = CreateMemTemp(I->Ty, "target_coerce");
5736 I->copyInto(*this, Src);
5737 } else {
5738 Src = I->hasLValue() ? I->getKnownLValue().getAddress()
5739 : I->getKnownRValue().getAggregateAddress();
5740 }
5741
5742 // If the value is offset in memory, apply the offset now.
5743 Src = emitAddressAtOffset(*this, Src, ArgInfo);
5744 llvm::Value *Load =
5745 CGM.getABIInfo().createCoercedLoad(Src, ArgInfo, *this);
5746 IRCallArgs[FirstIRArg] = Load;
5747 break;
5748 }
5749 }
5750 }
5751
5752 const CGCallee &ConcreteCallee = Callee.prepareConcreteCallee(*this);
5753 llvm::Value *CalleePtr = ConcreteCallee.getFunctionPointer();
5754
5755 // If we're using inalloca, set up that argument.
5756 if (ArgMemory.isValid()) {
5757 llvm::Value *Arg = ArgMemory.getPointer();
5758 assert(IRFunctionArgs.hasInallocaArg());
5759 IRCallArgs[IRFunctionArgs.getInallocaArgNo()] = Arg;
5760 }
5761
5762 // 2. Prepare the function pointer.
5763
5764 // If the callee is a bitcast of a non-variadic function to have a
5765 // variadic function pointer type, check to see if we can remove the
5766 // bitcast. This comes up with unprototyped functions.
5767 //
5768 // This makes the IR nicer, but more importantly it ensures that we
5769 // can inline the function at -O0 if it is marked always_inline.
5770 auto simplifyVariadicCallee = [](llvm::FunctionType *CalleeFT,
5771 llvm::Value *Ptr) -> llvm::Function * {
5772 if (!CalleeFT->isVarArg())
5773 return nullptr;
5774
5775 // Get underlying value if it's a bitcast
5776 if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Ptr)) {
5777 if (CE->getOpcode() == llvm::Instruction::BitCast)
5778 Ptr = CE->getOperand(0);
5779 }
5780
5781 llvm::Function *OrigFn = dyn_cast<llvm::Function>(Ptr);
5782 if (!OrigFn)
5783 return nullptr;
5784
5785 llvm::FunctionType *OrigFT = OrigFn->getFunctionType();
5786
5787 // If the original type is variadic, or if any of the component types
5788 // disagree, we cannot remove the cast.
5789 if (OrigFT->isVarArg() ||
5790 OrigFT->getNumParams() != CalleeFT->getNumParams() ||
5791 OrigFT->getReturnType() != CalleeFT->getReturnType())
5792 return nullptr;
5793
5794 for (unsigned i = 0, e = OrigFT->getNumParams(); i != e; ++i)
5795 if (OrigFT->getParamType(i) != CalleeFT->getParamType(i))
5796 return nullptr;
5797
5798 return OrigFn;
5799 };
5800
5801 if (llvm::Function *OrigFn = simplifyVariadicCallee(IRFuncTy, CalleePtr)) {
5802 CalleePtr = OrigFn;
5803 IRFuncTy = OrigFn->getFunctionType();
5804 }
5805
5806 // 3. Perform the actual call.
5807
5808 // Deactivate any cleanups that we're supposed to do immediately before
5809 // the call.
5810 if (!CallArgs.getCleanupsToDeactivate().empty())
5811 deactivateArgCleanupsBeforeCall(*this, CallArgs);
5812
5813 // Update the largest vector width if any arguments have vector types.
5814 for (unsigned i = 0; i < IRCallArgs.size(); ++i)
5815 LargestVectorWidth = std::max(LargestVectorWidth,
5816 getMaxVectorWidth(IRCallArgs[i]->getType()));
5817
5818 // Compute the calling convention and attributes.
5819 unsigned CallingConv;
5820 llvm::AttributeList Attrs;
5821 CGM.ConstructAttributeList(CalleePtr->getName(), CallInfo,
5822 Callee.getAbstractInfo(), Attrs, CallingConv,
5823 /*AttrOnCallSite=*/true,
5824 /*IsThunk=*/false);
5825
5826 if (CallingConv == llvm::CallingConv::X86_VectorCall &&
5827 getTarget().getTriple().isWindowsArm64EC()) {
5828 CGM.Error(Loc, "__vectorcall calling convention is not currently "
5829 "supported");
5830 }
5831
5832 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl)) {
5833 if (FD->hasAttr<StrictFPAttr>())
5834 // All calls within a strictfp function are marked strictfp
5835 Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::StrictFP);
5836
5837 // If -ffast-math is enabled and the function is guarded by an
5838 // '__attribute__((optnone)) adjust the memory attribute so the BE emits the
5839 // library call instead of the intrinsic.
5840 if (FD->hasAttr<OptimizeNoneAttr>() && getLangOpts().FastMath)
5841 CGM.AdjustMemoryAttribute(CalleePtr->getName(), Callee.getAbstractInfo(),
5842 Attrs);
5843 }
5844 // Add call-site nomerge attribute if exists.
5846 Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::NoMerge);
5847
5848 // Add call-site noinline attribute if exists.
5850 Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::NoInline);
5851
5852 // Add call-site always_inline attribute if exists.
5853 // Note: This corresponds to the [[clang::always_inline]] statement attribute.
5855 !CGM.getTargetCodeGenInfo().wouldInliningViolateFunctionCallABI(
5856 CallerDecl, CalleeDecl))
5857 Attrs =
5858 Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::AlwaysInline);
5859
5860 // Remove call-site convergent attribute if requested.
5862 Attrs =
5863 Attrs.removeFnAttribute(getLLVMContext(), llvm::Attribute::Convergent);
5864
5865 // Apply some call-site-specific attributes.
5866 // TODO: work this into building the attribute set.
5867
5868 // Apply always_inline to all calls within flatten functions.
5869 // FIXME: should this really take priority over __try, below?
5870 if (CurCodeDecl && CurCodeDecl->hasAttr<FlattenAttr>() &&
5872 !(TargetDecl && TargetDecl->hasAttr<NoInlineAttr>()) &&
5873 !CGM.getTargetCodeGenInfo().wouldInliningViolateFunctionCallABI(
5874 CallerDecl, CalleeDecl)) {
5875 Attrs =
5876 Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::AlwaysInline);
5877 }
5878
5879 // Disable inlining inside SEH __try blocks.
5880 if (isSEHTryScope()) {
5881 Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::NoInline);
5882 }
5883
5884 // Decide whether to use a call or an invoke.
5885 bool CannotThrow;
5887 // SEH cares about asynchronous exceptions, so everything can "throw."
5888 CannotThrow = false;
5889 } else if (isCleanupPadScope() &&
5890 EHPersonality::get(*this).isMSVCXXPersonality()) {
5891 // The MSVC++ personality will implicitly terminate the program if an
5892 // exception is thrown during a cleanup outside of a try/catch.
5893 // We don't need to model anything in IR to get this behavior.
5894 CannotThrow = true;
5895 } else {
5896 // Otherwise, nounwind call sites will never throw.
5897 CannotThrow = Attrs.hasFnAttr(llvm::Attribute::NoUnwind);
5898
5899 if (auto *FPtr = dyn_cast<llvm::Function>(CalleePtr))
5900 if (FPtr->hasFnAttribute(llvm::Attribute::NoUnwind))
5901 CannotThrow = true;
5902 }
5903
5904 // If we made a temporary, be sure to clean up after ourselves. Note that we
5905 // can't depend on being inside of an ExprWithCleanups, so we need to manually
5906 // pop this cleanup later on. Being eager about this is OK, since this
5907 // temporary is 'invisible' outside of the callee.
5908 if (NeedSRetLifetimeEnd)
5910
5911 llvm::BasicBlock *InvokeDest = CannotThrow ? nullptr : getInvokeDest();
5912
5914 getBundlesForFunclet(CalleePtr);
5915
5916 if (SanOpts.has(SanitizerKind::KCFI) &&
5917 !isa_and_nonnull<FunctionDecl>(TargetDecl))
5918 EmitKCFIOperandBundle(ConcreteCallee, BundleList);
5919
5920 // Add the pointer-authentication bundle.
5921 EmitPointerAuthOperandBundle(ConcreteCallee.getPointerAuthInfo(), BundleList);
5922
5923 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl))
5924 if (FD->hasAttr<StrictFPAttr>())
5925 // All calls within a strictfp function are marked strictfp
5926 Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::StrictFP);
5927
5928 AssumeAlignedAttrEmitter AssumeAlignedAttrEmitter(*this, TargetDecl);
5929 Attrs = AssumeAlignedAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
5930
5931 AllocAlignAttrEmitter AllocAlignAttrEmitter(*this, TargetDecl, CallArgs);
5932 Attrs = AllocAlignAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
5933
5934 // Emit the actual call/invoke instruction.
5935 llvm::CallBase *CI;
5936 if (!InvokeDest) {
5937 CI = Builder.CreateCall(IRFuncTy, CalleePtr, IRCallArgs, BundleList);
5938 } else {
5939 llvm::BasicBlock *Cont = createBasicBlock("invoke.cont");
5940 CI = Builder.CreateInvoke(IRFuncTy, CalleePtr, Cont, InvokeDest, IRCallArgs,
5941 BundleList);
5942 EmitBlock(Cont);
5943 }
5944 if (CI->getCalledFunction() && CI->getCalledFunction()->hasName() &&
5945 CI->getCalledFunction()->getName().starts_with("_Z4sqrt")) {
5947 }
5948 if (callOrInvoke)
5949 *callOrInvoke = CI;
5950
5951 // If this is within a function that has the guard(nocf) attribute and is an
5952 // indirect call, add the "guard_nocf" attribute to this call to indicate that
5953 // Control Flow Guard checks should not be added, even if the call is inlined.
5954 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl)) {
5955 if (const auto *A = FD->getAttr<CFGuardAttr>()) {
5956 if (A->getGuard() == CFGuardAttr::GuardArg::nocf &&
5957 !CI->getCalledFunction())
5958 Attrs = Attrs.addFnAttribute(getLLVMContext(), "guard_nocf");
5959 }
5960 }
5961
5962 // Apply the attributes and calling convention.
5963 CI->setAttributes(Attrs);
5964 CI->setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
5965
5966 // Apply various metadata.
5967
5968 if (!CI->getType()->isVoidTy())
5969 CI->setName("call");
5970
5971 if (CGM.shouldEmitConvergenceTokens() && CI->isConvergent())
5972 CI = addConvergenceControlToken(CI);
5973
5974 // Update largest vector width from the return type.
5975 LargestVectorWidth =
5976 std::max(LargestVectorWidth, getMaxVectorWidth(CI->getType()));
5977
5978 // Insert instrumentation or attach profile metadata at indirect call sites.
5979 // For more details, see the comment before the definition of
5980 // IPVK_IndirectCallTarget in InstrProfData.inc.
5981 if (!CI->getCalledFunction())
5982 PGO->valueProfile(Builder, llvm::IPVK_IndirectCallTarget, CI, CalleePtr);
5983
5984 // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
5985 // optimizer it can aggressively ignore unwind edges.
5986 if (CGM.getLangOpts().ObjCAutoRefCount)
5987 AddObjCARCExceptionMetadata(CI);
5988
5989 // Set tail call kind if necessary.
5990 if (llvm::CallInst *Call = dyn_cast<llvm::CallInst>(CI)) {
5991 if (TargetDecl && TargetDecl->hasAttr<NotTailCalledAttr>())
5992 Call->setTailCallKind(llvm::CallInst::TCK_NoTail);
5993 else if (IsMustTail) {
5994 if (getTarget().getTriple().isPPC()) {
5995 if (getTarget().getTriple().isOSAIX())
5996 CGM.getDiags().Report(Loc, diag::err_aix_musttail_unsupported);
5997 else if (!getTarget().hasFeature("pcrelative-memops")) {
5998 if (getTarget().hasFeature("longcall"))
5999 CGM.getDiags().Report(Loc, diag::err_ppc_impossible_musttail) << 0;
6000 else if (Call->isIndirectCall())
6001 CGM.getDiags().Report(Loc, diag::err_ppc_impossible_musttail) << 1;
6002 else if (isa_and_nonnull<FunctionDecl>(TargetDecl)) {
6003 if (!cast<FunctionDecl>(TargetDecl)->isDefined())
6004 // The undefined callee may be a forward declaration. Without
6005 // knowning all symbols in the module, we won't know the symbol is
6006 // defined or not. Collect all these symbols for later diagnosing.
6007 CGM.addUndefinedGlobalForTailCall(
6008 {cast<FunctionDecl>(TargetDecl), Loc});
6009 else {
6010 llvm::GlobalValue::LinkageTypes Linkage = CGM.getFunctionLinkage(
6011 GlobalDecl(cast<FunctionDecl>(TargetDecl)));
6012 if (llvm::GlobalValue::isWeakForLinker(Linkage) ||
6013 llvm::GlobalValue::isDiscardableIfUnused(Linkage))
6014 CGM.getDiags().Report(Loc, diag::err_ppc_impossible_musttail)
6015 << 2;
6016 }
6017 }
6018 }
6019 }
6020 Call->setTailCallKind(llvm::CallInst::TCK_MustTail);
6021 }
6022 }
6023
6024 // Add metadata for calls to MSAllocator functions
6025 if (getDebugInfo() && TargetDecl && TargetDecl->hasAttr<MSAllocatorAttr>())
6026 getDebugInfo()->addHeapAllocSiteMetadata(CI, RetTy->getPointeeType(), Loc);
6027
6028 // Add metadata if calling an __attribute__((error(""))) or warning fn.
6029 if (TargetDecl && TargetDecl->hasAttr<ErrorAttr>()) {
6030 llvm::ConstantInt *Line =
6031 llvm::ConstantInt::get(Int64Ty, Loc.getRawEncoding());
6032 llvm::ConstantAsMetadata *MD = llvm::ConstantAsMetadata::get(Line);
6033 llvm::MDTuple *MDT = llvm::MDNode::get(getLLVMContext(), {MD});
6034 CI->setMetadata("srcloc", MDT);
6035 }
6036
6037 // 4. Finish the call.
6038
6039 // If the call doesn't return, finish the basic block and clear the
6040 // insertion point; this allows the rest of IRGen to discard
6041 // unreachable code.
6042 if (CI->doesNotReturn()) {
6043 if (NeedSRetLifetimeEnd)
6045
6046 // Strip away the noreturn attribute to better diagnose unreachable UB.
6047 if (SanOpts.has(SanitizerKind::Unreachable)) {
6048 // Also remove from function since CallBase::hasFnAttr additionally checks
6049 // attributes of the called function.
6050 if (auto *F = CI->getCalledFunction())
6051 F->removeFnAttr(llvm::Attribute::NoReturn);
6052 CI->removeFnAttr(llvm::Attribute::NoReturn);
6053
6054 // Avoid incompatibility with ASan which relies on the `noreturn`
6055 // attribute to insert handler calls.
6056 if (SanOpts.hasOneOf(SanitizerKind::Address |
6057 SanitizerKind::KernelAddress)) {
6058 SanitizerScope SanScope(this);
6059 llvm::IRBuilder<>::InsertPointGuard IPGuard(Builder);
6060 Builder.SetInsertPoint(CI);
6061 auto *FnType = llvm::FunctionType::get(CGM.VoidTy, /*isVarArg=*/false);
6062 llvm::FunctionCallee Fn =
6063 CGM.CreateRuntimeFunction(FnType, "__asan_handle_no_return");
6065 }
6066 }
6067
6068 EmitUnreachable(Loc);
6069 Builder.ClearInsertionPoint();
6070
6071 // FIXME: For now, emit a dummy basic block because expr emitters in
6072 // generally are not ready to handle emitting expressions at unreachable
6073 // points.
6075
6076 // Return a reasonable RValue.
6077 return GetUndefRValue(RetTy);
6078 }
6079
6080 // If this is a musttail call, return immediately. We do not branch to the
6081 // epilogue in this case.
6082 if (IsMustTail) {
6083 for (auto it = EHStack.find(CurrentCleanupScopeDepth); it != EHStack.end();
6084 ++it) {
6085 EHCleanupScope *Cleanup = dyn_cast<EHCleanupScope>(&*it);
6086 // Fake uses can be safely emitted immediately prior to the tail call, so
6087 // we choose to emit them just before the call here.
6088 if (Cleanup && Cleanup->isFakeUse()) {
6089 CGBuilderTy::InsertPointGuard IPG(Builder);
6090 Builder.SetInsertPoint(CI);
6091 Cleanup->getCleanup()->Emit(*this, EHScopeStack::Cleanup::Flags());
6092 } else if (!(Cleanup &&
6093 Cleanup->getCleanup()->isRedundantBeforeReturn())) {
6094 CGM.ErrorUnsupported(MustTailCall, "tail call skipping over cleanups");
6095 }
6096 }
6097 if (CI->getType()->isVoidTy())
6098 Builder.CreateRetVoid();
6099 else
6100 Builder.CreateRet(CI);
6101 Builder.ClearInsertionPoint();
6103 return GetUndefRValue(RetTy);
6104 }
6105
6106 // Perform the swifterror writeback.
6107 if (swiftErrorTemp.isValid()) {
6108 llvm::Value *errorResult = Builder.CreateLoad(swiftErrorTemp);
6109 Builder.CreateStore(errorResult, swiftErrorArg);
6110 }
6111
6112 // Emit any call-associated writebacks immediately. Arguably this
6113 // should happen after any return-value munging.
6114 if (CallArgs.hasWritebacks())
6115 EmitWritebacks(CallArgs);
6116
6117 // The stack cleanup for inalloca arguments has to run out of the normal
6118 // lexical order, so deactivate it and run it manually here.
6119 CallArgs.freeArgumentMemory(*this);
6120
6121 // Extract the return value.
6122 RValue Ret;
6123
6124 // If the current function is a virtual function pointer thunk, avoid copying
6125 // the return value of the musttail call to a temporary.
6126 if (IsVirtualFunctionPointerThunk) {
6127 Ret = RValue::get(CI);
6128 } else {
6129 Ret = [&] {
6130 switch (RetAI.getKind()) {
6132 auto coercionType = RetAI.getCoerceAndExpandType();
6133
6134 Address addr = SRetPtr.withElementType(coercionType);
6135
6136 assert(CI->getType() == RetAI.getUnpaddedCoerceAndExpandType());
6137 bool requiresExtract = isa<llvm::StructType>(CI->getType());
6138
6139 unsigned unpaddedIndex = 0;
6140 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
6141 llvm::Type *eltType = coercionType->getElementType(i);
6143 continue;
6144 Address eltAddr = Builder.CreateStructGEP(addr, i);
6145 llvm::Value *elt = CI;
6146 if (requiresExtract)
6147 elt = Builder.CreateExtractValue(elt, unpaddedIndex++);
6148 else
6149 assert(unpaddedIndex == 0);
6150 Builder.CreateStore(elt, eltAddr);
6151 }
6152 [[fallthrough]];
6153 }
6154
6156 case ABIArgInfo::Indirect: {
6157 RValue ret = convertTempToRValue(SRetPtr, RetTy, SourceLocation());
6158 if (NeedSRetLifetimeEnd)
6160 return ret;
6161 }
6162
6163 case ABIArgInfo::Ignore:
6164 // If we are ignoring an argument that had a result, make sure to
6165 // construct the appropriate return value for our caller.
6166 return GetUndefRValue(RetTy);
6167
6168 case ABIArgInfo::Extend:
6169 case ABIArgInfo::Direct: {
6170 llvm::Type *RetIRTy = ConvertType(RetTy);
6171 if (RetAI.getCoerceToType() == RetIRTy &&
6172 RetAI.getDirectOffset() == 0) {
6173 switch (getEvaluationKind(RetTy)) {
6174 case TEK_Complex: {
6175 llvm::Value *Real = Builder.CreateExtractValue(CI, 0);
6176 llvm::Value *Imag = Builder.CreateExtractValue(CI, 1);
6177 return RValue::getComplex(std::make_pair(Real, Imag));
6178 }
6179 case TEK_Aggregate:
6180 break;
6181 case TEK_Scalar: {
6182 // If the argument doesn't match, perform a bitcast to coerce it.
6183 // This can happen due to trivial type mismatches.
6184 llvm::Value *V = CI;
6185 if (V->getType() != RetIRTy)
6186 V = Builder.CreateBitCast(V, RetIRTy);
6187 return RValue::get(V);
6188 }
6189 }
6190 }
6191
6192 // If coercing a fixed vector from a scalable vector for ABI
6193 // compatibility, and the types match, use the llvm.vector.extract
6194 // intrinsic to perform the conversion.
6195 if (auto *FixedDstTy = dyn_cast<llvm::FixedVectorType>(RetIRTy)) {
6196 llvm::Value *V = CI;
6197 if (auto *ScalableSrcTy =
6198 dyn_cast<llvm::ScalableVectorType>(V->getType())) {
6199 if (FixedDstTy->getElementType() ==
6200 ScalableSrcTy->getElementType()) {
6201 V = Builder.CreateExtractVector(FixedDstTy, V, uint64_t(0),
6202 "cast.fixed");
6203 return RValue::get(V);
6204 }
6205 }
6206 }
6207
6208 Address DestPtr = ReturnValue.getValue();
6209 bool DestIsVolatile = ReturnValue.isVolatile();
6210 uint64_t DestSize =
6211 getContext().getTypeInfoDataSizeInChars(RetTy).Width.getQuantity();
6212
6213 if (!DestPtr.isValid()) {
6214 DestPtr = CreateMemTemp(RetTy, "coerce");
6215 DestIsVolatile = false;
6216 DestSize = getContext().getTypeSizeInChars(RetTy).getQuantity();
6217 }
6218
6219 // An empty record can overlap other data (if declared with
6220 // no_unique_address); omit the store for such types - as there is no
6221 // actual data to store.
6222 if (!isEmptyRecord(getContext(), RetTy, true)) {
6223 // If the value is offset in memory, apply the offset now.
6224 Address StorePtr = emitAddressAtOffset(*this, DestPtr, RetAI);
6226 CI, StorePtr,
6227 llvm::TypeSize::getFixed(DestSize - RetAI.getDirectOffset()),
6228 DestIsVolatile);
6229 }
6230
6231 return convertTempToRValue(DestPtr, RetTy, SourceLocation());
6232 }
6233
6235 Address DestPtr = ReturnValue.getValue();
6236 Address StorePtr = emitAddressAtOffset(*this, DestPtr, RetAI);
6237 bool DestIsVolatile = ReturnValue.isVolatile();
6238 if (!DestPtr.isValid()) {
6239 DestPtr = CreateMemTemp(RetTy, "target_coerce");
6240 DestIsVolatile = false;
6241 }
6242 CGM.getABIInfo().createCoercedStore(CI, StorePtr, RetAI, DestIsVolatile,
6243 *this);
6244 return convertTempToRValue(DestPtr, RetTy, SourceLocation());
6245 }
6246
6247 case ABIArgInfo::Expand:
6249 llvm_unreachable("Invalid ABI kind for return argument");
6250 }
6251
6252 llvm_unreachable("Unhandled ABIArgInfo::Kind");
6253 }();
6254 }
6255
6256 // Emit the assume_aligned check on the return value.
6257 if (Ret.isScalar() && TargetDecl) {
6258 AssumeAlignedAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret);
6259 AllocAlignAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret);
6260 }
6261
6262 // Explicitly call CallLifetimeEnd::Emit just to re-use the code even though
6263 // we can't use the full cleanup mechanism.
6264 for (CallLifetimeEnd &LifetimeEnd : CallLifetimeEndAfterCall)
6265 LifetimeEnd.Emit(*this, /*Flags=*/{});
6266
6267 if (!ReturnValue.isExternallyDestructed() &&
6269 pushDestroy(QualType::DK_nontrivial_c_struct, Ret.getAggregateAddress(),
6270 RetTy);
6271
6272 return Ret;
6273}
6274
6276 if (isVirtual()) {
6277 const CallExpr *CE = getVirtualCallExpr();
6280 CE ? CE->getBeginLoc() : SourceLocation());
6281 }
6282
6283 return *this;
6284}
6285
6286/* VarArg handling */
6287
6289 AggValueSlot Slot) {
6290 VAListAddr = VE->isMicrosoftABI() ? EmitMSVAListRef(VE->getSubExpr())
6291 : EmitVAListRef(VE->getSubExpr());
6292 QualType Ty = VE->getType();
6293 if (Ty->isVariablyModifiedType())
6295 if (VE->isMicrosoftABI())
6296 return CGM.getABIInfo().EmitMSVAArg(*this, VAListAddr, Ty, Slot);
6297 return CGM.getABIInfo().EmitVAArg(*this, VAListAddr, Ty, Slot);
6298}
6299
6304
#define V(N, I)
static ExtParameterInfoList getExtParameterInfosForCall(const FunctionProtoType *proto, unsigned prefixArgs, unsigned totalArgs)
Definition CGCall.cpp:467
static bool isInAllocaArgument(CGCXXABI &ABI, QualType type)
Definition CGCall.cpp:4271
static uint64_t buildMultiCharMask(const SmallVectorImpl< uint64_t > &Bits, int Pos, int Size, int CharWidth, bool BigEndian)
Definition CGCall.cpp:3927
static llvm::Value * tryRemoveRetainOfSelf(CodeGenFunction &CGF, llvm::Value *result)
If this is a +1 of the value of an immutable 'self', remove it.
Definition CGCall.cpp:3670
static CanQualType GetReturnType(QualType RetTy)
Returns the "extra-canonicalized" return type, which discards qualifiers on the return type.
Definition CGCall.cpp:151
static const NonNullAttr * getNonNullAttr(const Decl *FD, const ParmVarDecl *PVD, QualType ArgType, unsigned ArgNo)
Returns the attribute (either parameter attribute, or function attribute), which declares argument Ar...
Definition CGCall.cpp:3068
static CanQualTypeList getArgTypesForCall(ASTContext &ctx, const CallArgList &args)
Definition CGCall.cpp:450
static Address emitAddressAtOffset(CodeGenFunction &CGF, Address addr, const ABIArgInfo &info)
Definition CGCall.cpp:1469
static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF, QualType Ty)
Definition CGCall.cpp:4276
static CallingConv getCallingConventionForDecl(const ObjCMethodDecl *D, bool IsTargetDefaultMSABI)
Definition CGCall.cpp:255
static void setBitRange(SmallVectorImpl< uint64_t > &Bits, int BitOffset, int BitWidth, int CharWidth)
Definition CGCall.cpp:3807
static bool isProvablyNull(llvm::Value *addr)
Definition CGCall.cpp:4342
static void AddAttributesFromFunctionProtoType(ASTContext &Ctx, llvm::AttrBuilder &FuncAttrs, const FunctionProtoType *FPT)
Definition CGCall.cpp:1839
static void eraseUnusedBitCasts(llvm::Instruction *insn)
Definition CGCall.cpp:3565
static bool isObjCMethodWithTypeParams(const ObjCMethodDecl *method)
Definition CGCall.cpp:4666
static void addNoBuiltinAttributes(llvm::AttrBuilder &FuncAttrs, const LangOptions &LangOpts, const NoBuiltinAttr *NBA=nullptr)
Definition CGCall.cpp:2242
static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args, const ObjCIndirectCopyRestoreExpr *CRE)
Emit an argument that's being passed call-by-writeback.
Definition CGCall.cpp:4444
static void overrideFunctionFeaturesWithTargetFeatures(llvm::AttrBuilder &FuncAttr, const llvm::Function &F, const TargetOptions &TargetOpts)
Merges target-features from \TargetOpts and \F, and sets the result in \FuncAttr.
Definition CGCall.cpp:2116
static llvm::Value * CreateCoercedLoad(Address Src, llvm::Type *Ty, CodeGenFunction &CGF)
CreateCoercedLoad - Create a load from.
Definition CGCall.cpp:1319
static int getExpansionSize(QualType Ty, const ASTContext &Context)
Definition CGCall.cpp:1055
static CanQual< FunctionProtoType > GetFormalType(const CXXMethodDecl *MD)
Returns the canonical formal type of the given C++ method.
Definition CGCall.cpp:141
static bool DetermineNoUndef(QualType QTy, CodeGenTypes &Types, const llvm::DataLayout &DL, const ABIArgInfo &AI, bool CheckCoerce=true)
Definition CGCall.cpp:2278
static const Expr * maybeGetUnaryAddrOfOperand(const Expr *E)
Definition CGCall.cpp:4433
static void addDenormalModeAttrs(llvm::DenormalMode FPDenormalMode, llvm::DenormalMode FP32DenormalMode, llvm::AttrBuilder &FuncAttrs)
Add denormal-fp-math and denormal-fp-math-f32 as appropriate for the requested denormal behavior,...
Definition CGCall.cpp:1940
static void deactivateArgCleanupsBeforeCall(CodeGenFunction &CGF, const CallArgList &CallArgs)
Definition CGCall.cpp:4422
static bool isProvablyNonNull(Address Addr, CodeGenFunction &CGF)
Definition CGCall.cpp:4346
static llvm::Value * emitArgumentDemotion(CodeGenFunction &CGF, const VarDecl *var, llvm::Value *value)
An argument came in as a promoted argument; demote it back to its declared type.
Definition CGCall.cpp:3047
SmallVector< CanQualType, 16 > CanQualTypeList
Definition CGCall.cpp:244
static std::pair< llvm::Value *, bool > CoerceScalableToFixed(CodeGenFunction &CGF, llvm::FixedVectorType *ToTy, llvm::ScalableVectorType *FromTy, llvm::Value *V, StringRef Name="")
Definition CGCall.cpp:1481
static const CGFunctionInfo & arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod, SmallVectorImpl< CanQualType > &prefix, CanQual< FunctionProtoType > FTP)
Arrange the LLVM function layout for a value of the given function type, on top of any implicit param...
Definition CGCall.cpp:230
static void addExtParameterInfosForCall(llvm::SmallVectorImpl< FunctionProtoType::ExtParameterInfo > &paramInfos, const FunctionProtoType *proto, unsigned prefixArgs, unsigned totalArgs)
Definition CGCall.cpp:166
static bool canApplyNoFPClass(const ABIArgInfo &AI, QualType ParamType, bool IsReturn)
Test if it's legal to apply nofpclass for the given parameter type and it's lowered IR type.
Definition CGCall.cpp:2351
static void getTrivialDefaultFunctionAttributes(StringRef Name, bool HasOptnone, const CodeGenOptions &CodeGenOpts, const LangOptions &LangOpts, bool AttrOnCallSite, llvm::AttrBuilder &FuncAttrs)
Definition CGCall.cpp:1960
static llvm::FPClassTest getNoFPClassTestMask(const LangOptions &LangOpts)
Return the nofpclass mask that can be applied to floating-point parameters.
Definition CGCall.cpp:2372
static void forConstantArrayExpansion(CodeGenFunction &CGF, ConstantArrayExpansion *CAE, Address BaseAddr, llvm::function_ref< void(Address)> Fn)
Definition CGCall.cpp:1096
static bool IsArgumentMaybeUndef(const Decl *TargetDecl, unsigned NumRequiredArgs, unsigned ArgNo)
Check if the argument of a function has maybe_undef attribute.
Definition CGCall.cpp:2329
static bool hasInAllocaArgs(CodeGenModule &CGM, CallingConv ExplicitCC, ArrayRef< QualType > ArgTypes)
Definition CGCall.cpp:4648
static std::unique_ptr< TypeExpansion > getTypeExpansion(QualType Ty, const ASTContext &Context)
Definition CGCall.cpp:1002
SmallVector< FunctionProtoType::ExtParameterInfo, 16 > ExtParameterInfoList
Definition CGCall.cpp:224
static RawAddress CreateTempAllocaForCoercion(CodeGenFunction &CGF, llvm::Type *Ty, CharUnits MinAlign, const Twine &Name="tmp")
Create a temporary allocation for the purposes of coercion.
Definition CGCall.cpp:1216
static void setUsedBits(CodeGenModule &, QualType, int, SmallVectorImpl< uint64_t > &)
Definition CGCall.cpp:3910
static llvm::StoreInst * findDominatingStoreToReturnValue(CodeGenFunction &CGF)
Heuristically search for a dominating store to the return-value slot.
Definition CGCall.cpp:3729
static void setCUDAKernelCallingConvention(CanQualType &FTy, CodeGenModule &CGM, const FunctionDecl *FD)
Set calling convention for CUDA/HIP kernel.
Definition CGCall.cpp:359
static llvm::Value * tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF, llvm::Value *result)
Try to emit a fused autorelease of a return result.
Definition CGCall.cpp:3578
static Address EnterStructPointerForCoercedAccess(Address SrcPtr, llvm::StructType *SrcSTy, uint64_t DstSize, CodeGenFunction &CGF)
EnterStructPointerForCoercedAccess - Given a struct pointer that we are accessing some number of byte...
Definition CGCall.cpp:1231
static llvm::Value * emitAutoreleaseOfResult(CodeGenFunction &CGF, llvm::Value *result)
Emit an ARC autorelease of the result of a function.
Definition CGCall.cpp:3711
static void emitWriteback(CodeGenFunction &CGF, const CallArgList::Writeback &writeback)
Emit the actual writing-back of a writeback.
Definition CGCall.cpp:4351
static bool HasStrictReturn(const CodeGenModule &Module, QualType RetTy, const Decl *TargetDecl)
Definition CGCall.cpp:1905
static CanQualTypeList getArgTypesForDeclaration(ASTContext &ctx, const FunctionArgList &args)
Definition CGCall.cpp:458
static void addMergableDefaultFunctionAttributes(const CodeGenOptions &CodeGenOpts, llvm::AttrBuilder &FuncAttrs)
Add default attributes to a function, which have merge semantics under -mlink-builtin-bitcode and sho...
Definition CGCall.cpp:1954
static llvm::Value * CoerceIntOrPtrToIntOrPtr(llvm::Value *Val, llvm::Type *Ty, CodeGenFunction &CGF)
CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both are either integers or p...
Definition CGCall.cpp:1268
static void AddAttributesFromOMPAssumes(llvm::AttrBuilder &FuncAttrs, const Decl *Callee)
Definition CGCall.cpp:1878
static unsigned getMaxVectorWidth(const llvm::Type *Ty)
Definition CGCall.cpp:5228
CodeGenFunction::ComplexPairTy ComplexPairTy
static void appendParameterTypes(const CIRGenTypes &cgt, SmallVectorImpl< CanQualType > &prefix, CanQual< FunctionProtoType > fpt)
Adds the formal parameters in FPT to the given prefix.
static const CIRGenFunctionInfo & arrangeFreeFunctionLikeCall(CIRGenTypes &cgt, CIRGenModule &cgm, const CallArgList &args, const FunctionType *fnType)
Defines the C++ Decl subclasses, other than those for templates (found in DeclTemplate....
TokenType getType() const
Returns the token's type, e.g.
tooling::Replacements cleanup(const FormatStyle &Style, StringRef Code, ArrayRef< tooling::Range > Ranges, StringRef FileName="<stdin>")
Clean up any erroneous/redundant code in the given Ranges in Code.
#define CC_VLS_CASE(ABI_VLEN)
static bool hasFeature(StringRef Feature, const LangOptions &LangOpts, const TargetInfo &Target)
Determine whether a translation unit built using the current language options has the given feature.
Definition Module.cpp:95
SanitizerHandler
static QualType getPointeeType(const MemRegion *R)
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition ASTContext.h:188
CanQualType getCanonicalParamType(QualType T) const
Return the canonical parameter type corresponding to the specific potentially non-canonical one.
CanQualType getCanonicalSizeType() const
const TargetInfo & getTargetInfo() const
Definition ASTContext.h:856
uint64_t getCharWidth() const
Return the size of the character type, in bits.
ASTRecordLayout - This class contains layout information for one RecordDecl, which is a struct/union/...
uint64_t getFieldOffset(unsigned FieldNo) const
getFieldOffset - Get the offset of the given field index, in bits.
Represents an array type, per C99 6.7.5.2 - Array Declarators.
Definition TypeBase.h:3720
Attr - This represents one attribute.
Definition Attr.h:44
This class is used for builtin types like 'int'.
Definition TypeBase.h:3164
QualType getType() const
Retrieves the type of the base class.
Definition DeclCXX.h:249
Represents a C++ constructor within a class.
Definition DeclCXX.h:2604
Represents a C++ destructor within a class.
Definition DeclCXX.h:2869
Represents a static or instance method of a struct/union/class.
Definition DeclCXX.h:2129
bool isImplicitObjectMemberFunction() const
[C++2b][dcl.fct]/p7 An implicit object member function is a non-static member function without an exp...
Definition DeclCXX.cpp:2710
bool isVirtual() const
Definition DeclCXX.h:2184
const CXXRecordDecl * getParent() const
Return the parent of this method declaration, which is the class in which this method is defined.
Definition DeclCXX.h:2255
Qualifiers getMethodQualifiers() const
Definition DeclCXX.h:2290
Represents a C++ struct/union/class.
Definition DeclCXX.h:258
CXXDestructorDecl * getDestructor() const
Returns the destructor decl for this class.
Definition DeclCXX.cpp:2121
unsigned getNumVBases() const
Retrieves the number of virtual base classes of this class.
Definition DeclCXX.h:623
CallExpr - Represents a function call (C99 6.5.2.2, C++ [expr.call]).
Definition Expr.h:2877
SourceLocation getBeginLoc() const
Definition Expr.h:3211
ConstExprIterator const_arg_iterator
Definition Expr.h:3125
Represents a canonical, potentially-qualified type.
static CanQual< Type > CreateUnsafe(QualType Other)
CanProxy< U > castAs() const
CanQual< T > getUnqualifiedType() const
Retrieve the unqualified form of this type.
CanProxy< U > getAs() const
Retrieve a canonical type pointer with a different static type, upcasting or downcasting as needed.
const T * getTypePtr() const
Retrieve the underlying type pointer, which refers to a canonical type.
CharUnits - This is an opaque type for sizes expressed in character units.
Definition CharUnits.h:38
bool isZero() const
isZero - Test whether the quantity equals zero.
Definition CharUnits.h:122
llvm::Align getAsAlign() const
getAsAlign - Returns Quantity as a valid llvm::Align, Beware llvm::Align assumes power of two 8-bit b...
Definition CharUnits.h:189
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
Definition CharUnits.h:185
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
Definition CharUnits.h:63
static CharUnits Zero()
Zero - Construct a CharUnits quantity of zero.
Definition CharUnits.h:53
CodeGenOptions - Track various options which control how the code is optimized and passed to the back...
llvm::DenormalMode FPDenormalMode
The floating-point denormal mode to use.
static StringRef getFramePointerKindName(FramePointerKind Kind)
std::vector< std::string > Reciprocals
llvm::DenormalMode FP32DenormalMode
The floating-point denormal mode to use, for float.
std::string TrapFuncName
If not an empty string, trap intrinsics are lowered to calls to this function instead of to trap inst...
std::vector< std::string > DefaultFunctionAttrs
std::string PreferVectorWidth
The preferred width for auto-vectorization transforms.
ABIArgInfo - Helper class to encapsulate information about how a specific C type should be passed to ...
unsigned getInAllocaFieldIndex() const
llvm::StructType * getCoerceAndExpandType() const
void setCoerceToType(llvm::Type *T)
llvm::Type * getUnpaddedCoerceAndExpandType() const
unsigned getDirectOffset() const
static bool isPaddingForCoerceAndExpand(llvm::Type *eltType)
bool getInAllocaSRet() const
Return true if this field of an inalloca struct should be returned to implement a struct return calli...
llvm::Type * getPaddingType() const
unsigned getDirectAlign() const
unsigned getIndirectAddrSpace() const
@ Extend
Extend - Valid only for integer argument types.
@ Ignore
Ignore - Ignore the argument (treat as void).
@ IndirectAliased
IndirectAliased - Similar to Indirect, but the pointer may be to an object that is otherwise referenc...
@ Expand
Expand - Only valid for aggregate argument types.
@ TargetSpecific
TargetSpecific - Some argument types are passed as target specific types such as RISC-V's tuple type,...
@ InAlloca
InAlloca - Pass the argument directly using the LLVM inalloca attribute.
@ Indirect
Indirect - Pass the argument indirectly via a hidden pointer with the specified alignment (0 indicate...
@ CoerceAndExpand
CoerceAndExpand - Only valid for aggregate argument types.
@ Direct
Direct - Pass the argument directly using the normal converted LLVM type, or by coercing to another s...
ArrayRef< llvm::Type * > getCoerceAndExpandTypeSequence() const
unsigned getInAllocaIndirect() const
llvm::Type * getCoerceToType() const
CharUnits getIndirectAlign() const
Like RawAddress, an abstract representation of an aligned address, but the pointer contained in this ...
Definition Address.h:128
llvm::Value * getBasePointer() const
Definition Address.h:198
static Address invalid()
Definition Address.h:176
llvm::Value * emitRawPointer(CodeGenFunction &CGF) const
Return the pointer contained in this class after authenticating it and adding offset to it if necessa...
Definition Address.h:253
CharUnits getAlignment() const
Definition Address.h:194
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Definition Address.h:209
Address withPointer(llvm::Value *NewPointer, KnownNonNull_t IsKnownNonNull) const
Return address with different pointer, but same element type and alignment.
Definition Address.h:261
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
Definition Address.h:276
unsigned getAddressSpace() const
Return the address space that this address resides in.
Definition Address.h:215
KnownNonNull_t isKnownNonNull() const
Whether the pointer is known not to be null.
Definition Address.h:233
llvm::StringRef getName() const
Return the IR name of the pointer value.
Definition Address.h:218
bool isValid() const
Definition Address.h:177
An aggregate value slot.
Definition CGValue.h:504
Address getAddress() const
Definition CGValue.h:644
void setExternallyDestructed(bool destructed=true)
Definition CGValue.h:613
static AggValueSlot forAddr(Address addr, Qualifiers quals, IsDestructed_t isDestructed, NeedsGCBarriers_t needsGC, IsAliased_t isAliased, Overlap_t mayOverlap, IsZeroed_t isZeroed=IsNotZeroed, IsSanitizerChecked_t isChecked=IsNotSanitizerChecked)
forAddr - Make a slot for an aggregate value.
Definition CGValue.h:587
RValue asRValue() const
Definition CGValue.h:666
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
Definition CGBuilder.h:140
Address CreateConstInBoundsByteGEP(Address Addr, CharUnits Offset, const llvm::Twine &Name="")
Given a pointer to i8, adjust it by a given constant offset.
Definition CGBuilder.h:309
llvm::Value * CreateIsNull(Address Addr, const Twine &Name="")
Definition CGBuilder.h:360
Address CreateConstGEP2_32(Address Addr, unsigned Idx0, unsigned Idx1, const llvm::Twine &Name="")
Definition CGBuilder.h:335
Address CreateStructGEP(Address Addr, unsigned Index, const llvm::Twine &Name="")
Definition CGBuilder.h:223
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
Definition CGBuilder.h:112
llvm::CallInst * CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
Definition CGBuilder.h:369
llvm::LoadInst * CreateAlignedLoad(llvm::Type *Ty, llvm::Value *Addr, CharUnits Align, const llvm::Twine &Name="")
Definition CGBuilder.h:132
Implements C++ ABI-specific code generation functions.
Definition CGCXXABI.h:43
virtual bool hasMostDerivedReturn(GlobalDecl GD) const
Definition CGCXXABI.h:131
virtual bool HasThisReturn(GlobalDecl GD) const
Returns true if the given constructor or destructor is one of the kinds that the ABI says returns 'th...
Definition CGCXXABI.h:123
@ RAA_DirectInMemory
Pass it on the stack using its defined layout.
Definition CGCXXABI.h:158
virtual CGCallee getVirtualFunctionPointer(CodeGenFunction &CGF, GlobalDecl GD, Address This, llvm::Type *Ty, SourceLocation Loc)=0
Build a virtual function pointer in the ABI-specific way.
virtual RecordArgABI getRecordArgABI(const CXXRecordDecl *RD) const =0
Returns how an argument of the given record type should be passed.
virtual const CXXRecordDecl * getThisArgumentTypeForMethod(GlobalDecl GD)
Get the type of the implicit "this" parameter used by a method.
Definition CGCXXABI.h:395
virtual AddedStructorArgCounts buildStructorSignature(GlobalDecl GD, SmallVectorImpl< CanQualType > &ArgTys)=0
Build the signature of the given constructor or destructor variant by adding any required parameters.
Abstract information about a function or function prototype.
Definition CGCall.h:41
const GlobalDecl getCalleeDecl() const
Definition CGCall.h:59
const FunctionProtoType * getCalleeFunctionProtoType() const
Definition CGCall.h:56
All available information about a concrete callee.
Definition CGCall.h:63
CGCallee prepareConcreteCallee(CodeGenFunction &CGF) const
If this is a delayed callee computation of some sort, prepare a concrete callee.
Definition CGCall.cpp:6275
bool isVirtual() const
Definition CGCall.h:204
Address getThisAddress() const
Definition CGCall.h:215
const CallExpr * getVirtualCallExpr() const
Definition CGCall.h:207
llvm::Value * getFunctionPointer() const
Definition CGCall.h:190
llvm::FunctionType * getVirtualFunctionType() const
Definition CGCall.h:219
const CGPointerAuthInfo & getPointerAuthInfo() const
Definition CGCall.h:186
GlobalDecl getVirtualMethodDecl() const
Definition CGCall.h:211
CGFunctionInfo - Class to encapsulate the information about a function definition.
bool usesInAlloca() const
Return true if this function uses inalloca arguments.
FunctionType::ExtInfo getExtInfo() const
bool isReturnsRetained() const
In ARC, whether this function retains its return value.
void Profile(llvm::FoldingSetNodeID &ID)
const_arg_iterator arg_begin() const
ArrayRef< ExtParameterInfo > getExtParameterInfos() const
CanQualType getReturnType() const
static CGFunctionInfo * create(unsigned llvmCC, bool instanceMethod, bool chainCall, bool delegateCall, const FunctionType::ExtInfo &extInfo, ArrayRef< ExtParameterInfo > paramInfos, CanQualType resultType, ArrayRef< CanQualType > argTypes, RequiredArgs required)
Definition CGCall.cpp:894
MutableArrayRef< ArgInfo > arguments()
const_arg_iterator arg_end() const
unsigned getEffectiveCallingConvention() const
getEffectiveCallingConvention - Return the actual calling convention to use, which may depend on the ...
ExtParameterInfo getExtParameterInfo(unsigned argIndex) const
CharUnits getArgStructAlignment() const
RequiredArgs getRequiredArgs() const
llvm::StructType * getArgStruct() const
Get the struct type used to represent all the arguments in memory.
CGRecordLayout - This class handles struct and union layout info while lowering AST types to LLVM typ...
const CGBitFieldInfo & getBitFieldInfo(const FieldDecl *FD) const
Return the BitFieldInfo that corresponds to the field FD.
CallArgList - Type for representing both the value and type of arguments in a call.
Definition CGCall.h:274
void addWriteback(LValue srcLV, Address temporary, llvm::Value *toUse, const Expr *writebackExpr=nullptr)
Definition CGCall.h:320
llvm::Instruction * getStackBase() const
Definition CGCall.h:348
void addUncopiedAggregate(LValue LV, QualType type)
Definition CGCall.h:304
void addArgCleanupDeactivation(EHScopeStack::stable_iterator Cleanup, llvm::Instruction *IsActiveIP)
Definition CGCall.h:335
ArrayRef< CallArgCleanup > getCleanupsToDeactivate() const
Definition CGCall.h:343
bool hasWritebacks() const
Definition CGCall.h:326
void add(RValue rvalue, QualType type)
Definition CGCall.h:302
bool isUsingInAlloca() const
Returns if we're using an inalloca struct to pass arguments in memory.
Definition CGCall.h:353
void allocateArgumentMemory(CodeGenFunction &CGF)
Definition CGCall.cpp:4570
void freeArgumentMemory(CodeGenFunction &CGF) const
Definition CGCall.cpp:4577
writeback_const_range writebacks() const
Definition CGCall.h:331
An abstract representation of regular/ObjC call/message targets.
const ParmVarDecl * getParamDecl(unsigned I) const
An object to manage conditionally-evaluated expressions.
static ParamValue forIndirect(Address addr)
static ParamValue forDirect(llvm::Value *value)
RAII object to set/unset CodeGenFunction::IsSanitizerScope.
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
void CreateCoercedStore(llvm::Value *Src, Address Dst, llvm::TypeSize DstSize, bool DstIsVolatile)
Create a store to.
Definition CGCall.cpp:1398
EHScopeStack::stable_iterator CurrentCleanupScopeDepth
llvm::Value * EmitARCRetainAutoreleaseReturnValue(llvm::Value *value)
Do a fused retain/autorelease of the given object.
Definition CGObjC.cpp:2599
SanitizerSet SanOpts
Sanitizers enabled for this function.
void checkTargetFeatures(const CallExpr *E, const FunctionDecl *TargetDecl)
static bool hasScalarEvaluationKind(QualType T)
llvm::Type * ConvertType(QualType T)
bool isCleanupPadScope() const
Returns true while emitting a cleanuppad.
void addInstToNewSourceAtom(llvm::Instruction *KeyInstruction, llvm::Value *Backup)
Add KeyInstruction and an optional Backup instruction to a new atom group (See ApplyAtomGroup for mor...
llvm::CallBase * EmitCallOrInvoke(llvm::FunctionCallee Callee, ArrayRef< llvm::Value * > Args, const Twine &Name="")
Emits a call or invoke instruction to the given function, depending on the current state of the EH st...
Definition CGCall.cpp:5095
void EmitNoreturnRuntimeCallOrInvoke(llvm::FunctionCallee callee, ArrayRef< llvm::Value * > args)
Emits a call or invoke to the given noreturn runtime function.
Definition CGCall.cpp:5058
llvm::CallBase * EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee, ArrayRef< llvm::Value * > args, const Twine &name="")
Emits a call or invoke instruction to the given runtime function.
Definition CGCall.cpp:5085
ComplexPairTy EmitLoadOfComplex(LValue src, SourceLocation loc)
EmitLoadOfComplex - Load a complex number from the specified l-value.
llvm::Value * EmitARCAutoreleaseReturnValue(llvm::Value *value)
Autorelease the given object.
Definition CGObjC.cpp:2589
bool CurFuncIsThunk
In C++, whether we are code generating a thunk.
bool isSEHTryScope() const
Returns true inside SEH __try blocks.
RValue convertTempToRValue(Address addr, QualType type, SourceLocation Loc)
Given the address of a temporary variable, produce an r-value of its type.
Definition CGExpr.cpp:6631
llvm::Constant * EmitCheckSourceLocation(SourceLocation Loc)
Emit a description of a source location in a format suitable for passing to a runtime sanitizer handl...
Definition CGExpr.cpp:3648
void SetSqrtFPAccuracy(llvm::Value *Val)
Set the minimum required accuracy of the given sqrt operation based on CodeGenOpts.
Definition CGExpr.cpp:6657
RValue EmitVAArg(VAArgExpr *VE, Address &VAListAddr, AggValueSlot Slot=AggValueSlot::ignored())
Generate code to get an argument from the passed in pointer and update it accordingly.
Definition CGCall.cpp:6288
void EmitReturnValueCheck(llvm::Value *RV)
Emit a test that checks if the return value RV is nonnull.
Definition CGCall.cpp:4207
llvm::Value * getAsNaturalPointerTo(Address Addr, QualType PointeeType)
void EmitDelegateCallArg(CallArgList &args, const VarDecl *param, SourceLocation loc)
EmitDelegateCallArg - We are performing a delegate call; that is, the current function is delegating ...
Definition CGCall.cpp:4294
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
void addInstToCurrentSourceAtom(llvm::Instruction *KeyInstruction, llvm::Value *Backup)
See CGDebugInfo::addInstToCurrentSourceAtom.
const LangOptions & getLangOpts() const
void addInstToSpecificSourceAtom(llvm::Instruction *KeyInstruction, llvm::Value *Backup, uint64_t Atom)
See CGDebugInfo::addInstToSpecificSourceAtom.
RValue EmitReferenceBindingToExpr(const Expr *E)
Emits a reference binding to the passed in expression.
Definition CGExpr.cpp:684
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
bool InNoConvergentAttributedStmt
True if the current statement has noconvergent attribute.
void pushDestroy(QualType::DestructionKind dtorKind, Address addr, QualType type)
pushDestroy - Push the standard destructor for the given type as at least a normal cleanup.
Definition CGDecl.cpp:2278
const CodeGen::CGBlockInfo * BlockInfo
void EmitKCFIOperandBundle(const CGCallee &Callee, SmallVectorImpl< llvm::OperandBundleDef > &Bundles)
Address makeNaturalAddressForPointer(llvm::Value *Ptr, QualType T, CharUnits Alignment=CharUnits::Zero(), bool ForPointeeType=false, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
Construct an address with the natural alignment of T.
void EmitCXXDestructorCall(const CXXDestructorDecl *D, CXXDtorType Type, bool ForVirtualBase, bool Delegating, Address This, QualType ThisTy)
Definition CGClass.cpp:2577
bool InNoMergeAttributedStmt
True if the current statement has nomerge attribute.
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
llvm::BasicBlock * getUnreachableBlock()
void EmitARCRelease(llvm::Value *value, ARCPreciseLifetime_t precise)
Release the given object.
Definition CGObjC.cpp:2481
JumpDest ReturnBlock
ReturnBlock - Unified return block.
@ ForceLeftToRight
! Language semantics require left-to-right evaluation.
@ ForceRightToLeft
! Language semantics require right-to-left evaluation.
void EmitNonNullArgCheck(RValue RV, QualType ArgType, SourceLocation ArgLoc, AbstractCallee AC, unsigned ParmNum)
Create a check for a function parameter that may potentially be declared as non-null.
Definition CGCall.cpp:4584
void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy, AggValueSlot::Overlap_t MayOverlap, bool isVolatile=false)
EmitAggregateCopy - Emit an aggregate copy.
const TargetInfo & getTarget() const
LValue EmitHLSLOutArgExpr(const HLSLOutArgExpr *E, CallArgList &Args, QualType Ty)
Definition CGExpr.cpp:5853
void EmitWritebacks(const CallArgList &Args)
EmitWriteback - Emit callbacks for function.
Definition CGCall.cpp:4875
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
Definition CGExpr.cpp:242
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
Definition CGExpr.cpp:2336
void DeactivateCleanupBlock(EHScopeStack::stable_iterator Cleanup, llvm::Instruction *DominatingIP)
DeactivateCleanupBlock - Deactivates the given cleanup block.
void pushFullExprCleanup(CleanupKind kind, As... A)
pushFullExprCleanup - Push a cleanup to be run at the end of the current full-expression.
void EmitCallArg(CallArgList &args, const Expr *E, QualType ArgType)
EmitCallArg - Emit a single call argument.
Definition CGCall.cpp:4880
void EmitPointerAuthOperandBundle(const CGPointerAuthInfo &Info, SmallVectorImpl< llvm::OperandBundleDef > &Bundles)
void EmitCheck(ArrayRef< std::pair< llvm::Value *, SanitizerKind::SanitizerOrdinal > > Checked, SanitizerHandler Check, ArrayRef< llvm::Constant * > StaticArgs, ArrayRef< llvm::Value * > DynamicArgs, const TrapReason *TR=nullptr)
Create a basic block that will either trap or call a handler function in the UBSan runtime with the p...
Definition CGExpr.cpp:3788
AggValueSlot CreateAggTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateAggTemp - Create a temporary memory object for the given aggregate type.
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
bool EmitLifetimeStart(llvm::Value *Addr)
Emit a lifetime.begin marker if some criteria are satisfied.
Definition CGDecl.cpp:1356
llvm::AllocaInst * CreateTempAlloca(llvm::Type *Ty, const Twine &Name="tmp", llvm::Value *ArraySize=nullptr)
CreateTempAlloca - This creates an alloca and inserts it into the entry block if ArraySize is nullptr...
Definition CGExpr.cpp:151
RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee, ReturnValueSlot ReturnValue, const CallArgList &Args, llvm::CallBase **CallOrInvoke, bool IsMustTail, SourceLocation Loc, bool IsVirtualFunctionPointerThunk=false)
EmitCall - Generate a call of the given function, expecting the given result type,...
Definition CGCall.cpp:5241
const TargetCodeGenInfo & getTargetHooks() const
void EmitLifetimeEnd(llvm::Value *Addr)
Definition CGDecl.cpp:1368
RawAddress CreateMemTempWithoutCast(QualType T, const Twine &Name="tmp")
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen without...
Definition CGExpr.cpp:215
bool InNoInlineAttributedStmt
True if the current statement has noinline attribute.
SmallVector< llvm::OperandBundleDef, 1 > getBundlesForFunclet(llvm::Value *Callee)
Definition CGCall.cpp:5023
RValue EmitAnyExprToTemp(const Expr *E)
EmitAnyExprToTemp - Similarly to EmitAnyExpr(), however, the result will always be accessible even if...
Definition CGExpr.cpp:283
llvm::CallInst * EmitNounwindRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit)
EmitStoreOfComplex - Store a complex number into the specified l-value.
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
void EmitFunctionProlog(const CGFunctionInfo &FI, llvm::Function *Fn, const FunctionArgList &Args)
EmitFunctionProlog - Emit the target specific LLVM code to load the arguments for the given function.
Definition CGCall.cpp:3106
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
Definition CGExpr.cpp:2533
Address EmitVAListRef(const Expr *E)
RValue GetUndefRValue(QualType Ty)
GetUndefRValue - Get an appropriate 'undef' rvalue for the given type.
Definition CGExpr.cpp:1532
void EmitParmDecl(const VarDecl &D, ParamValue Arg, unsigned ArgNo)
EmitParmDecl - Emit a ParmVarDecl or an ImplicitParamDecl.
Definition CGDecl.cpp:2653
bool AutoreleaseResult
In ARC, whether we should autorelease the return value.
llvm::CallInst * EmitRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
llvm::Value * EmitARCRetainNonBlock(llvm::Value *value)
Retain the given object, with normal retain semantics.
Definition CGObjC.cpp:2337
llvm::Type * ConvertTypeForMem(QualType T)
CodeGenTypes & getTypes() const
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
bool InAlwaysInlineAttributedStmt
True if the current statement has always_inline attribute.
void EmitFunctionEpilog(const CGFunctionInfo &FI, bool EmitRetDbgLoc, SourceLocation EndLoc, uint64_t RetKeyInstructionsSourceAtom)
EmitFunctionEpilog - Emit the target specific LLVM code to return the given temporary.
Definition CGCall.cpp:3993
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
Definition CGExpr.cpp:1515
RawAddress CreateMemTemp(QualType T, const Twine &Name="tmp", RawAddress *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
Definition CGExpr.cpp:186
void EmitAggExpr(const Expr *E, AggValueSlot AS)
EmitAggExpr - Emit the computation of the specified expression of aggregate type.
Address EmitMSVAListRef(const Expr *E)
Emit a "reference" to a __builtin_ms_va_list; this is always the value of the expression,...
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
static bool hasAggregateEvaluationKind(QualType T)
void EmitCallArgs(CallArgList &Args, PrototypeWrapper Prototype, llvm::iterator_range< CallExpr::const_arg_iterator > ArgRange, AbstractCallee AC=AbstractCallee(), unsigned ParamsToSkip=0, EvaluationOrder Order=EvaluationOrder::Default)
EmitCallArgs - Emit call arguments for a function.
Definition CGCall.cpp:4681
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
void EmitUnreachable(SourceLocation Loc)
Emit a reached-unreachable diagnostic if Loc is valid and runtime checking is enabled.
Definition CGExpr.cpp:4128
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
Address ReturnValue
ReturnValue - The temporary alloca to hold the return value.
LValue EmitLValue(const Expr *E, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
Definition CGExpr.cpp:1631
llvm::Instruction * CurrentFuncletPad
void EnsureInsertPoint()
EnsureInsertPoint - Ensure that an insertion point is defined so that emitted IR has a place to go.
llvm::LLVMContext & getLLVMContext()
void emitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty, SourceLocation Loc, SourceLocation AssumptionLoc, llvm::Value *Alignment, llvm::Value *OffsetValue=nullptr)
void EmitVariablyModifiedType(QualType Ty)
EmitVLASize - Capture all the sizes for the VLA expressions in the given variably-modified type and s...
llvm::Value * EmitNonNullRValueCheck(RValue RV, QualType T)
Create a check that a scalar RValue is non-null.
Definition CGExpr.cpp:1525
void EmitARCIntrinsicUse(ArrayRef< llvm::Value * > values)
Given a number of pointers, inform the optimizer that they're being intrinsically used up until this ...
Definition CGObjC.cpp:2167
llvm::Value * EmitCMSEClearRecord(llvm::Value *V, llvm::IntegerType *ITy, QualType RTy)
Definition CGCall.cpp:3947
void PopCleanupBlock(bool FallThroughIsBranchThrough=false, bool ForDeactivation=false)
PopCleanupBlock - Will pop the cleanup entry on the stack and process all branch fixups.
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
Definition CGStmt.cpp:652
This class organizes the cross-function state that is used while generating LLVM code.
bool ReturnTypeUsesFPRet(QualType ResultType)
Return true iff the given type uses 'fpret' when used as a return type.
Definition CGCall.cpp:1668
const LangOptions & getLangOpts() const
CharUnits getNaturalTypeAlignment(QualType T, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, bool forPointeeType=false)
const TargetInfo & getTarget() const
const llvm::DataLayout & getDataLayout() const
ObjCEntrypoints & getObjCEntrypoints() const
bool ReturnTypeUsesFP2Ret(QualType ResultType)
Return true iff the given type uses 'fp2ret' when used as a return type.
Definition CGCall.cpp:1685
bool ReturnSlotInterferesWithArgs(const CGFunctionInfo &FI)
Return true iff the given type uses an argument slot when 'sret' is used as a return type.
Definition CGCall.cpp:1663
bool ReturnTypeHasInReg(const CGFunctionInfo &FI)
Return true iff the given type has inreg set.
Definition CGCall.cpp:1658
void AdjustMemoryAttribute(StringRef Name, CGCalleeInfo CalleeInfo, llvm::AttributeList &Attrs)
Adjust Memory attribute to ensure that the BE gets the right attribute.
Definition CGCall.cpp:2381
void ConstructAttributeList(StringRef Name, const CGFunctionInfo &Info, CGCalleeInfo CalleeInfo, llvm::AttributeList &Attrs, unsigned &CallingConv, bool AttrOnCallSite, bool IsThunk)
Get the LLVM attributes and calling convention to use for a particular function type.
Definition CGCall.cpp:2409
ASTContext & getContext() const
bool ReturnTypeUsesSRet(const CGFunctionInfo &FI)
Return true iff the given type uses 'sret' when used as a return type.
Definition CGCall.cpp:1653
const TargetCodeGenInfo & getTargetCodeGenInfo()
const CodeGenOptions & getCodeGenOpts() const
void addDefaultFunctionDefinitionAttributes(llvm::AttrBuilder &attrs)
Like the overload taking a Function &, but intended specifically for frontends that want to build on ...
Definition CGCall.cpp:2235
CharUnits getNaturalPointeeTypeAlignment(QualType T, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
llvm::LLVMContext & getLLVMContext()
CharUnits getMinimumObjectSize(QualType Ty)
Returns the minimum object size for an object of the given type.
bool MayDropFunctionReturn(const ASTContext &Context, QualType ReturnType) const
Whether this function's return type has no side effects, and thus may be trivially discarded if it is...
Definition CGCall.cpp:1893
This class organizes the cross-module state that is used while lowering AST types to LLVM types.
const CGFunctionInfo & arrangeCXXMethodType(const CXXRecordDecl *RD, const FunctionProtoType *FTP, const CXXMethodDecl *MD)
Arrange the argument and result information for a call to an unknown C++ non-static member function o...
Definition CGCall.cpp:345
llvm::Type * ConvertType(QualType T)
ConvertType - Convert type T into a llvm::Type.
CGCXXABI & getCXXABI() const
const CGFunctionInfo & arrangeCXXMethodDeclaration(const CXXMethodDecl *MD)
C++ methods have some special rules and also have implicit parameters.
Definition CGCall.cpp:373
ASTContext & getContext() const
const CGFunctionInfo & arrangeLLVMFunctionInfo(CanQualType returnType, FnInfoOpts opts, ArrayRef< CanQualType > argTypes, FunctionType::ExtInfo info, ArrayRef< FunctionProtoType::ExtParameterInfo > paramInfos, RequiredArgs args)
"Arrange" the LLVM information for a call or type with the given signature.
Definition CGCall.cpp:830
const CGFunctionInfo & arrangeFreeFunctionType(CanQual< FunctionProtoType > Ty)
Arrange the argument and result information for a value of the given freestanding function type.
Definition CGCall.cpp:249
CanQualType DeriveThisType(const CXXRecordDecl *RD, const CXXMethodDecl *MD)
Derives the 'this' type for codegen purposes, i.e.
Definition CGCall.cpp:126
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
Definition CGCall.cpp:1701
bool inheritingCtorHasParams(const InheritedConstructor &Inherited, CXXCtorType Type)
Determine if a C++ inheriting constructor should have parameters matching those of its inherited cons...
Definition CGCall.cpp:391
bool isFuncTypeConvertible(const FunctionType *FT)
isFuncTypeConvertible - Utility to check whether a function type can be converted to an LLVM type (i....
const CGFunctionInfo & arrangeBlockFunctionCall(const CallArgList &args, const FunctionType *type)
A block function is essentially a free function with an extra implicit argument.
Definition CGCall.cpp:708
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
Definition CGCall.cpp:739
const CGFunctionInfo & arrangeUnprototypedObjCMessageSend(QualType returnType, const CallArgList &args)
Definition CGCall.cpp:602
const CGRecordLayout & getCGRecordLayout(const RecordDecl *)
getCGRecordLayout - Return record layout info for the given record decl.
void getExpandedTypes(QualType Ty, SmallVectorImpl< llvm::Type * >::iterator &TI)
getExpandedTypes - Expand the type
Definition CGCall.cpp:1074
llvm::Type * ConvertTypeForMem(QualType T)
ConvertTypeForMem - Convert type T into a llvm::Type.
const CGFunctionInfo & arrangeObjCMethodDeclaration(const ObjCMethodDecl *MD)
Objective-C methods are C functions with some implicit parameters.
Definition CGCall.cpp:555
llvm::LLVMContext & getLLVMContext()
const CGFunctionInfo & arrangeDeviceKernelCallerDeclaration(QualType resultType, const FunctionArgList &args)
A device kernel caller function is an offload device entry point function with a target device depend...
Definition CGCall.cpp:755
const CGFunctionInfo & arrangeGlobalDeclaration(GlobalDecl GD)
Definition CGCall.cpp:611
const CGFunctionInfo & arrangeUnprototypedMustTailThunk(const CXXMethodDecl *MD)
Arrange a thunk that takes 'this' as the first parameter followed by varargs.
Definition CGCall.cpp:626
const CGFunctionInfo & arrangeCXXMethodCall(const CallArgList &args, const FunctionProtoType *type, RequiredArgs required, unsigned numPrefixArgs)
Arrange a call to a C++ method, passing the given arguments.
Definition CGCall.cpp:769
const CGFunctionInfo & arrangeFreeFunctionCall(const CallArgList &Args, const FunctionType *Ty, bool ChainCall)
Figure out the rules for calling a function with the given formal type using the given arguments.
Definition CGCall.cpp:699
const CGFunctionInfo & arrangeBuiltinFunctionCall(QualType resultType, const CallArgList &args)
Definition CGCall.cpp:728
const CGFunctionInfo & arrangeBlockFunctionDeclaration(const FunctionProtoType *type, const FunctionArgList &args)
Block invocation functions are C functions with an implicit parameter.
Definition CGCall.cpp:715
unsigned ClangCallConvToLLVMCallConv(CallingConv CC)
Convert clang calling convention to LLVM callilng convention.
Definition CGCall.cpp:52
llvm::Type * GetFunctionTypeForVTable(GlobalDecl GD)
GetFunctionTypeForVTable - Get the LLVM function type for use in a vtable, given a CXXMethodDecl.
Definition CGCall.cpp:1829
const CGFunctionInfo & arrangeCXXConstructorCall(const CallArgList &Args, const CXXConstructorDecl *D, CXXCtorType CtorKind, unsigned ExtraPrefixArgs, unsigned ExtraSuffixArgs, bool PassProtoArgs=true)
Arrange a call to a C++ method, passing the given arguments.
Definition CGCall.cpp:484
const CGFunctionInfo & arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD, QualType receiverType)
Arrange the argument and result information for the function type through which to perform a send to ...
Definition CGCall.cpp:568
const CGFunctionInfo & arrangeCXXStructorDeclaration(GlobalDecl GD)
Definition CGCall.cpp:401
const CGFunctionInfo & arrangeFunctionDeclaration(const GlobalDecl GD)
Free functions are functions that are compatible with an ordinary C function pointer type.
Definition CGCall.cpp:523
const CGFunctionInfo & arrangeMSCtorClosure(const CXXConstructorDecl *CD, CXXCtorType CT)
Definition CGCall.cpp:635
const CGFunctionInfo & arrangeCall(const CGFunctionInfo &declFI, const CallArgList &args)
Given a function info for a declaration, return the function info for a call with the given arguments...
Definition CGCall.cpp:793
const CGFunctionInfo & arrangeNullaryFunction()
A nullary function is a freestanding function of type 'void ()'.
Definition CGCall.cpp:787
A cleanup scope which generates the cleanup blocks lazily.
Definition CGCleanup.h:247
A saved depth on the scope stack.
FunctionArgList - Type for representing both the decl and type of parameters to a function.
Definition CGCall.h:375
LValue - This represents an lvalue references.
Definition CGValue.h:182
bool isSimple() const
Definition CGValue.h:278
static LValue MakeAddr(Address Addr, QualType type, ASTContext &Context, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo)
Definition CGValue.h:432
Address getAddress() const
Definition CGValue.h:361
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
Definition CGValue.h:42
bool isScalar() const
Definition CGValue.h:64
static RValue get(llvm::Value *V)
Definition CGValue.h:98
static RValue getAggregate(Address addr, bool isVolatile=false)
Convert an Address to an RValue.
Definition CGValue.h:125
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
Definition CGValue.h:108
Address getAggregateAddress() const
getAggregateAddr() - Return the Value* of the address of the aggregate.
Definition CGValue.h:83
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
Definition CGValue.h:71
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
Definition CGValue.h:78
An abstract representation of an aligned address.
Definition Address.h:42
CharUnits getAlignment() const
Return the alignment of this pointer.
Definition Address.h:93
llvm::Value * getPointer() const
Definition Address.h:66
static RawAddress invalid()
Definition Address.h:61
A class for recording the number of arguments that a function signature requires.
unsigned getNumRequiredArgs() const
static RequiredArgs forPrototypePlus(const FunctionProtoType *prototype, unsigned additional)
Compute the arguments required by the given formal prototype, given that there may be some additional...
ReturnValueSlot - Contains the address where the return value of a function can be stored,...
Definition CGCall.h:379
virtual void setCUDAKernelCallingConvention(const FunctionType *&FT) const
Definition TargetInfo.h:405
static void initPointerAuthFnAttributes(const PointerAuthOptions &Opts, llvm::AttrBuilder &FuncAttrs)
static void initBranchProtectionFnAttributes(const TargetInfo::BranchProtectionInfo &BPI, llvm::AttrBuilder &FuncAttrs)
virtual bool isNoProtoCallVariadic(const CodeGen::CallArgList &args, const FunctionNoProtoType *fnType) const
Determine whether a call to an unprototyped functions under the given calling convention should use t...
Complex values, per C99 6.2.5p11.
Definition TypeBase.h:3275
Represents the canonical version of C arrays with a specified constant size.
Definition TypeBase.h:3758
bool constructsVirtualBase() const
Returns true if the constructed base class is a virtual base class subobject of this declaration's cl...
Definition DeclCXX.h:3777
DeclContext - This is used only as base class of specific decl types that can act as declaration cont...
Definition DeclBase.h:1449
Decl - This represents one declaration (or definition), e.g.
Definition DeclBase.h:86
T * getAttr() const
Definition DeclBase.h:573
llvm::iterator_range< specific_attr_iterator< T > > specific_attrs() const
Definition DeclBase.h:559
DeclContext * getDeclContext()
Definition DeclBase.h:448
bool hasAttr() const
Definition DeclBase.h:577
SourceLocation getBeginLoc() const LLVM_READONLY
Definition Decl.h:830
This represents one expression.
Definition Expr.h:112
bool isGLValue() const
Definition Expr.h:287
Expr * IgnoreParens() LLVM_READONLY
Skip past any parentheses which might surround this expression until reaching a fixed point.
Definition Expr.cpp:3081
@ NPC_ValueDependentIsNotNull
Specifies that a value-dependent expression should be considered to never be a null pointer constant.
Definition Expr.h:835
ExprObjectKind getObjectKind() const
getObjectKind - The object kind that this expression produces.
Definition Expr.h:451
NullPointerConstantKind isNullPointerConstant(ASTContext &Ctx, NullPointerConstantValueDependence NPC) const
isNullPointerConstant - C99 6.3.2.3p3 - Test if this reduces down to a Null pointer constant.
Definition Expr.cpp:4042
QualType getType() const
Definition Expr.h:144
Represents a member of a struct/union/class.
Definition Decl.h:3157
bool isBitField() const
Determines whether this field is a bitfield.
Definition Decl.h:3260
bool isUnnamedBitField() const
Determines whether this is an unnamed bitfield.
Definition Decl.h:3263
bool isZeroLengthBitField() const
Is this a zero-length bit-field?
Definition Decl.cpp:4702
Represents a function declaration or definition.
Definition Decl.h:1999
bool isTrivial() const
Whether this function is "trivial" in some specialized C++ senses.
Definition Decl.h:2376
Represents a K&R-style 'int foo()' function, which has no information available about its arguments.
Definition TypeBase.h:4842
Represents a prototype with parameter type info, e.g.
Definition TypeBase.h:5264
ExceptionSpecificationType getExceptionSpecType() const
Get the kind of exception specification on this function.
Definition TypeBase.h:5571
unsigned getNumParams() const
Definition TypeBase.h:5542
unsigned getAArch64SMEAttributes() const
Return a bitmask describing the SME attributes on the function type, see AArch64SMETypeAttributes for...
Definition TypeBase.h:5761
bool isNothrow(bool ResultIfDependent=false) const
Determine whether this function type has a non-throwing exception specification.
Definition TypeBase.h:5663
ArrayRef< ExtParameterInfo > getExtParameterInfos() const
Definition TypeBase.h:5737
bool hasExtParameterInfos() const
Is there any interesting extra information for any of the parameters of this function type?
Definition TypeBase.h:5733
Wrapper for source info for functions.
Definition TypeLoc.h:1624
A class which abstracts out some details necessary for making a call.
Definition TypeBase.h:4571
ExtInfo withCallingConv(CallingConv cc) const
Definition TypeBase.h:4683
CallingConv getCC() const
Definition TypeBase.h:4630
ExtInfo withProducesResult(bool producesResult) const
Definition TypeBase.h:4649
unsigned getRegParm() const
Definition TypeBase.h:4623
bool getNoCallerSavedRegs() const
Definition TypeBase.h:4619
Interesting information about a specific parameter that can't simply be reflected in parameter's type...
Definition TypeBase.h:4486
ParameterABI getABI() const
Return the ABI treatment of this parameter.
Definition TypeBase.h:4499
ExtParameterInfo withIsNoEscape(bool NoEscape) const
Definition TypeBase.h:4526
FunctionType - C99 6.7.5.3 - Function Declarators.
Definition TypeBase.h:4460
ExtInfo getExtInfo() const
Definition TypeBase.h:4816
static ArmStateValue getArmZT0State(unsigned AttrBits)
Definition TypeBase.h:4769
static ArmStateValue getArmZAState(unsigned AttrBits)
Definition TypeBase.h:4765
QualType getReturnType() const
Definition TypeBase.h:4800
GlobalDecl - represents a global declaration.
Definition GlobalDecl.h:57
CXXCtorType getCtorType() const
Definition GlobalDecl.h:108
KernelReferenceKind getKernelReferenceKind() const
Definition GlobalDecl.h:135
const Decl * getDecl() const
Definition GlobalDecl.h:106
This class represents temporary values used to represent inout and out arguments in HLSL.
Definition Expr.h:7271
Description of a constructor that was inherited from a base class.
Definition DeclCXX.h:2575
ConstructorUsingShadowDecl * getShadowDecl() const
Definition DeclCXX.h:2587
@ FPE_Ignore
Assume that floating-point exceptions are masked.
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
std::vector< std::string > NoBuiltinFuncs
A list of all -fno-builtin-* function names (e.g., memset).
FPExceptionModeKind getDefaultExceptionMode() const
bool isNoBuiltinFunc(StringRef Name) const
Is this a libc/libm function that is no longer recognized as a builtin because a -fno-builtin-* optio...
bool assumeFunctionsAreConvergent() const
Represents a matrix type, as defined in the Matrix Types clang extensions.
Definition TypeBase.h:4335
Describes a module or submodule.
Definition Module.h:144
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
Definition Decl.h:300
ObjCCategoryDecl - Represents a category declaration.
Definition DeclObjC.h:2329
ObjCIndirectCopyRestoreExpr - Represents the passing of a function argument by indirect copy-restore ...
Definition ExprObjC.h:1582
bool shouldCopy() const
shouldCopy - True if we should do the 'copy' part of the copy-restore.
Definition ExprObjC.h:1610
Represents an ObjC class declaration.
Definition DeclObjC.h:1154
ObjCMethodDecl - Represents an instance or class method declaration.
Definition DeclObjC.h:140
ImplicitParamDecl * getSelfDecl() const
Definition DeclObjC.h:418
ArrayRef< ParmVarDecl * > parameters() const
Definition DeclObjC.h:373
bool isVariadic() const
Definition DeclObjC.h:431
bool isDirectMethod() const
True if the method is tagged as objc_direct.
Definition DeclObjC.cpp:868
QualType getReturnType() const
Definition DeclObjC.h:329
Represents a parameter to a function.
Definition Decl.h:1789
PointerType - C99 6.7.5.1 - Pointer Declarators.
Definition TypeBase.h:3328
QualType getPointeeType() const
Definition TypeBase.h:3338
A (possibly-)qualified type.
Definition TypeBase.h:937
bool isRestrictQualified() const
Determine whether this type is restrict-qualified.
Definition TypeBase.h:8363
bool isTriviallyCopyableType(const ASTContext &Context) const
Return true if this is a trivially copyable type (C++0x [basic.types]p9)
Definition Type.cpp:2867
LangAS getAddressSpace() const
Return the address space of this type.
Definition TypeBase.h:8411
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
Definition TypeBase.h:8325
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
Definition TypeBase.h:8470
QualType getCanonicalType() const
Definition TypeBase.h:8337
bool isConstQualified() const
Determine whether this type is const-qualified.
Definition TypeBase.h:8358
DestructionKind isDestructedType() const
Returns a nonzero value if objects of this type require non-trivial work to clean up after.
Definition TypeBase.h:1545
@ OCL_Strong
Assigning into this object requires the old value to be released and the new value to be retained.
Definition TypeBase.h:361
LangAS getAddressSpace() const
Definition TypeBase.h:571
Represents a struct/union/class.
Definition Decl.h:4309
field_iterator field_end() const
Definition Decl.h:4515
bool isParamDestroyedInCallee() const
Definition Decl.h:4459
field_iterator field_begin() const
Definition Decl.cpp:5154
Base for LValueReferenceType and RValueReferenceType.
Definition TypeBase.h:3571
Encodes a location in the source.
UIntTy getRawEncoding() const
When a SourceLocation itself cannot be used, this returns an (opaque) 32-bit integer encoding for it.
bool isMicrosoft() const
Is this ABI an MSVC-compatible ABI?
const llvm::Triple & getTriple() const
Returns the target triple of the primary target.
TargetCXXABI getCXXABI() const
Get the C++ ABI currently in use.
Options for controlling the target.
std::vector< std::string > Features
The list of target specific features to enable or disable – this should be a list of strings starting...
std::string TuneCPU
If given, the name of the target CPU to tune code for.
std::string CPU
If given, the name of the target CPU to generate code for.
bool isVoidType() const
Definition TypeBase.h:8878
bool isIncompleteArrayType() const
Definition TypeBase.h:8629
CXXRecordDecl * getAsCXXRecordDecl() const
Retrieves the CXXRecordDecl that this type refers to, either because the type is a RecordType or beca...
Definition Type.h:26
RecordDecl * getAsRecordDecl() const
Retrieves the RecordDecl this type refers to.
Definition Type.h:41
bool isConstantSizeType() const
Return true if this is not a variable sized type, according to the rules of C99 6....
Definition Type.cpp:2426
bool isPointerType() const
Definition TypeBase.h:8522
CanQualType getCanonicalTypeUnqualified() const
bool isIntegerType() const
isIntegerType() does not include complex integers (a GCC extension).
Definition TypeBase.h:8922
const T * castAs() const
Member-template castAs<specific type>.
Definition TypeBase.h:9165
bool isReferenceType() const
Definition TypeBase.h:8546
bool isScalarType() const
Definition TypeBase.h:8980
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
Definition Type.cpp:752
bool isBitIntType() const
Definition TypeBase.h:8787
RecordDecl * castAsRecordDecl() const
Definition Type.h:48
bool isMemberPointerType() const
Definition TypeBase.h:8603
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
Definition TypeBase.h:2800
bool isObjectType() const
Determine whether this type is an object type.
Definition TypeBase.h:2510
bool isIncompleteType(NamedDecl **Def=nullptr) const
Types are partitioned into 3 broad categories (C99 6.2.5p1): object types, function types,...
Definition Type.cpp:2436
bool hasFloatingRepresentation() const
Determine whether this type has a floating-point representation of some sort, e.g....
Definition Type.cpp:2312
const T * getAsCanonical() const
If this type is canonically the specified type, return its canonical type cast to that specified type...
Definition TypeBase.h:2921
const T * castAsCanonical() const
Return this type's canonical type cast to the specified type.
Definition TypeBase.h:2928
const T * getAs() const
Member-template getAs<specific type>'.
Definition TypeBase.h:9098
bool isNullPtrType() const
Definition TypeBase.h:8915
bool isRecordType() const
Definition TypeBase.h:8649
bool isObjCRetainableType() const
Definition Type.cpp:5291
UnaryOperator - This represents the unary-expression's (except sizeof and alignof),...
Definition Expr.h:2244
Represents a call to the builtin function __builtin_va_arg.
Definition Expr.h:4891
QualType getType() const
Definition Decl.h:722
Represents a variable declaration or definition.
Definition Decl.h:925
QualType::DestructionKind needsDestruction(const ASTContext &Ctx) const
Would the destruction of this variable have any effect, and if so, what kind?
Definition Decl.cpp:2851
Represents a GCC generic vector type.
Definition TypeBase.h:4173
Defines the clang::TargetInfo interface.
void computeABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI)
Compute the ABI information of a swiftcall function.
@ Type
The l-value was considered opaque, so the alignment was determined from a type.
Definition CGValue.h:154
@ Decl
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
Definition CGValue.h:145
void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI)
Definition SPIR.cpp:214
@ NormalCleanup
Denotes a cleanup that should run when a scope is exited using normal control flow (falling off the e...
void mergeDefaultFunctionDefinitionAttributes(llvm::Function &F, const CodeGenOptions &CodeGenOpts, const LangOptions &LangOpts, const TargetOptions &TargetOpts, bool WillInternalize)
Adds attributes to F according to our CodeGenOpts and LangOpts, as though we had emitted it ourselves...
Definition CGCall.cpp:2147
bool isEmptyRecord(ASTContext &Context, QualType T, bool AllowArrays, bool AsIfNoUniqueAddr=false)
isEmptyRecord - Return true iff a structure contains only empty fields.
VE builtins.
const internal::VariadicAllOfMatcher< Type > type
Matches Types in the clang AST.
bool This(InterpState &S, CodePtr OpPC)
Definition Interp.h:2816
bool Ret(InterpState &S, CodePtr &PC)
Definition Interp.h:312
RangeSelector name(std::string ID)
Given a node with a "name", (like NamedDecl, DeclRefExpr, CxxCtorInitializer, and TypeLoc) selects th...
The JSON file list parser is used to communicate input to InstallAPI.
CanQual< Type > CanQualType
Represents a canonical, potentially-qualified type.
CXXCtorType
C++ constructor types.
Definition ABI.h:24
@ Ctor_DefaultClosure
Default closure variant of a ctor.
Definition ABI.h:29
@ Ctor_CopyingClosure
Copying closure variant of a ctor.
Definition ABI.h:28
@ Ctor_Complete
Complete object ctor.
Definition ABI.h:25
bool isa(CodeGen::Address addr)
Definition Address.h:330
bool isUnresolvedExceptionSpec(ExceptionSpecificationType ESpecType)
bool isInstanceMethod(const Decl *D)
Definition Attr.h:120
@ NonNull
Values of this type can never be null.
Definition Specifiers.h:350
@ OK_Ordinary
An ordinary object is located at an address in memory.
Definition Specifiers.h:151
@ Vector
'vector' clause, allowed on 'loop', Combined, and 'routine' directives.
nullptr
This class represents a compute construct, representing a 'Kind' of ‘parallel’, 'serial',...
Expr * Cond
};
static bool classof(const Stmt *T)
Linkage
Describes the different kinds of linkage (C++ [basic.link], C99 6.2.2) that an entity may have.
Definition Linkage.h:24
@ SwiftAsyncContext
This parameter (which must have pointer type) uses the special Swift asynchronous context-pointer ABI...
Definition Specifiers.h:399
@ SwiftErrorResult
This parameter (which must have pointer-to-pointer type) uses the special Swift error-result ABI trea...
Definition Specifiers.h:389
@ Ordinary
This parameter uses ordinary ABI rules for its type.
Definition Specifiers.h:380
@ SwiftIndirectResult
This parameter (which must have pointer type) is a Swift indirect result parameter.
Definition Specifiers.h:384
@ SwiftContext
This parameter (which must have pointer type) uses the special Swift context-pointer ABI treatment.
Definition Specifiers.h:394
const FunctionProtoType * T
@ Dtor_Complete
Complete object dtor.
Definition ABI.h:36
LangAS
Defines the address space values used by the address space qualifier of QualType.
@ CanPassInRegs
The argument of this type can be passed directly in registers.
Definition Decl.h:4288
CallingConv
CallingConv - Specifies the calling convention that a function uses.
Definition Specifiers.h:278
@ CC_X86Pascal
Definition Specifiers.h:284
@ CC_Swift
Definition Specifiers.h:293
@ CC_IntelOclBicc
Definition Specifiers.h:290
@ CC_PreserveMost
Definition Specifiers.h:295
@ CC_Win64
Definition Specifiers.h:285
@ CC_X86ThisCall
Definition Specifiers.h:282
@ CC_AArch64VectorCall
Definition Specifiers.h:297
@ CC_DeviceKernel
Definition Specifiers.h:292
@ CC_AAPCS
Definition Specifiers.h:288
@ CC_PreserveNone
Definition Specifiers.h:300
@ CC_M68kRTD
Definition Specifiers.h:299
@ CC_SwiftAsync
Definition Specifiers.h:294
@ CC_X86RegCall
Definition Specifiers.h:287
@ CC_RISCVVectorCall
Definition Specifiers.h:301
@ CC_X86VectorCall
Definition Specifiers.h:283
@ CC_SpirFunction
Definition Specifiers.h:291
@ CC_AArch64SVEPCS
Definition Specifiers.h:298
@ CC_X86StdCall
Definition Specifiers.h:280
@ CC_X86_64SysV
Definition Specifiers.h:286
@ CC_PreserveAll
Definition Specifiers.h:296
@ CC_X86FastCall
Definition Specifiers.h:281
@ CC_AAPCS_VFP
Definition Specifiers.h:289
U cast(CodeGen::Address addr)
Definition Address.h:327
LangAS getLangASFromTargetAS(unsigned TargetAS)
unsigned long uint64_t
__DEVICE__ _Tp arg(const std::complex< _Tp > &__c)
Structure with information about how a bitfield should be accessed.
CharUnits StorageOffset
The offset of the bitfield storage from the start of the struct.
unsigned Offset
The offset within a contiguous run of bitfields that are represented as a single "field" within the L...
unsigned Size
The total size of the bit-field, in bits.
unsigned StorageSize
The storage size in bits which should be used when accessing this bitfield.
Similar to AddedStructorArgs, but only notes the number of additional arguments.
Definition CGCXXABI.h:358
llvm::Value * ToUse
A value to "use" after the writeback, or null.
Definition CGCall.h:287
LValue Source
The original argument.
Definition CGCall.h:281
Address Temporary
The temporary alloca.
Definition CGCall.h:284
const Expr * WritebackExpr
An Expression (optional) that performs the writeback with any required casting.
Definition CGCall.h:291
LValue getKnownLValue() const
Definition CGCall.h:254
RValue getKnownRValue() const
Definition CGCall.h:258
void copyInto(CodeGenFunction &CGF, Address A) const
Definition CGCall.cpp:4858
bool hasLValue() const
Definition CGCall.h:247
RValue getRValue(CodeGenFunction &CGF) const
Definition CGCall.cpp:4848
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
llvm::CallingConv::ID getRuntimeCC() const
DisableDebugLocationUpdates(CodeGenFunction &CGF)
Definition CGCall.cpp:6300
static const EHPersonality & get(CodeGenModule &CGM, const FunctionDecl *FD)
llvm::Function * objc_retainAutoreleasedReturnValue
id objc_retainAutoreleasedReturnValue(id);
llvm::Function * objc_retain
id objc_retain(id);
llvm::InlineAsm * retainAutoreleasedReturnValueMarker
A void(void) inline asm to use to mark that the return value of a call will be immediately retain.
bool has(SanitizerMask K) const
Check if a certain (single) sanitizer is enabled.
Definition Sanitizers.h:174