//===- SPIRVWriter.cpp - Converts LLVM to SPIR-V ----------------*- C++ -*-===//

//

// The LLVM/SPIR-V Translator

//

// This file is distributed under the University of Illinois Open Source

// License. See LICENSE.TXT for details.

//

// Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved.

//

// Permission is hereby granted, free of charge, to any person obtaining a

// copy of this software and associated documentation files (the "Software"),

// to deal with the Software without restriction, including without limitation

// the rights to use, copy, modify, merge, publish, distribute, sublicense,

// and/or sell copies of the Software, and to permit persons to whom the

// Software is furnished to do so, subject to the following conditions:

//

// Redistributions of source code must retain the above copyright notice,

// this list of conditions and the following disclaimers.

// Redistributions in binary form must reproduce the above copyright notice,

// this list of conditions and the following disclaimers in the documentation

// and/or other materials provided with the distribution.

// Neither the names of Advanced Micro Devices, Inc., nor the names of its

// contributors may be used to endorse or promote products derived from this

// Software without specific prior written permission.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

// CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH

// THE SOFTWARE.

//

//===----------------------------------------------------------------------===//

/// \file

///

/// This file implements conversion of LLVM intermediate language to SPIR-V

/// binary.

///

//===----------------------------------------------------------------------===//

#include "SPIRVWriter.h"

#include "LLVMToSPIRVDbgTran.h"

#include "OCLToSPIRV.h"

#include "PreprocessMetadata.h"

#include "SPIRVAsm.h"

#include "SPIRVBasicBlock.h"

#include "SPIRVEntry.h"

#include "SPIRVEnum.h"

#include "SPIRVExtInst.h"

#include "SPIRVFunction.h"

#include "SPIRVInstruction.h"

#include "SPIRVInternal.h"

#include "SPIRVLLVMUtil.h"

#include "SPIRVLowerBitCastToNonStandardType.h"

#include "SPIRVLowerBool.h"

#include "SPIRVLowerConstExpr.h"

#include "SPIRVLowerMemmove.h"

#include "SPIRVLowerOCLBlocks.h"

#include "SPIRVLowerSaddWithOverflow.h"

#include "SPIRVMDWalker.h"

#include "SPIRVMemAliasingINTEL.h"

#include "SPIRVModule.h"

#include "SPIRVRegularizeLLVM.h"

#include "SPIRVType.h"

#include "SPIRVUtil.h"

#include "SPIRVValue.h"

#include "VectorComputeUtil.h"

#include "llvm/ADT/DenseMap.h"

#include "llvm/ADT/StringSwitch.h"

#include "llvm/Analysis/LoopAnalysisManager.h"

#include "llvm/Analysis/LoopInfo.h"

#include "llvm/Analysis/ValueTracking.h"

#include "llvm/IR/Constants.h"

#include "llvm/IR/DerivedTypes.h"

#include "llvm/IR/Dominators.h"

#include "llvm/IR/Function.h"

#include "llvm/IR/InlineAsm.h"

#include "llvm/IR/InstrTypes.h"

#include "llvm/IR/Instructions.h"

#include "llvm/IR/IntrinsicInst.h"

#include "llvm/IR/Module.h"

#include "llvm/IR/Operator.h"

#include "llvm/IR/TypedPointerType.h"

#include "llvm/Pass.h"

#include "llvm/Passes/PassBuilder.h"

#include "llvm/Support/Casting.h"

#include "llvm/Support/CommandLine.h"

#include "llvm/Support/Debug.h"

#include "llvm/TargetParser/Triple.h"

#include "llvm/Transforms/Utils/LoopSimplify.h"

#include "llvm/Transforms/Utils/Mem2Reg.h"

#include <cstdlib>

#include <functional>

#include <iostream>

#include <memory>

#include <queue>

#include <regex>

#include <set>

#include <vector>

#define DEBUG_TYPE "spirv"

using namespace llvm;

using namespace SPIRV;

using namespace OCLUtil;

namespace {

static SPIRVWord convertFloatToSPIRVWord(float F) {

union {

float F;

SPIRVWord Spir;

} FPMaxError;

FPMaxError.F = F;

return FPMaxError.Spir;

}

} // namespace

namespace SPIRV {

static void foreachKernelArgMD(

MDNode *MD, SPIRVFunction *BF,

std::function<void(const std::string &Str, SPIRVFunctionParameter *BA)>

Func) {

assert(BF->getNumArguments() == MD->getNumOperands() &&

"Invalid kernel metadata: Number of metadata operands and kernel "

"arguments do not match");

for (unsigned I = 0, E = MD->getNumOperands(); I != E; ++I) {

SPIRVFunctionParameter *BA = BF->getArgument(I);

Func(getMDOperandAsString(MD, I).str(), BA);

}

}

static void foreachKernelArgMD(

MDNode *MD, SPIRVFunction *BF,

std::function<void(Metadata *MDOp, SPIRVFunctionParameter *BA)> Func) {

assert(BF->getNumArguments() == MD->getNumOperands() &&

"Invalid kernel metadata: Number of metadata operands and kernel "

"arguments do not match");

for (unsigned I = 0, E = MD->getNumOperands(); I != E; ++I) {

SPIRVFunctionParameter *BA = BF->getArgument(I);

Func(MD->getOperand(I), BA);

}

}

static SPIRVMemoryModelKind getMemoryModel(Module &M) {

auto *MemoryModelMD = M.getNamedMetadata(kSPIRVMD::MemoryModel);

if (MemoryModelMD && (MemoryModelMD->getNumOperands() > 0)) {

auto *Ref0 = MemoryModelMD->getOperand(0);

if (Ref0 && Ref0->getNumOperands() > 1) {

auto &&ModelOp = Ref0->getOperand(1);

auto *ModelCI = mdconst::dyn_extract<ConstantInt>(ModelOp);

if (ModelCI && (ModelCI->getValue().getActiveBits() <= 64)) {

auto Model = static_cast<SPIRVMemoryModelKind>(ModelCI->getZExtValue());

return Model;

}

}

}

return SPIRVMemoryModelKind::MemoryModelMax;

}

static void translateSEVDecoration(Attribute Sev, SPIRVValue *Val) {

assert(Sev.isStringAttribute() &&

Sev.getKindAsString() == kVCMetadata::VCSingleElementVector);

auto *Ty = Val->getType();

assert((Ty->isTypeBool() || Ty->isTypeFloat() || Ty->isTypeInt() ||

Ty->isTypePointer()) &&

"This decoration is valid only for Scalar or Pointer types");

if (Ty->isTypePointer()) {

SPIRVWord IndirectLevelsOnElement = 0;

Sev.getValueAsString().getAsInteger(0, IndirectLevelsOnElement);

Val->addDecorate(DecorationSingleElementVectorINTEL,

IndirectLevelsOnElement);

} else

Val->addDecorate(DecorationSingleElementVectorINTEL);

}

LLVMToSPIRVBase::LLVMToSPIRVBase(SPIRVModule *SMod)

: BuiltinCallHelper(ManglingRules::None), M(nullptr), Ctx(nullptr),

BM(SMod), SrcLang(0), SrcLangVer(0) {

DbgTran = std::make_unique<LLVMToSPIRVDbgTran>(nullptr, SMod, this);

}

LLVMToSPIRVBase::~LLVMToSPIRVBase() {

for (auto *I : UnboundInst)

I->deleteValue();

}

bool LLVMToSPIRVBase::runLLVMToSPIRV(Module &Mod) {

M = &Mod;

initialize(Mod);

CG = std::make_unique<CallGraph>(Mod);

Ctx = &M->getContext();

DbgTran->setModule(M);

assert(BM && "SPIR-V module not initialized");

translate();

return true;

}

SPIRVValue *LLVMToSPIRVBase::getTranslatedValue(const Value *V) const {

auto Loc = ValueMap.find(V);

if (Loc != ValueMap.end())

return Loc->second;

return nullptr;

}

bool LLVMToSPIRVBase::isKernel(Function *F) {

if (F->getCallingConv() == CallingConv::SPIR_KERNEL)

return true;

return false;

}

bool LLVMToSPIRVBase::isBuiltinTransToInst(Function *F) {

StringRef DemangledName;

if (!oclIsBuiltin(F->getName(), DemangledName) &&

!isDecoratedSPIRVFunc(F, DemangledName))

return false;

SPIRVDBG(spvdbgs() << "CallInst: demangled name: " << DemangledName.str()

<< '\n');

return getSPIRVFuncOC(DemangledName) != OpNop;

}

bool LLVMToSPIRVBase::isBuiltinTransToExtInst(

Function *F, SPIRVExtInstSetKind *ExtSet, SPIRVWord *ExtOp,

SmallVectorImpl<std::string> *Dec) {

StringRef DemangledName;

if (!oclIsBuiltin(F->getName(), DemangledName))

return false;

LLVM_DEBUG(dbgs() << "[oclIsBuiltinTransToExtInst] CallInst: demangled name: "

<< DemangledName << '\n');

StringRef S = DemangledName;

if (!S.startswith(kSPIRVName::Prefix))

return false;

S = S.drop_front(strlen(kSPIRVName::Prefix));

auto Loc = S.find(kSPIRVPostfix::Divider);

auto ExtSetName = S.substr(0, Loc);

SPIRVExtInstSetKind Set = SPIRVEIS_Count;

if (!SPIRVExtSetShortNameMap::rfind(ExtSetName.str(), &Set))

return false;

assert((Set == SPIRVEIS_OpenCL || Set == BM->getDebugInfoEIS()) &&

"Unsupported extended instruction set");

auto ExtOpName = S.substr(Loc + 1);

auto Splited = ExtOpName.split(kSPIRVPostfix::ExtDivider);

OCLExtOpKind EOC;

if (!OCLExtOpMap::rfind(Splited.first.str(), &EOC))

return false;

if (ExtSet)

*ExtSet = Set;

if (ExtOp)

*ExtOp = EOC;

if (Dec) {

SmallVector<StringRef, 2> P;

Splited.second.split(P, kSPIRVPostfix::Divider);

for (auto &I : P)

Dec->push_back(I.str());

}

return true;

}

bool isUniformGroupOperation(Function *F) {

auto Name = F->getName();

if (Name.contains("GroupIMulKHR") || Name.contains("GroupFMulKHR") ||

Name.contains("GroupBitwiseAndKHR") ||

Name.contains("GroupBitwiseOrKHR") ||

Name.contains("GroupBitwiseXorKHR") ||

Name.contains("GroupLogicalAndKHR") ||

Name.contains("GroupLogicalOrKHR") || Name.contains("GroupLogicalXorKHR"))

return true;

return false;

}

static bool recursiveType(const StructType *ST, const Type *Ty) {

SmallPtrSet<const StructType *, 4> Seen;

std::function<bool(const Type *Ty)> Run = [&](const Type *Ty) {

if (auto *StructTy = dyn_cast<StructType>(Ty)) {

if (StructTy == ST)

return true;

if (Seen.count(StructTy))

return false;

Seen.insert(StructTy);

return find_if(StructTy->element_begin(), StructTy->element_end(), Run) !=

StructTy->element_end();

}

if (auto *ArrayTy = dyn_cast<ArrayType>(Ty))

return Run(ArrayTy->getArrayElementType());

return false;

};

return Run(Ty);

}

SPIRVType *LLVMToSPIRVBase::transType(Type *T) {

LLVMToSPIRVTypeMap::iterator Loc = TypeMap.find(T);

if (Loc != TypeMap.end())

return Loc->second;

SPIRVDBG(dbgs() << "[transType] " << *T << '\n');

if (T->isVoidTy())

return mapType(T, BM->addVoidType());

if (T->isIntegerTy(1))

return mapType(T, BM->addBoolType());

if (T->isIntegerTy()) {

unsigned BitWidth = T->getIntegerBitWidth();

// SPIR-V 2.16.1. Universal Validation Rules: Scalar integer types can be

// parameterized only as 32 bit, plus any additional sizes enabled by

// capabilities.

if (BM->isAllowedToUseExtension(

ExtensionID::SPV_INTEL_arbitrary_precision_integers) ||

BM->getErrorLog().checkError(

BitWidth == 8 || BitWidth == 16 || BitWidth == 32 || BitWidth == 64,

SPIRVEC_InvalidBitWidth, std::to_string(BitWidth))) {

return mapType(T, BM->addIntegerType(T->getIntegerBitWidth()));

}

}

if (T->isFloatingPointTy())

return mapType(T, BM->addFloatType(T->getPrimitiveSizeInBits()));

if (T->isTokenTy()) {

BM->getErrorLog().checkError(

BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_token_type),

SPIRVEC_RequiresExtension,

"SPV_INTEL_token_type\n"

"NOTE: LLVM module contains token type, which doesn't have analogs in "

"SPIR-V without extensions");

return mapType(T, BM->addTokenTypeINTEL());

}

// A pointer to image or pipe type in LLVM is translated to a SPIRV

// (non-pointer) image or pipe type.

if (T->isPointerTy()) {

auto *ET = Type::getInt8Ty(T->getContext());

auto AddrSpc = T->getPointerAddressSpace();

return transPointerType(ET, AddrSpc);

}

if (auto *TPT = dyn_cast<TypedPointerType>(T)) {

return transPointerType(TPT->getElementType(), TPT->getAddressSpace());

}

if (auto *VecTy = dyn_cast<FixedVectorType>(T)) {

if (VecTy->getElementType()->isPointerTy() ||

isa<TypedPointerType>(VecTy->getElementType())) {

// SPV_INTEL_masked_gather_scatter extension changes 2.16.1. Universal

// Validation Rules:

// Vector types must be parameterized only with numerical types,

// [Physical Pointer Type] types or the [OpTypeBool] type.

// Without it vector of pointers is not allowed in SPIR-V.

if (!BM->isAllowedToUseExtension(

ExtensionID::SPV_INTEL_masked_gather_scatter)) {

BM->getErrorLog().checkError(

false, SPIRVEC_RequiresExtension,

"SPV_INTEL_masked_gather_scatter\n"

"NOTE: LLVM module contains vector of pointers, translation "

"of which requires this extension");

return nullptr;

}

BM->addExtension(ExtensionID::SPV_INTEL_masked_gather_scatter);

BM->addCapability(internal::CapabilityMaskedGatherScatterINTEL);

}

return mapType(T, BM->addVectorType(transType(VecTy->getElementType()),

VecTy->getNumElements()));

}

if (T->isArrayTy()) {

// SPIR-V 1.3 s3.32.6: Length is the number of elements in the array.

// It must be at least 1.

if (T->getArrayNumElements() < 1) {

std::string Str;

llvm::raw_string_ostream OS(Str);

OS << *T;

SPIRVCK(T->getArrayNumElements() >= 1, InvalidArraySize, OS.str());

}

Type *ElTy = T->getArrayElementType();

SPIRVType *TransType = BM->addArrayType(

transType(ElTy),

static_cast<SPIRVConstant *>(transValue(

ConstantInt::get(getSizetType(), T->getArrayNumElements(), false),

nullptr)));

mapType(T, TransType);

if (ElTy->isPointerTy()) {

mapType(

ArrayType::get(TypedPointerType::get(Type::getInt8Ty(*Ctx),

ElTy->getPointerAddressSpace()),

T->getArrayNumElements()),

TransType);

}

return TransType;

}

if (T->isStructTy() && !T->isSized()) {

auto *ST = dyn_cast<StructType>(T);

(void)ST; // Silence warning

assert(!ST->getName().startswith(kSPR2TypeName::PipeRO));

assert(!ST->getName().startswith(kSPR2TypeName::PipeWO));

assert(!ST->getName().startswith(kSPR2TypeName::ImagePrefix));

return mapType(T, BM->addOpaqueType(T->getStructName().str()));

}

if (auto *ST = dyn_cast<StructType>(T)) {

assert(ST->isSized());

StringRef Name;

if (ST->hasName())

Name = ST->getName();

if (Name == getSPIRVTypeName(kSPIRVTypeName::ConstantSampler))

return transSPIRVOpaqueType("spirv.Sampler", SPIRAS_Constant);

if (Name == getSPIRVTypeName(kSPIRVTypeName::ConstantPipeStorage))

return transSPIRVOpaqueType("spirv.PipeStorage", SPIRAS_Constant);

constexpr size_t MaxNumElements = MaxWordCount - SPIRVTypeStruct::FixedWC;

const size_t NumElements = ST->getNumElements();

size_t SPIRVStructNumElements = NumElements;

// In case number of elements is greater than maximum WordCount and

// SPV_INTEL_long_constant_composite is not enabled, the error will be

// emitted by validate functionality of SPIRVTypeStruct class.

if (NumElements > MaxNumElements &&

BM->isAllowedToUseExtension(

ExtensionID::SPV_INTEL_long_constant_composite)) {

SPIRVStructNumElements = MaxNumElements;

}

auto *Struct = BM->openStructType(SPIRVStructNumElements, Name.str());

mapType(T, Struct);

if (NumElements > MaxNumElements &&

BM->isAllowedToUseExtension(

ExtensionID::SPV_INTEL_long_constant_composite)) {

uint64_t NumOfContinuedInstructions = NumElements / MaxNumElements - 1;

for (uint64_t J = 0; J < NumOfContinuedInstructions; J++) {

auto *Continued = BM->addTypeStructContinuedINTEL(MaxNumElements);

Struct->addContinuedInstruction(

static_cast<SPIRVTypeStruct::ContinuedInstType>(Continued));

}

uint64_t Remains = NumElements % MaxNumElements;

if (Remains) {

auto *Continued = BM->addTypeStructContinuedINTEL(Remains);

Struct->addContinuedInstruction(

static_cast<SPIRVTypeStruct::ContinuedInstType>(Continued));

}

}

SmallVector<unsigned, 4> ForwardRefs;

for (unsigned I = 0, E = T->getStructNumElements(); I != E; ++I) {

auto *ElemTy = ST->getElementType(I);

if ((isa<StructType>(ElemTy) || isa<ArrayType>(ElemTy) ||

isa<VectorType>(ElemTy) || isa<PointerType>(ElemTy)) &&

recursiveType(ST, ElemTy))

ForwardRefs.push_back(I);

else

Struct->setMemberType(I, transType(ST->getElementType(I)));

}

BM->closeStructType(Struct, ST->isPacked());

for (auto I : ForwardRefs)

Struct->setMemberType(I, transType(ST->getElementType(I)));

return Struct;

}

if (FunctionType *FT = dyn_cast<FunctionType>(T)) {

SPIRVType *RT = transType(FT->getReturnType());

std::vector<SPIRVType *> PT;

for (FunctionType::param_iterator I = FT->param_begin(),

E = FT->param_end();

I != E; ++I)

PT.push_back(transType(*I));

return mapType(T, getSPIRVFunctionType(RT, PT));

}

if (auto *TargetTy = dyn_cast<TargetExtType>(T)) {

StringRef Name = TargetTy->getName();

if (Name.consume_front(kSPIRVTypeName::PrefixAndDelim)) {

auto Opcode = SPIRVOpaqueTypeOpCodeMap::map(Name.str());

auto CastAccess = [](unsigned Val) {

return static_cast<SPIRVAccessQualifierKind>(Val);

};

switch (static_cast<size_t>(Opcode)) {

case OpTypePipe: {

auto *PipeT = BM->addPipeType();

PipeT->setPipeAcessQualifier(CastAccess(TargetTy->getIntParameter(0)));

return mapType(T, PipeT);

}

case OpTypeImage: {

auto *SampledTy = transType(TargetTy->getTypeParameter(0));

ArrayRef<unsigned> Ops = TargetTy->int_params();

SPIRVTypeImageDescriptor Desc(static_cast<SPIRVImageDimKind>(Ops[0]),

Ops[1], Ops[2], Ops[3], Ops[4], Ops[5]);

return mapType(T,

BM->addImageType(SampledTy, Desc, CastAccess(Ops[6])));

}

case OpTypeSampledImage: {

auto *ImageTy = static_cast<SPIRVTypeImage *>(transType(adjustImageType(

T, kSPIRVTypeName::SampledImg, kSPIRVTypeName::Image)));

return mapType(T, BM->addSampledImageType(ImageTy));

}

case OpTypeVmeImageINTEL: {

auto *ImageTy = static_cast<SPIRVTypeImage *>(transType(adjustImageType(

T, kSPIRVTypeName::VmeImageINTEL, kSPIRVTypeName::Image)));

return mapType(T, BM->addVmeImageINTELType(ImageTy));

}

case OpTypeQueue:

return mapType(T, BM->addQueueType());

case OpTypeDeviceEvent:

return mapType(T, BM->addDeviceEventType());

case OpTypeBufferSurfaceINTEL: {

ArrayRef<unsigned> Ops = TargetTy->int_params();

return mapType(T, BM->addBufferSurfaceINTELType(CastAccess(Ops[0])));

}

case internal::OpTypeJointMatrixINTEL: {

// The expected representation is:

// target("spirv.JointMatrixINTEL", %element_type, %rows%, %cols%,

// %layout%, %scope%, %use%,

// (optional) %element_type_interpretation%)

auto *ElemTy = transType(TargetTy->getTypeParameter(0));

ArrayRef<unsigned> Ops = TargetTy->int_params();

std::vector<SPIRVValue *> Args;

for (const auto &Op : Ops)

Args.emplace_back(transConstant(getUInt32(M, Op)));

return mapType(T, BM->addJointMatrixINTELType(ElemTy, Args));

}

case OpTypeCooperativeMatrixKHR: {

// The expected representation is:

// target("spirv.CooperativeMatrixKHR", %element_type, %scope%, %rows%,

// %cols%, %use%)

auto *ElemTy = transType(TargetTy->getTypeParameter(0));

ArrayRef<unsigned> Ops = TargetTy->int_params();

std::vector<SPIRVValue *> Args;

for (const auto &Op : Ops)

Args.emplace_back(transConstant(getUInt32(M, Op)));

return mapType(T, BM->addCooperativeMatrixKHRType(ElemTy, Args));

}

default:

if (isSubgroupAvcINTELTypeOpCode(Opcode))

return mapType(T, BM->addSubgroupAvcINTELType(Opcode));

return mapType(T, BM->addOpaqueGenericType(Opcode));

}

}

}

llvm_unreachable("Not implemented!");

return 0;

}

SPIRVType *LLVMToSPIRVBase::transPointerType(Type *ET, unsigned AddrSpc) {

Type *T = PointerType::get(ET, AddrSpc);

if (ET->isFunctionTy() &&

!BM->checkExtension(ExtensionID::SPV_INTEL_function_pointers,

SPIRVEC_FunctionPointers, toString(T)))

return nullptr;

std::string TypeKey = (Twine((uintptr_t)ET) + Twine(AddrSpc)).str();

auto Loc = PointeeTypeMap.find(TypeKey);

if (Loc != PointeeTypeMap.end())

return Loc->second;

// A pointer to image or pipe type in LLVM is translated to a SPIRV

// (non-pointer) image or pipe type.

auto *ST = dyn_cast<StructType>(ET);

// Lower global_device and global_host address spaces that were added in

// SYCL as part of SYCL_INTEL_usm_address_spaces extension to just global

// address space if device doesn't support SPV_INTEL_usm_storage_classes

// extension

if (!BM->isAllowedToUseExtension(

ExtensionID::SPV_INTEL_usm_storage_classes) &&

((AddrSpc == SPIRAS_GlobalDevice) || (AddrSpc == SPIRAS_GlobalHost))) {

return transPointerType(ET, SPIRAS_Global);

}

if (ST && !ST->isSized()) {

Op OpCode;

StringRef STName = ST->getName();

// Workaround for non-conformant SPIR binary

if (STName == "struct._event_t") {

STName = kSPR2TypeName::Event;

ST->setName(STName);

}

std::pair<StringRef, unsigned> Key = {STName, AddrSpc};

if (auto *MappedTy = OpaqueStructMap.lookup(Key))

return MappedTy;

auto SaveType = [&](SPIRVType *MappedTy) {

OpaqueStructMap[Key] = MappedTy;

PointeeTypeMap[TypeKey] = MappedTy;

return MappedTy;

};

if (STName.startswith(kSPR2TypeName::PipeRO) ||

STName.startswith(kSPR2TypeName::PipeWO)) {

auto *PipeT = BM->addPipeType();

PipeT->setPipeAcessQualifier(STName.startswith(kSPR2TypeName::PipeRO)

? AccessQualifierReadOnly

: AccessQualifierWriteOnly);

return SaveType(PipeT);

}

if (STName.startswith(kSPR2TypeName::ImagePrefix)) {

assert(AddrSpc == SPIRAS_Global);

Type *ImageTy =

adjustImageType(TypedPointerType::get(ST, AddrSpc),

kSPIRVTypeName::Image, kSPIRVTypeName::Image);

return SaveType(transType(ImageTy));

}

if (STName == kSPR2TypeName::Sampler)

return SaveType(transType(getSPIRVType(OpTypeSampler)));

if (STName.startswith(kSPIRVTypeName::PrefixAndDelim))

return transSPIRVOpaqueType(STName, AddrSpc);

if (STName.startswith(kOCLSubgroupsAVCIntel::TypePrefix))

return SaveType(BM->addSubgroupAvcINTELType(

OCLSubgroupINTELTypeOpCodeMap::map(ST->getName().str())));

if (OCLOpaqueTypeOpCodeMap::find(STName.str(), &OpCode)) {

Type *RealType = getSPIRVType(OpCode);

return SaveType(transType(RealType));

}

if (BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_vector_compute)) {

if (STName.startswith(kVCType::VCBufferSurface)) {

// VCBufferSurface always have Access Qualifier

auto Access = getAccessQualifier(STName);

return SaveType(BM->addBufferSurfaceINTELType(Access));

}

}

if (ST->isOpaque()) {

return SaveType(BM->addPointerType(

SPIRSPIRVAddrSpaceMap::map(static_cast<SPIRAddressSpace>(AddrSpc)),

transType(ET)));

}

} else {

SPIRVType *ElementType = transType(ET);

// ET, as a recursive type, may contain exactly the same pointer T, so it

// may happen that after translation of ET we already have translated T,

// added the translated pointer to the SPIR-V module and mapped T to this

// pointer. Now we have to check PointeeTypeMap again.

auto Loc = PointeeTypeMap.find(TypeKey);

if (Loc != PointeeTypeMap.end()) {

return Loc->second;

}

SPIRVType *TranslatedTy = transPointerType(ElementType, AddrSpc);

PointeeTypeMap[TypeKey] = TranslatedTy;

return TranslatedTy;

}

llvm_unreachable("Not implemented!");

return nullptr;

}

SPIRVType *LLVMToSPIRVBase::transPointerType(SPIRVType *ET, unsigned AddrSpc) {

std::string TypeKey = (Twine((uintptr_t)ET) + Twine(AddrSpc)).str();

auto Loc = PointeeTypeMap.find(TypeKey);

if (Loc != PointeeTypeMap.end())

return Loc->second;

SPIRVType *TranslatedTy = BM->addPointerType(

SPIRSPIRVAddrSpaceMap::map(static_cast<SPIRAddressSpace>(AddrSpc)), ET);

PointeeTypeMap[TypeKey] = TranslatedTy;

return TranslatedTy;

}

SPIRVType *LLVMToSPIRVBase::transSPIRVOpaqueType(StringRef STName,

unsigned AddrSpace) {

std::pair<StringRef, unsigned> Key = {STName, AddrSpace};

if (auto *MappedTy = OpaqueStructMap.lookup(Key))

return MappedTy;

auto SaveType = [&](SPIRVType *MappedTy) {

OpaqueStructMap[Key] = MappedTy;

return MappedTy;

};

StructType *ST = StructType::getTypeByName(M->getContext(), STName);

assert(STName.startswith(kSPIRVTypeName::PrefixAndDelim) &&

"Invalid SPIR-V opaque type name");

SmallVector<std::string, 8> Postfixes;

auto TN = decodeSPIRVTypeName(STName, Postfixes);

if (TN == kSPIRVTypeName::Pipe) {

assert(AddrSpace == SPIRAS_Global);

assert(Postfixes.size() == 1 && "Invalid pipe type ops");

auto *PipeT = BM->addPipeType();

PipeT->setPipeAcessQualifier(

static_cast<spv::AccessQualifier>(atoi(Postfixes[0].c_str())));

return SaveType(PipeT);

} else if (TN == kSPIRVTypeName::Image) {

assert(AddrSpace == SPIRAS_Global);

// The sampled type needs to be translated through LLVM type to guarantee

// uniqueness.

auto *SampledT = transType(

getLLVMTypeForSPIRVImageSampledTypePostfix(Postfixes[0], *Ctx));

SmallVector<int, 7> Ops;

for (unsigned I = 1; I < 8; ++I)

Ops.push_back(atoi(Postfixes[I].c_str()));

SPIRVTypeImageDescriptor Desc(static_cast<SPIRVImageDimKind>(Ops[0]),

Ops[1], Ops[2], Ops[3], Ops[4], Ops[5]);

return SaveType(BM->addImageType(

SampledT, Desc, static_cast<spv::AccessQualifier>(Ops[6])));

} else if (TN == kSPIRVTypeName::SampledImg) {

return SaveType(BM->addSampledImageType(static_cast<SPIRVTypeImage *>(

transType(adjustImageType(TypedPointerType::get(ST, SPIRAS_Global),

kSPIRVTypeName::SampledImg,

kSPIRVTypeName::Image)))));

} else if (TN == kSPIRVTypeName::VmeImageINTEL) {

// This type is the same as SampledImageType, but consumed by Subgroup AVC

// Intel extension instructions.

return SaveType(BM->addVmeImageINTELType(static_cast<SPIRVTypeImage *>(

transType(adjustImageType(TypedPointerType::get(ST, SPIRAS_Global),

kSPIRVTypeName::VmeImageINTEL,

kSPIRVTypeName::Image)))));

} else if (TN == kSPIRVTypeName::Sampler)

return SaveType(BM->addSamplerType());

else if (TN == kSPIRVTypeName::DeviceEvent)

return SaveType(BM->addDeviceEventType());

else if (TN == kSPIRVTypeName::Queue)

return SaveType(BM->addQueueType());

else if (TN == kSPIRVTypeName::PipeStorage)

return SaveType(BM->addPipeStorageType());

else if (BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_vector_compute) &&

TN == kSPIRVTypeName::BufferSurfaceINTEL) {

auto Access = getAccessQualifier(STName);

return SaveType(BM->addBufferSurfaceINTELType(Access));

} else

return SaveType(

BM->addOpaqueGenericType(SPIRVOpaqueTypeOpCodeMap::map(TN)));

}

SPIRVType *LLVMToSPIRVBase::transScavengedType(Value *V) {

if (auto *F = dyn_cast<Function>(V)) {

FunctionType *FnTy = Scavenger->getFunctionType(F);

SPIRVType *RT = transType(FnTy->getReturnType());

std::vector<SPIRVType *> PT;

for (Argument &Arg : F->args()) {

assert(OCLTypeToSPIRVPtr);

Type *Ty = OCLTypeToSPIRVPtr->getAdaptedArgumentType(F, Arg.getArgNo());

if (!Ty) {

Ty = FnTy->getParamType(Arg.getArgNo());

}

PT.push_back(transType(Ty));

}

return getSPIRVFunctionType(RT, PT);

}

return transType(Scavenger->getScavengedType(V));

}

SPIRVType *

LLVMToSPIRVBase::getSPIRVFunctionType(SPIRVType *RT,

const std::vector<SPIRVType *> &Args) {

// Come up with a unique string identifier for the arguments. This is a hacky

// way of doing so, but it works.

std::string TypeKey;

llvm::raw_string_ostream TKS(TypeKey);

TKS << (uintptr_t)RT << ",";

for (SPIRVType *ArgTy : Args) {

TKS << (uintptr_t)ArgTy << ",";

}

// Create a SPIRVType for the function type. Since SPIRVModule doesn't do

// any type uniquing for SPIRVType, we have to do it ourself.

TKS.flush();

auto It = PointeeTypeMap.find(TypeKey);

if (It == PointeeTypeMap.end())

It = PointeeTypeMap.insert({TypeKey, BM->addFunctionType(RT, Args)}).first;

return It->second;

}

SPIRVFunction *LLVMToSPIRVBase::transFunctionDecl(Function *F) {

if (auto *BF = getTranslatedValue(F))

return static_cast<SPIRVFunction *>(BF);

if (F->isIntrinsic() && (!BM->isSPIRVAllowUnknownIntrinsicsEnabled() ||

isKnownIntrinsic(F->getIntrinsicID()))) {

// We should not translate LLVM intrinsics as a function

assert(none_of(F->users(),

[this](User *U) { return getTranslatedValue(U); }) &&

"LLVM intrinsics shouldn't be called in SPIRV");

return nullptr;

}

SPIRVTypeFunction *BFT =

static_cast<SPIRVTypeFunction *>(transScavengedType(F));

SPIRVFunction *BF =

static_cast<SPIRVFunction *>(mapValue(F, BM->addFunction(BFT)));

BF->setFunctionControlMask(transFunctionControlMask(F));

if (F->hasName()) {

if (isKernel(F)) {

// If found, strip the entry point wrapper prefix as the runtime will

// be looking for this name

StringRef Name = F->getName();

(void) Name.consume_front(kSPIRVName::EntrypointPrefix);

BM->setName(BF, Name.str());

} else {

if (isUniformGroupOperation(F))

BM->getErrorLog().checkError(

BM->isAllowedToUseExtension(

ExtensionID::SPV_KHR_uniform_group_instructions),

SPIRVEC_RequiresExtension, "SPV_KHR_uniform_group_instructions\n");

BM->setName(BF, F->getName().str());

}

}

if (!isKernel(F) && F->getLinkage() != GlobalValue::InternalLinkage)

BF->setLinkageType(transLinkageType(F));

// Translate OpenCL/SYCL buffer_location metadata if it's attached to the

// translated function declaration

MDNode *BufferLocation = nullptr;

if (BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_fpga_buffer_location))

BufferLocation = F->getMetadata("kernel_arg_buffer_location");

// Translate runtime_aligned metadata if it's attached to the translated

// function declaration

MDNode *RuntimeAligned = nullptr;

if (BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_runtime_aligned))

RuntimeAligned = F->getMetadata("kernel_arg_runtime_aligned");

auto Attrs = F->getAttributes();

for (Function::arg_iterator I = F->arg_begin(), E = F->arg_end(); I != E;

++I) {

auto ArgNo = I->getArgNo();

SPIRVFunctionParameter *BA = BF->getArgument(ArgNo);

if (I->hasName())

BM->setName(BA, I->getName().str());

if (I->hasByValAttr())

BA->addAttr(FunctionParameterAttributeByVal);

if (I->hasNoAliasAttr())

BA->addAttr(FunctionParameterAttributeNoAlias);

if (I->hasNoCaptureAttr())

BA->addAttr(FunctionParameterAttributeNoCapture);

if (I->hasStructRetAttr())

BA->addAttr(FunctionParameterAttributeSret);

if (Attrs.hasParamAttr(ArgNo, Attribute::ReadOnly))

BA->addAttr(FunctionParameterAttributeNoWrite);

if (Attrs.hasParamAttr(ArgNo, Attribute::ReadNone))

BA->addAttr(FunctionParameterAttributeNoReadWrite);

if (Attrs.hasParamAttr(ArgNo, Attribute::ZExt))

BA->addAttr(FunctionParameterAttributeZext);

if (Attrs.hasParamAttr(ArgNo, Attribute::SExt))

BA->addAttr(FunctionParameterAttributeSext);

if (Attrs.hasParamAttr(ArgNo, Attribute::Alignment)) {

SPIRVWord AlignmentBytes = Attrs.getParamAttr(ArgNo, Attribute::Alignment)

.getAlignment()

.valueOrOne()

.value();

BA->setAlignment(AlignmentBytes);

}

if (BM->isAllowedToUseVersion(VersionNumber::SPIRV_1_1) &&

Attrs.hasParamAttr(ArgNo, Attribute::Dereferenceable))

BA->addDecorate(DecorationMaxByteOffset,

Attrs.getParamAttr(ArgNo, Attribute::Dereferenceable)

.getDereferenceableBytes());

if (BufferLocation && I->getType()->isPointerTy()) {

// Order of integer numbers in MD node follows the order of function

// parameters on which we shall attach the appropriate decoration. Add

// decoration only if MD value is not negative.

int LocID = -1;

if (!isa<MDString>(BufferLocation->getOperand(ArgNo)) &&

!isa<MDNode>(BufferLocation->getOperand(ArgNo)))

LocID = getMDOperandAsInt(BufferLocation, ArgNo);

if (LocID >= 0)

BA->addDecorate(DecorationBufferLocationINTEL, LocID);

}

if (RuntimeAligned && I->getType()->isPointerTy()) {

// Order of integer numbers in MD node follows the order of function

// parameters on which we shall attach the appropriate decoration. Add

// decoration only if MD value is 1.

int LocID = 0;

if (!isa<MDString>(RuntimeAligned->getOperand(ArgNo)) &&

!isa<MDNode>(RuntimeAligned->getOperand(ArgNo)))

LocID = getMDOperandAsInt(RuntimeAligned, ArgNo);

if (LocID == 1)

BA->addDecorate(internal::DecorationRuntimeAlignedINTEL, LocID);

}

}

if (Attrs.hasRetAttr(Attribute::ZExt))

BF->addDecorate(DecorationFuncParamAttr, FunctionParameterAttributeZext);

if (Attrs.hasRetAttr(Attribute::SExt))

BF->addDecorate(DecorationFuncParamAttr, FunctionParameterAttributeSext);

if (Attrs.hasFnAttr("referenced-indirectly")) {

assert(!isKernel(F) &&

"kernel function was marked as referenced-indirectly");

BF->addDecorate(DecorationReferencedIndirectlyINTEL);

}

if (Attrs.hasFnAttr(kVCMetadata::VCCallable) &&

BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_fast_composite)) {

BF->addDecorate(internal::DecorationCallableFunctionINTEL);

}

if (BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_vector_compute))

transVectorComputeMetadata(F);

transFPGAFunctionMetadata(BF, F);

transFunctionMetadataAsUserSemanticDecoration(BF, F);

transAuxDataInst(BF, F);

SPIRVDBG(dbgs() << "[transFunction] " << *F << " => ";

spvdbgs() << *BF << '\n';)

return BF;

}

void LLVMToSPIRVBase::transVectorComputeMetadata(Function *F) {

using namespace VectorComputeUtil;

if (!BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_vector_compute))

return;

auto *BF = static_cast<SPIRVFunction *>(getTranslatedValue(F));

assert(BF && "The SPIRVFunction pointer shouldn't be nullptr");

auto Attrs = F->getAttributes();

if (Attrs.hasFnAttr(kVCMetadata::VCStackCall))

BF->addDecorate(DecorationStackCallINTEL);

if (Attrs.hasFnAttr(kVCMetadata::VCFunction))

BF->addDecorate(DecorationVectorComputeFunctionINTEL);

if (Attrs.hasFnAttr(kVCMetadata::VCSIMTCall)) {

SPIRVWord SIMTMode = 0;

Attrs.getFnAttr(kVCMetadata::VCSIMTCall)

.getValueAsString()

.getAsInteger(0, SIMTMode);

BF->addDecorate(DecorationSIMTCallINTEL, SIMTMode);

}

if (Attrs.hasRetAttr(kVCMetadata::VCSingleElementVector))

translateSEVDecoration(

Attrs.getAttributeAtIndex(AttributeList::ReturnIndex,

kVCMetadata::VCSingleElementVector),

BF);

for (Function::arg_iterator I = F->arg_begin(), E = F->arg_end(); I != E;

++I) {

auto ArgNo = I->getArgNo();

SPIRVFunctionParameter *BA = BF->getArgument(ArgNo);

if (Attrs.hasParamAttr(ArgNo, kVCMetadata::VCArgumentIOKind)) {

SPIRVWord Kind = {};

Attrs.getParamAttr(ArgNo, kVCMetadata::VCArgumentIOKind)

.getValueAsString()

.getAsInteger(0, Kind);

BA->addDecorate(DecorationFuncParamIOKindINTEL, Kind);

}

if (Attrs.hasParamAttr(ArgNo, kVCMetadata::VCSingleElementVector))

translateSEVDecoration(

Attrs.getParamAttr(ArgNo, kVCMetadata::VCSingleElementVector), BA);

if (Attrs.hasParamAttr(ArgNo, kVCMetadata::VCMediaBlockIO)) {

assert(BA->getType()->isTypeImage() &&

"VCMediaBlockIO attribute valid only on image parameters");

BA->addDecorate(DecorationMediaBlockIOINTEL);

}

}

if (!isKernel(F) &&

BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_float_controls2) &&

Attrs.hasFnAttr(kVCMetadata::VCFloatControl)) {

SPIRVWord Mode = 0;

Attrs.getFnAttr(kVCMetadata::VCFloatControl)

.getValueAsString()

.getAsInteger(0, Mode);

VCFloatTypeSizeMap::foreach (

[&](VCFloatType FloatType, unsigned TargetWidth) {

BF->addDecorate(new SPIRVDecorateFunctionDenormModeINTEL(

BF, TargetWidth, getFPDenormMode(Mode, FloatType)));

BF->addDecorate(new SPIRVDecorateFunctionRoundingModeINTEL(

BF, TargetWidth, getFPRoundingMode(Mode)));

BF->addDecorate(new SPIRVDecorateFunctionFloatingPointModeINTEL(

BF, TargetWidth, getFPOperationMode(Mode)));

});

}

}

static void transMetadataDecorations(Metadata *MD, SPIRVEntry *Target);

void LLVMToSPIRVBase::transFPGAFunctionMetadata(SPIRVFunction *BF,

Function *F) {

if (MDNode *StallEnable = F->getMetadata(kSPIR2MD::StallEnable)) {

if (BM->isAllowedToUseExtension(

ExtensionID::SPV_INTEL_fpga_cluster_attributes)) {

if (getMDOperandAsInt(StallEnable, 0))

BF->addDecorate(new SPIRVDecorateStallEnableINTEL(BF));

}

}