| //===- OpenMPIRBuilder.cpp - Builder for LLVM-IR for OpenMP directives ----===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| /// \file |
| /// |
| /// This file implements the OpenMPIRBuilder class, which is used as a |
| /// convenient way to create LLVM instructions for OpenMP directives. |
| /// |
| //===----------------------------------------------------------------------===// |
| |
| #include "llvm/Frontend/OpenMP/OMPIRBuilder.h" |
| |
| #include "llvm/ADT/StringRef.h" |
| #include "llvm/ADT/StringSwitch.h" |
| #include "llvm/IR/CFG.h" |
| #include "llvm/IR/DebugInfo.h" |
| #include "llvm/IR/IRBuilder.h" |
| #include "llvm/IR/MDBuilder.h" |
| #include "llvm/Support/CommandLine.h" |
| #include "llvm/Support/Error.h" |
| #include "llvm/Transforms/Utils/BasicBlockUtils.h" |
| #include "llvm/Transforms/Utils/CodeExtractor.h" |
| |
| #include <sstream> |
| |
| #define DEBUG_TYPE "openmp-ir-builder" |
| |
| using namespace llvm; |
| using namespace omp; |
| |
| static cl::opt<bool> |
| OptimisticAttributes("openmp-ir-builder-optimistic-attributes", cl::Hidden, |
| cl::desc("Use optimistic attributes describing " |
| "'as-if' properties of runtime calls."), |
| cl::init(false)); |
| |
| void OpenMPIRBuilder::addAttributes(omp::RuntimeFunction FnID, Function &Fn) { |
| LLVMContext &Ctx = Fn.getContext(); |
| |
| #define OMP_ATTRS_SET(VarName, AttrSet) AttributeSet VarName = AttrSet; |
| #include "llvm/Frontend/OpenMP/OMPKinds.def" |
| |
| // Add attributes to the new declaration. |
| switch (FnID) { |
| #define OMP_RTL_ATTRS(Enum, FnAttrSet, RetAttrSet, ArgAttrSets) \ |
| case Enum: \ |
| Fn.setAttributes( \ |
| AttributeList::get(Ctx, FnAttrSet, RetAttrSet, ArgAttrSets)); \ |
| break; |
| #include "llvm/Frontend/OpenMP/OMPKinds.def" |
| default: |
| // Attributes are optional. |
| break; |
| } |
| } |
| |
| FunctionCallee |
| OpenMPIRBuilder::getOrCreateRuntimeFunction(Module &M, RuntimeFunction FnID) { |
| FunctionType *FnTy = nullptr; |
| Function *Fn = nullptr; |
| |
| // Try to find the declation in the module first. |
| switch (FnID) { |
| #define OMP_RTL(Enum, Str, IsVarArg, ReturnType, ...) \ |
| case Enum: \ |
| FnTy = FunctionType::get(ReturnType, ArrayRef<Type *>{__VA_ARGS__}, \ |
| IsVarArg); \ |
| Fn = M.getFunction(Str); \ |
| break; |
| #include "llvm/Frontend/OpenMP/OMPKinds.def" |
| } |
| |
| if (!Fn) { |
| // Create a new declaration if we need one. |
| switch (FnID) { |
| #define OMP_RTL(Enum, Str, ...) \ |
| case Enum: \ |
| Fn = Function::Create(FnTy, GlobalValue::ExternalLinkage, Str, M); \ |
| break; |
| #include "llvm/Frontend/OpenMP/OMPKinds.def" |
| } |
| |
| // Add information if the runtime function takes a callback function |
| if (FnID == OMPRTL___kmpc_fork_call || FnID == OMPRTL___kmpc_fork_teams) { |
| if (!Fn->hasMetadata(LLVMContext::MD_callback)) { |
| LLVMContext &Ctx = Fn->getContext(); |
| MDBuilder MDB(Ctx); |
| // Annotate the callback behavior of the runtime function: |
| // - The callback callee is argument number 2 (microtask). |
| // - The first two arguments of the callback callee are unknown (-1). |
| // - All variadic arguments to the runtime function are passed to the |
| // callback callee. |
| Fn->addMetadata( |
| LLVMContext::MD_callback, |
| *MDNode::get(Ctx, {MDB.createCallbackEncoding( |
| 2, {-1, -1}, /* VarArgsArePassed */ true)})); |
| } |
| } |
| |
| LLVM_DEBUG(dbgs() << "Created OpenMP runtime function " << Fn->getName() |
| << " with type " << *Fn->getFunctionType() << "\n"); |
| addAttributes(FnID, *Fn); |
| |
| } else { |
| LLVM_DEBUG(dbgs() << "Found OpenMP runtime function " << Fn->getName() |
| << " with type " << *Fn->getFunctionType() << "\n"); |
| } |
| |
| assert(Fn && "Failed to create OpenMP runtime function"); |
| |
| // Cast the function to the expected type if necessary |
| Constant *C = ConstantExpr::getBitCast(Fn, FnTy->getPointerTo()); |
| return {FnTy, C}; |
| } |
| |
| Function *OpenMPIRBuilder::getOrCreateRuntimeFunctionPtr(RuntimeFunction FnID) { |
| FunctionCallee RTLFn = getOrCreateRuntimeFunction(M, FnID); |
| auto *Fn = dyn_cast<llvm::Function>(RTLFn.getCallee()); |
| assert(Fn && "Failed to create OpenMP runtime function pointer"); |
| return Fn; |
| } |
| |
| void OpenMPIRBuilder::initialize() { initializeTypes(M); } |
| |
| void OpenMPIRBuilder::finalize() { |
| SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet; |
| SmallVector<BasicBlock *, 32> Blocks; |
| for (OutlineInfo &OI : OutlineInfos) { |
| ParallelRegionBlockSet.clear(); |
| Blocks.clear(); |
| OI.collectBlocks(ParallelRegionBlockSet, Blocks); |
| |
| Function *OuterFn = OI.EntryBB->getParent(); |
| CodeExtractorAnalysisCache CEAC(*OuterFn); |
| CodeExtractor Extractor(Blocks, /* DominatorTree */ nullptr, |
| /* AggregateArgs */ false, |
| /* BlockFrequencyInfo */ nullptr, |
| /* BranchProbabilityInfo */ nullptr, |
| /* AssumptionCache */ nullptr, |
| /* AllowVarArgs */ true, |
| /* AllowAlloca */ true, |
| /* Suffix */ ".omp_par"); |
| |
| LLVM_DEBUG(dbgs() << "Before outlining: " << *OuterFn << "\n"); |
| LLVM_DEBUG(dbgs() << "Entry " << OI.EntryBB->getName() |
| << " Exit: " << OI.ExitBB->getName() << "\n"); |
| assert(Extractor.isEligible() && |
| "Expected OpenMP outlining to be possible!"); |
| |
| Function *OutlinedFn = Extractor.extractCodeRegion(CEAC); |
| |
| LLVM_DEBUG(dbgs() << "After outlining: " << *OuterFn << "\n"); |
| LLVM_DEBUG(dbgs() << " Outlined function: " << *OutlinedFn << "\n"); |
| assert(OutlinedFn->getReturnType()->isVoidTy() && |
| "OpenMP outlined functions should not return a value!"); |
| |
| // For compability with the clang CG we move the outlined function after the |
| // one with the parallel region. |
| OutlinedFn->removeFromParent(); |
| M.getFunctionList().insertAfter(OuterFn->getIterator(), OutlinedFn); |
| |
| // Remove the artificial entry introduced by the extractor right away, we |
| // made our own entry block after all. |
| { |
| BasicBlock &ArtificialEntry = OutlinedFn->getEntryBlock(); |
| assert(ArtificialEntry.getUniqueSuccessor() == OI.EntryBB); |
| assert(OI.EntryBB->getUniquePredecessor() == &ArtificialEntry); |
| OI.EntryBB->moveBefore(&ArtificialEntry); |
| ArtificialEntry.eraseFromParent(); |
| } |
| assert(&OutlinedFn->getEntryBlock() == OI.EntryBB); |
| assert(OutlinedFn && OutlinedFn->getNumUses() == 1); |
| |
| // Run a user callback, e.g. to add attributes. |
| if (OI.PostOutlineCB) |
| OI.PostOutlineCB(*OutlinedFn); |
| } |
| |
| // Allow finalize to be called multiple times. |
| OutlineInfos.clear(); |
| } |
| |
| Value *OpenMPIRBuilder::getOrCreateIdent(Constant *SrcLocStr, |
| IdentFlag LocFlags, |
| unsigned Reserve2Flags) { |
| // Enable "C-mode". |
| LocFlags |= OMP_IDENT_FLAG_KMPC; |
| |
| Value *&Ident = |
| IdentMap[{SrcLocStr, uint64_t(LocFlags) << 31 | Reserve2Flags}]; |
| if (!Ident) { |
| Constant *I32Null = ConstantInt::getNullValue(Int32); |
| Constant *IdentData[] = { |
| I32Null, ConstantInt::get(Int32, uint32_t(LocFlags)), |
| ConstantInt::get(Int32, Reserve2Flags), I32Null, SrcLocStr}; |
| Constant *Initializer = ConstantStruct::get( |
| cast<StructType>(IdentPtr->getPointerElementType()), IdentData); |
| |
| // Look for existing encoding of the location + flags, not needed but |
| // minimizes the difference to the existing solution while we transition. |
| for (GlobalVariable &GV : M.getGlobalList()) |
| if (GV.getType() == IdentPtr && GV.hasInitializer()) |
| if (GV.getInitializer() == Initializer) |
| return Ident = &GV; |
| |
| auto *GV = new GlobalVariable(M, IdentPtr->getPointerElementType(), |
| /* isConstant = */ true, |
| GlobalValue::PrivateLinkage, Initializer); |
| GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); |
| GV->setAlignment(Align(8)); |
| Ident = GV; |
| } |
| return Ident; |
| } |
| |
| Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(StringRef LocStr) { |
| Constant *&SrcLocStr = SrcLocStrMap[LocStr]; |
| if (!SrcLocStr) { |
| Constant *Initializer = |
| ConstantDataArray::getString(M.getContext(), LocStr); |
| |
| // Look for existing encoding of the location, not needed but minimizes the |
| // difference to the existing solution while we transition. |
| for (GlobalVariable &GV : M.getGlobalList()) |
| if (GV.isConstant() && GV.hasInitializer() && |
| GV.getInitializer() == Initializer) |
| return SrcLocStr = ConstantExpr::getPointerCast(&GV, Int8Ptr); |
| |
| SrcLocStr = Builder.CreateGlobalStringPtr(LocStr, /* Name */ "", |
| /* AddressSpace */ 0, &M); |
| } |
| return SrcLocStr; |
| } |
| |
| Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(StringRef FunctionName, |
| StringRef FileName, |
| unsigned Line, |
| unsigned Column) { |
| SmallString<128> Buffer; |
| Buffer.push_back(';'); |
| Buffer.append(FileName); |
| Buffer.push_back(';'); |
| Buffer.append(FunctionName); |
| Buffer.push_back(';'); |
| Buffer.append(std::to_string(Line)); |
| Buffer.push_back(';'); |
| Buffer.append(std::to_string(Column)); |
| Buffer.push_back(';'); |
| Buffer.push_back(';'); |
| return getOrCreateSrcLocStr(Buffer.str()); |
| } |
| |
| Constant *OpenMPIRBuilder::getOrCreateDefaultSrcLocStr() { |
| return getOrCreateSrcLocStr(";unknown;unknown;0;0;;"); |
| } |
| |
| Constant * |
| OpenMPIRBuilder::getOrCreateSrcLocStr(const LocationDescription &Loc) { |
| DILocation *DIL = Loc.DL.get(); |
| if (!DIL) |
| return getOrCreateDefaultSrcLocStr(); |
| StringRef FileName = |
| !DIL->getFilename().empty() ? DIL->getFilename() : M.getName(); |
| StringRef Function = DIL->getScope()->getSubprogram()->getName(); |
| Function = |
| !Function.empty() ? Function : Loc.IP.getBlock()->getParent()->getName(); |
| return getOrCreateSrcLocStr(Function, FileName, DIL->getLine(), |
| DIL->getColumn()); |
| } |
| |
| Value *OpenMPIRBuilder::getOrCreateThreadID(Value *Ident) { |
| return Builder.CreateCall( |
| getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_global_thread_num), Ident, |
| "omp_global_thread_num"); |
| } |
| |
| OpenMPIRBuilder::InsertPointTy |
| OpenMPIRBuilder::CreateBarrier(const LocationDescription &Loc, Directive DK, |
| bool ForceSimpleCall, bool CheckCancelFlag) { |
| if (!updateToLocation(Loc)) |
| return Loc.IP; |
| return emitBarrierImpl(Loc, DK, ForceSimpleCall, CheckCancelFlag); |
| } |
| |
| OpenMPIRBuilder::InsertPointTy |
| OpenMPIRBuilder::emitBarrierImpl(const LocationDescription &Loc, Directive Kind, |
| bool ForceSimpleCall, bool CheckCancelFlag) { |
| // Build call __kmpc_cancel_barrier(loc, thread_id) or |
| // __kmpc_barrier(loc, thread_id); |
| |
| IdentFlag BarrierLocFlags; |
| switch (Kind) { |
| case OMPD_for: |
| BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_FOR; |
| break; |
| case OMPD_sections: |
| BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SECTIONS; |
| break; |
| case OMPD_single: |
| BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SINGLE; |
| break; |
| case OMPD_barrier: |
| BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_EXPL; |
| break; |
| default: |
| BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL; |
| break; |
| } |
| |
| Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); |
| Value *Args[] = {getOrCreateIdent(SrcLocStr, BarrierLocFlags), |
| getOrCreateThreadID(getOrCreateIdent(SrcLocStr))}; |
| |
| // If we are in a cancellable parallel region, barriers are cancellation |
| // points. |
| // TODO: Check why we would force simple calls or to ignore the cancel flag. |
| bool UseCancelBarrier = |
| !ForceSimpleCall && isLastFinalizationInfoCancellable(OMPD_parallel); |
| |
| Value *Result = |
| Builder.CreateCall(getOrCreateRuntimeFunctionPtr( |
| UseCancelBarrier ? OMPRTL___kmpc_cancel_barrier |
| : OMPRTL___kmpc_barrier), |
| Args); |
| |
| if (UseCancelBarrier && CheckCancelFlag) |
| emitCancelationCheckImpl(Result, OMPD_parallel); |
| |
| return Builder.saveIP(); |
| } |
| |
| OpenMPIRBuilder::InsertPointTy |
| OpenMPIRBuilder::CreateCancel(const LocationDescription &Loc, |
| Value *IfCondition, |
| omp::Directive CanceledDirective) { |
| if (!updateToLocation(Loc)) |
| return Loc.IP; |
| |
| // LLVM utilities like blocks with terminators. |
| auto *UI = Builder.CreateUnreachable(); |
| |
| Instruction *ThenTI = UI, *ElseTI = nullptr; |
| if (IfCondition) |
| SplitBlockAndInsertIfThenElse(IfCondition, UI, &ThenTI, &ElseTI); |
| Builder.SetInsertPoint(ThenTI); |
| |
| Value *CancelKind = nullptr; |
| switch (CanceledDirective) { |
| #define OMP_CANCEL_KIND(Enum, Str, DirectiveEnum, Value) \ |
| case DirectiveEnum: \ |
| CancelKind = Builder.getInt32(Value); \ |
| break; |
| #include "llvm/Frontend/OpenMP/OMPKinds.def" |
| default: |
| llvm_unreachable("Unknown cancel kind!"); |
| } |
| |
| Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); |
| Value *Ident = getOrCreateIdent(SrcLocStr); |
| Value *Args[] = {Ident, getOrCreateThreadID(Ident), CancelKind}; |
| Value *Result = Builder.CreateCall( |
| getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_cancel), Args); |
| |
| // The actual cancel logic is shared with others, e.g., cancel_barriers. |
| emitCancelationCheckImpl(Result, CanceledDirective); |
| |
| // Update the insertion point and remove the terminator we introduced. |
| Builder.SetInsertPoint(UI->getParent()); |
| UI->eraseFromParent(); |
| |
| return Builder.saveIP(); |
| } |
| |
| void OpenMPIRBuilder::emitCancelationCheckImpl( |
| Value *CancelFlag, omp::Directive CanceledDirective) { |
| assert(isLastFinalizationInfoCancellable(CanceledDirective) && |
| "Unexpected cancellation!"); |
| |
| // For a cancel barrier we create two new blocks. |
| BasicBlock *BB = Builder.GetInsertBlock(); |
| BasicBlock *NonCancellationBlock; |
| if (Builder.GetInsertPoint() == BB->end()) { |
| // TODO: This branch will not be needed once we moved to the |
| // OpenMPIRBuilder codegen completely. |
| NonCancellationBlock = BasicBlock::Create( |
| BB->getContext(), BB->getName() + ".cont", BB->getParent()); |
| } else { |
| NonCancellationBlock = SplitBlock(BB, &*Builder.GetInsertPoint()); |
| BB->getTerminator()->eraseFromParent(); |
| Builder.SetInsertPoint(BB); |
| } |
| BasicBlock *CancellationBlock = BasicBlock::Create( |
| BB->getContext(), BB->getName() + ".cncl", BB->getParent()); |
| |
| // Jump to them based on the return value. |
| Value *Cmp = Builder.CreateIsNull(CancelFlag); |
| Builder.CreateCondBr(Cmp, NonCancellationBlock, CancellationBlock, |
| /* TODO weight */ nullptr, nullptr); |
| |
| // From the cancellation block we finalize all variables and go to the |
| // post finalization block that is known to the FiniCB callback. |
| Builder.SetInsertPoint(CancellationBlock); |
| auto &FI = FinalizationStack.back(); |
| FI.FiniCB(Builder.saveIP()); |
| |
| // The continuation block is where code generation continues. |
| Builder.SetInsertPoint(NonCancellationBlock, NonCancellationBlock->begin()); |
| } |
| |
| IRBuilder<>::InsertPoint OpenMPIRBuilder::CreateParallel( |
| const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB, |
| PrivatizeCallbackTy PrivCB, FinalizeCallbackTy FiniCB, Value *IfCondition, |
| Value *NumThreads, omp::ProcBindKind ProcBind, bool IsCancellable) { |
| if (!updateToLocation(Loc)) |
| return Loc.IP; |
| |
| Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); |
| Value *Ident = getOrCreateIdent(SrcLocStr); |
| Value *ThreadID = getOrCreateThreadID(Ident); |
| |
| if (NumThreads) { |
| // Build call __kmpc_push_num_threads(&Ident, global_tid, num_threads) |
| Value *Args[] = { |
| Ident, ThreadID, |
| Builder.CreateIntCast(NumThreads, Int32, /*isSigned*/ false)}; |
| Builder.CreateCall( |
| getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_push_num_threads), Args); |
| } |
| |
| if (ProcBind != OMP_PROC_BIND_default) { |
| // Build call __kmpc_push_proc_bind(&Ident, global_tid, proc_bind) |
| Value *Args[] = { |
| Ident, ThreadID, |
| ConstantInt::get(Int32, unsigned(ProcBind), /*isSigned=*/true)}; |
| Builder.CreateCall( |
| getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_push_proc_bind), Args); |
| } |
| |
| BasicBlock *InsertBB = Builder.GetInsertBlock(); |
| Function *OuterFn = InsertBB->getParent(); |
| |
| // Vector to remember instructions we used only during the modeling but which |
| // we want to delete at the end. |
| SmallVector<Instruction *, 4> ToBeDeleted; |
| |
| Builder.SetInsertPoint(OuterFn->getEntryBlock().getFirstNonPHI()); |
| AllocaInst *TIDAddr = Builder.CreateAlloca(Int32, nullptr, "tid.addr"); |
| AllocaInst *ZeroAddr = Builder.CreateAlloca(Int32, nullptr, "zero.addr"); |
| |
| // If there is an if condition we actually use the TIDAddr and ZeroAddr in the |
| // program, otherwise we only need them for modeling purposes to get the |
| // associated arguments in the outlined function. In the former case, |
| // initialize the allocas properly, in the latter case, delete them later. |
| if (IfCondition) { |
| Builder.CreateStore(Constant::getNullValue(Int32), TIDAddr); |
| Builder.CreateStore(Constant::getNullValue(Int32), ZeroAddr); |
| } else { |
| ToBeDeleted.push_back(TIDAddr); |
| ToBeDeleted.push_back(ZeroAddr); |
| } |
| |
| // Create an artificial insertion point that will also ensure the blocks we |
| // are about to split are not degenerated. |
| auto *UI = new UnreachableInst(Builder.getContext(), InsertBB); |
| |
| Instruction *ThenTI = UI, *ElseTI = nullptr; |
| if (IfCondition) |
| SplitBlockAndInsertIfThenElse(IfCondition, UI, &ThenTI, &ElseTI); |
| |
| BasicBlock *ThenBB = ThenTI->getParent(); |
| BasicBlock *PRegEntryBB = ThenBB->splitBasicBlock(ThenTI, "omp.par.entry"); |
| BasicBlock *PRegBodyBB = |
| PRegEntryBB->splitBasicBlock(ThenTI, "omp.par.region"); |
| BasicBlock *PRegPreFiniBB = |
| PRegBodyBB->splitBasicBlock(ThenTI, "omp.par.pre_finalize"); |
| BasicBlock *PRegExitBB = |
| PRegPreFiniBB->splitBasicBlock(ThenTI, "omp.par.exit"); |
| |
| auto FiniCBWrapper = [&](InsertPointTy IP) { |
| // Hide "open-ended" blocks from the given FiniCB by setting the right jump |
| // target to the region exit block. |
| if (IP.getBlock()->end() == IP.getPoint()) { |
| IRBuilder<>::InsertPointGuard IPG(Builder); |
| Builder.restoreIP(IP); |
| Instruction *I = Builder.CreateBr(PRegExitBB); |
| IP = InsertPointTy(I->getParent(), I->getIterator()); |
| } |
| assert(IP.getBlock()->getTerminator()->getNumSuccessors() == 1 && |
| IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB && |
| "Unexpected insertion point for finalization call!"); |
| return FiniCB(IP); |
| }; |
| |
| FinalizationStack.push_back({FiniCBWrapper, OMPD_parallel, IsCancellable}); |
| |
| // Generate the privatization allocas in the block that will become the entry |
| // of the outlined function. |
| InsertPointTy AllocaIP(PRegEntryBB, |
| PRegEntryBB->getTerminator()->getIterator()); |
| Builder.restoreIP(AllocaIP); |
| AllocaInst *PrivTIDAddr = |
| Builder.CreateAlloca(Int32, nullptr, "tid.addr.local"); |
| Instruction *PrivTID = Builder.CreateLoad(PrivTIDAddr, "tid"); |
| |
| // Add some fake uses for OpenMP provided arguments. |
| ToBeDeleted.push_back(Builder.CreateLoad(TIDAddr, "tid.addr.use")); |
| ToBeDeleted.push_back(Builder.CreateLoad(ZeroAddr, "zero.addr.use")); |
| |
| // ThenBB |
| // | |
| // V |
| // PRegionEntryBB <- Privatization allocas are placed here. |
| // | |
| // V |
| // PRegionBodyBB <- BodeGen is invoked here. |
| // | |
| // V |
| // PRegPreFiniBB <- The block we will start finalization from. |
| // | |
| // V |
| // PRegionExitBB <- A common exit to simplify block collection. |
| // |
| |
| LLVM_DEBUG(dbgs() << "Before body codegen: " << *OuterFn << "\n"); |
| |
| // Let the caller create the body. |
| assert(BodyGenCB && "Expected body generation callback!"); |
| InsertPointTy CodeGenIP(PRegBodyBB, PRegBodyBB->begin()); |
| BodyGenCB(AllocaIP, CodeGenIP, *PRegPreFiniBB); |
| |
| LLVM_DEBUG(dbgs() << "After body codegen: " << *OuterFn << "\n"); |
| |
| FunctionCallee RTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_call); |
| if (auto *F = dyn_cast<llvm::Function>(RTLFn.getCallee())) { |
| if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) { |
| llvm::LLVMContext &Ctx = F->getContext(); |
| MDBuilder MDB(Ctx); |
| // Annotate the callback behavior of the __kmpc_fork_call: |
| // - The callback callee is argument number 2 (microtask). |
| // - The first two arguments of the callback callee are unknown (-1). |
| // - All variadic arguments to the __kmpc_fork_call are passed to the |
| // callback callee. |
| F->addMetadata( |
| llvm::LLVMContext::MD_callback, |
| *llvm::MDNode::get( |
| Ctx, {MDB.createCallbackEncoding(2, {-1, -1}, |
| /* VarArgsArePassed */ true)})); |
| } |
| } |
| |
| OutlineInfo OI; |
| OI.PostOutlineCB = [=](Function &OutlinedFn) { |
| // Add some known attributes. |
| OutlinedFn.addParamAttr(0, Attribute::NoAlias); |
| OutlinedFn.addParamAttr(1, Attribute::NoAlias); |
| OutlinedFn.addFnAttr(Attribute::NoUnwind); |
| OutlinedFn.addFnAttr(Attribute::NoRecurse); |
| |
| assert(OutlinedFn.arg_size() >= 2 && |
| "Expected at least tid and bounded tid as arguments"); |
| unsigned NumCapturedVars = |
| OutlinedFn.arg_size() - /* tid & bounded tid */ 2; |
| |
| CallInst *CI = cast<CallInst>(OutlinedFn.user_back()); |
| CI->getParent()->setName("omp_parallel"); |
| Builder.SetInsertPoint(CI); |
| |
| // Build call __kmpc_fork_call(Ident, n, microtask, var1, .., varn); |
| Value *ForkCallArgs[] = { |
| Ident, Builder.getInt32(NumCapturedVars), |
| Builder.CreateBitCast(&OutlinedFn, ParallelTaskPtr)}; |
| |
| SmallVector<Value *, 16> RealArgs; |
| RealArgs.append(std::begin(ForkCallArgs), std::end(ForkCallArgs)); |
| RealArgs.append(CI->arg_begin() + /* tid & bound tid */ 2, CI->arg_end()); |
| |
| Builder.CreateCall(RTLFn, RealArgs); |
| |
| LLVM_DEBUG(dbgs() << "With fork_call placed: " |
| << *Builder.GetInsertBlock()->getParent() << "\n"); |
| |
| InsertPointTy ExitIP(PRegExitBB, PRegExitBB->end()); |
| |
| // Initialize the local TID stack location with the argument value. |
| Builder.SetInsertPoint(PrivTID); |
| Function::arg_iterator OutlinedAI = OutlinedFn.arg_begin(); |
| Builder.CreateStore(Builder.CreateLoad(OutlinedAI), PrivTIDAddr); |
| |
| // If no "if" clause was present we do not need the call created during |
| // outlining, otherwise we reuse it in the serialized parallel region. |
| if (!ElseTI) { |
| CI->eraseFromParent(); |
| } else { |
| |
| // If an "if" clause was present we are now generating the serialized |
| // version into the "else" branch. |
| Builder.SetInsertPoint(ElseTI); |
| |
| // Build calls __kmpc_serialized_parallel(&Ident, GTid); |
| Value *SerializedParallelCallArgs[] = {Ident, ThreadID}; |
| Builder.CreateCall( |
| getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_serialized_parallel), |
| SerializedParallelCallArgs); |
| |
| // OutlinedFn(>id, &zero, CapturedStruct); |
| CI->removeFromParent(); |
| Builder.Insert(CI); |
| |
| // __kmpc_end_serialized_parallel(&Ident, GTid); |
| Value *EndArgs[] = {Ident, ThreadID}; |
| Builder.CreateCall( |
| getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_serialized_parallel), |
| EndArgs); |
| |
| LLVM_DEBUG(dbgs() << "With serialized parallel region: " |
| << *Builder.GetInsertBlock()->getParent() << "\n"); |
| } |
| |
| for (Instruction *I : ToBeDeleted) |
| I->eraseFromParent(); |
| }; |
| |
| // Adjust the finalization stack, verify the adjustment, and call the |
| // finalize function a last time to finalize values between the pre-fini |
| // block and the exit block if we left the parallel "the normal way". |
| auto FiniInfo = FinalizationStack.pop_back_val(); |
| (void)FiniInfo; |
| assert(FiniInfo.DK == OMPD_parallel && |
| "Unexpected finalization stack state!"); |
| |
| Instruction *PRegPreFiniTI = PRegPreFiniBB->getTerminator(); |
| |
| InsertPointTy PreFiniIP(PRegPreFiniBB, PRegPreFiniTI->getIterator()); |
| FiniCB(PreFiniIP); |
| |
| OI.EntryBB = PRegEntryBB; |
| OI.ExitBB = PRegExitBB; |
| |
| SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet; |
| SmallVector<BasicBlock *, 32> Blocks; |
| OI.collectBlocks(ParallelRegionBlockSet, Blocks); |
| |
| // Ensure a single exit node for the outlined region by creating one. |
| // We might have multiple incoming edges to the exit now due to finalizations, |
| // e.g., cancel calls that cause the control flow to leave the region. |
| BasicBlock *PRegOutlinedExitBB = PRegExitBB; |
| PRegExitBB = SplitBlock(PRegExitBB, &*PRegExitBB->getFirstInsertionPt()); |
| PRegOutlinedExitBB->setName("omp.par.outlined.exit"); |
| Blocks.push_back(PRegOutlinedExitBB); |
| |
| CodeExtractorAnalysisCache CEAC(*OuterFn); |
| CodeExtractor Extractor(Blocks, /* DominatorTree */ nullptr, |
| /* AggregateArgs */ false, |
| /* BlockFrequencyInfo */ nullptr, |
| /* BranchProbabilityInfo */ nullptr, |
| /* AssumptionCache */ nullptr, |
| /* AllowVarArgs */ true, |
| /* AllowAlloca */ true, |
| /* Suffix */ ".omp_par"); |
| |
| // Find inputs to, outputs from the code region. |
| BasicBlock *CommonExit = nullptr; |
| SetVector<Value *> Inputs, Outputs, SinkingCands, HoistingCands; |
| Extractor.findAllocas(CEAC, SinkingCands, HoistingCands, CommonExit); |
| Extractor.findInputsOutputs(Inputs, Outputs, SinkingCands); |
| |
| LLVM_DEBUG(dbgs() << "Before privatization: " << *OuterFn << "\n"); |
| |
| FunctionCallee TIDRTLFn = |
| getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_global_thread_num); |
| |
| auto PrivHelper = [&](Value &V) { |
| if (&V == TIDAddr || &V == ZeroAddr) |
| return; |
| |
| SmallVector<Use *, 8> Uses; |
| for (Use &U : V.uses()) |
| if (auto *UserI = dyn_cast<Instruction>(U.getUser())) |
| if (ParallelRegionBlockSet.count(UserI->getParent())) |
| Uses.push_back(&U); |
| |
| Value *ReplacementValue = nullptr; |
| CallInst *CI = dyn_cast<CallInst>(&V); |
| if (CI && CI->getCalledFunction() == TIDRTLFn.getCallee()) { |
| ReplacementValue = PrivTID; |
| } else { |
| Builder.restoreIP( |
| PrivCB(AllocaIP, Builder.saveIP(), V, ReplacementValue)); |
| assert(ReplacementValue && |
| "Expected copy/create callback to set replacement value!"); |
| if (ReplacementValue == &V) |
| return; |
| } |
| |
| for (Use *UPtr : Uses) |
| UPtr->set(ReplacementValue); |
| }; |
| |
| for (Value *Input : Inputs) { |
| LLVM_DEBUG(dbgs() << "Captured input: " << *Input << "\n"); |
| PrivHelper(*Input); |
| } |
| assert(Outputs.empty() && |
| "OpenMP outlining should not produce live-out values!"); |
| |
| LLVM_DEBUG(dbgs() << "After privatization: " << *OuterFn << "\n"); |
| LLVM_DEBUG({ |
| for (auto *BB : Blocks) |
| dbgs() << " PBR: " << BB->getName() << "\n"; |
| }); |
| |
| // Register the outlined info. |
| addOutlineInfo(std::move(OI)); |
| |
| InsertPointTy AfterIP(UI->getParent(), UI->getParent()->end()); |
| UI->eraseFromParent(); |
| |
| return AfterIP; |
| } |
| |
| void OpenMPIRBuilder::emitFlush(const LocationDescription &Loc) { |
| // Build call void __kmpc_flush(ident_t *loc) |
| Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); |
| Value *Args[] = {getOrCreateIdent(SrcLocStr)}; |
| |
| Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_flush), Args); |
| } |
| |
| void OpenMPIRBuilder::CreateFlush(const LocationDescription &Loc) { |
| if (!updateToLocation(Loc)) |
| return; |
| emitFlush(Loc); |
| } |
| |
| void OpenMPIRBuilder::emitTaskwaitImpl(const LocationDescription &Loc) { |
| // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32 |
| // global_tid); |
| Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); |
| Value *Ident = getOrCreateIdent(SrcLocStr); |
| Value *Args[] = {Ident, getOrCreateThreadID(Ident)}; |
| |
| // Ignore return result until untied tasks are supported. |
| Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_taskwait), |
| Args); |
| } |
| |
| void OpenMPIRBuilder::CreateTaskwait(const LocationDescription &Loc) { |
| if (!updateToLocation(Loc)) |
| return; |
| emitTaskwaitImpl(Loc); |
| } |
| |
| void OpenMPIRBuilder::emitTaskyieldImpl(const LocationDescription &Loc) { |
| // Build call __kmpc_omp_taskyield(loc, thread_id, 0); |
| Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); |
| Value *Ident = getOrCreateIdent(SrcLocStr); |
| Constant *I32Null = ConstantInt::getNullValue(Int32); |
| Value *Args[] = {Ident, getOrCreateThreadID(Ident), I32Null}; |
| |
| Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_taskyield), |
| Args); |
| } |
| |
| void OpenMPIRBuilder::CreateTaskyield(const LocationDescription &Loc) { |
| if (!updateToLocation(Loc)) |
| return; |
| emitTaskyieldImpl(Loc); |
| } |
| |
| OpenMPIRBuilder::InsertPointTy |
| OpenMPIRBuilder::CreateMaster(const LocationDescription &Loc, |
| BodyGenCallbackTy BodyGenCB, |
| FinalizeCallbackTy FiniCB) { |
| |
| if (!updateToLocation(Loc)) |
| return Loc.IP; |
| |
| Directive OMPD = Directive::OMPD_master; |
| Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); |
| Value *Ident = getOrCreateIdent(SrcLocStr); |
| Value *ThreadId = getOrCreateThreadID(Ident); |
| Value *Args[] = {Ident, ThreadId}; |
| |
| Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_master); |
| Instruction *EntryCall = Builder.CreateCall(EntryRTLFn, Args); |
| |
| Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_master); |
| Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args); |
| |
| return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB, |
| /*Conditional*/ true, /*hasFinalize*/ true); |
| } |
| |
| OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::CreateCritical( |
| const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB, |
| FinalizeCallbackTy FiniCB, StringRef CriticalName, Value *HintInst) { |
| |
| if (!updateToLocation(Loc)) |
| return Loc.IP; |
| |
| Directive OMPD = Directive::OMPD_critical; |
| Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); |
| Value *Ident = getOrCreateIdent(SrcLocStr); |
| Value *ThreadId = getOrCreateThreadID(Ident); |
| Value *LockVar = getOMPCriticalRegionLock(CriticalName); |
| Value *Args[] = {Ident, ThreadId, LockVar}; |
| |
| SmallVector<llvm::Value *, 4> EnterArgs(std::begin(Args), std::end(Args)); |
| Function *RTFn = nullptr; |
| if (HintInst) { |
| // Add Hint to entry Args and create call |
| EnterArgs.push_back(HintInst); |
| RTFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_critical_with_hint); |
| } else { |
| RTFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_critical); |
| } |
| Instruction *EntryCall = Builder.CreateCall(RTFn, EnterArgs); |
| |
| Function *ExitRTLFn = |
| getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_critical); |
| Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args); |
| |
| return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB, |
| /*Conditional*/ false, /*hasFinalize*/ true); |
| } |
| |
| OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::EmitOMPInlinedRegion( |
| Directive OMPD, Instruction *EntryCall, Instruction *ExitCall, |
| BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, bool Conditional, |
| bool HasFinalize) { |
| |
| if (HasFinalize) |
| FinalizationStack.push_back({FiniCB, OMPD, /*IsCancellable*/ false}); |
| |
| // Create inlined region's entry and body blocks, in preparation |
| // for conditional creation |
| BasicBlock *EntryBB = Builder.GetInsertBlock(); |
| Instruction *SplitPos = EntryBB->getTerminator(); |
| if (!isa_and_nonnull<BranchInst>(SplitPos)) |
| SplitPos = new UnreachableInst(Builder.getContext(), EntryBB); |
| BasicBlock *ExitBB = EntryBB->splitBasicBlock(SplitPos, "omp_region.end"); |
| BasicBlock *FiniBB = |
| EntryBB->splitBasicBlock(EntryBB->getTerminator(), "omp_region.finalize"); |
| |
| Builder.SetInsertPoint(EntryBB->getTerminator()); |
| emitCommonDirectiveEntry(OMPD, EntryCall, ExitBB, Conditional); |
| |
| // generate body |
| BodyGenCB(/* AllocaIP */ InsertPointTy(), |
| /* CodeGenIP */ Builder.saveIP(), *FiniBB); |
| |
| // If we didn't emit a branch to FiniBB during body generation, it means |
| // FiniBB is unreachable (e.g. while(1);). stop generating all the |
| // unreachable blocks, and remove anything we are not going to use. |
| auto SkipEmittingRegion = FiniBB->hasNPredecessors(0); |
| if (SkipEmittingRegion) { |
| FiniBB->eraseFromParent(); |
| ExitCall->eraseFromParent(); |
| // Discard finalization if we have it. |
| if (HasFinalize) { |
| assert(!FinalizationStack.empty() && |
| "Unexpected finalization stack state!"); |
| FinalizationStack.pop_back(); |
| } |
| } else { |
| // emit exit call and do any needed finalization. |
| auto FinIP = InsertPointTy(FiniBB, FiniBB->getFirstInsertionPt()); |
| assert(FiniBB->getTerminator()->getNumSuccessors() == 1 && |
| FiniBB->getTerminator()->getSuccessor(0) == ExitBB && |
| "Unexpected control flow graph state!!"); |
| emitCommonDirectiveExit(OMPD, FinIP, ExitCall, HasFinalize); |
| assert(FiniBB->getUniquePredecessor()->getUniqueSuccessor() == FiniBB && |
| "Unexpected Control Flow State!"); |
| MergeBlockIntoPredecessor(FiniBB); |
| } |
| |
| // If we are skipping the region of a non conditional, remove the exit |
| // block, and clear the builder's insertion point. |
| assert(SplitPos->getParent() == ExitBB && |
| "Unexpected Insertion point location!"); |
| if (!Conditional && SkipEmittingRegion) { |
| ExitBB->eraseFromParent(); |
| Builder.ClearInsertionPoint(); |
| } else { |
| auto merged = MergeBlockIntoPredecessor(ExitBB); |
| BasicBlock *ExitPredBB = SplitPos->getParent(); |
| auto InsertBB = merged ? ExitPredBB : ExitBB; |
| if (!isa_and_nonnull<BranchInst>(SplitPos)) |
| SplitPos->eraseFromParent(); |
| Builder.SetInsertPoint(InsertBB); |
| } |
| |
| return Builder.saveIP(); |
| } |
| |
| OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveEntry( |
| Directive OMPD, Value *EntryCall, BasicBlock *ExitBB, bool Conditional) { |
| |
| // if nothing to do, Return current insertion point. |
| if (!Conditional) |
| return Builder.saveIP(); |
| |
| BasicBlock *EntryBB = Builder.GetInsertBlock(); |
| Value *CallBool = Builder.CreateIsNotNull(EntryCall); |
| auto *ThenBB = BasicBlock::Create(M.getContext(), "omp_region.body"); |
| auto *UI = new UnreachableInst(Builder.getContext(), ThenBB); |
| |
| // Emit thenBB and set the Builder's insertion point there for |
| // body generation next. Place the block after the current block. |
| Function *CurFn = EntryBB->getParent(); |
| CurFn->getBasicBlockList().insertAfter(EntryBB->getIterator(), ThenBB); |
| |
| // Move Entry branch to end of ThenBB, and replace with conditional |
| // branch (If-stmt) |
| Instruction *EntryBBTI = EntryBB->getTerminator(); |
| Builder.CreateCondBr(CallBool, ThenBB, ExitBB); |
| EntryBBTI->removeFromParent(); |
| Builder.SetInsertPoint(UI); |
| Builder.Insert(EntryBBTI); |
| UI->eraseFromParent(); |
| Builder.SetInsertPoint(ThenBB->getTerminator()); |
| |
| // return an insertion point to ExitBB. |
| return IRBuilder<>::InsertPoint(ExitBB, ExitBB->getFirstInsertionPt()); |
| } |
| |
| OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveExit( |
| omp::Directive OMPD, InsertPointTy FinIP, Instruction *ExitCall, |
| bool HasFinalize) { |
| |
| Builder.restoreIP(FinIP); |
| |
| // If there is finalization to do, emit it before the exit call |
| if (HasFinalize) { |
| assert(!FinalizationStack.empty() && |
| "Unexpected finalization stack state!"); |
| |
| FinalizationInfo Fi = FinalizationStack.pop_back_val(); |
| assert(Fi.DK == OMPD && "Unexpected Directive for Finalization call!"); |
| |
| Fi.FiniCB(FinIP); |
| |
| BasicBlock *FiniBB = FinIP.getBlock(); |
| Instruction *FiniBBTI = FiniBB->getTerminator(); |
| |
| // set Builder IP for call creation |
| Builder.SetInsertPoint(FiniBBTI); |
| } |
| |
| // place the Exitcall as last instruction before Finalization block terminator |
| ExitCall->removeFromParent(); |
| Builder.Insert(ExitCall); |
| |
| return IRBuilder<>::InsertPoint(ExitCall->getParent(), |
| ExitCall->getIterator()); |
| } |
| |
| OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::CreateCopyinClauseBlocks( |
| InsertPointTy IP, Value *MasterAddr, Value *PrivateAddr, |
| llvm::IntegerType *IntPtrTy, bool BranchtoEnd) { |
| if (!IP.isSet()) |
| return IP; |
| |
| IRBuilder<>::InsertPointGuard IPG(Builder); |
| |
| // creates the following CFG structure |
| // OMP_Entry : (MasterAddr != PrivateAddr)? |
| // F T |
| // | \ |
| // | copin.not.master |
| // | / |
| // v / |
| // copyin.not.master.end |
| // | |
| // v |
| // OMP.Entry.Next |
| |
| BasicBlock *OMP_Entry = IP.getBlock(); |
| Function *CurFn = OMP_Entry->getParent(); |
| BasicBlock *CopyBegin = |
| BasicBlock::Create(M.getContext(), "copyin.not.master", CurFn); |
| BasicBlock *CopyEnd = nullptr; |
| |
| // If entry block is terminated, split to preserve the branch to following |
| // basic block (i.e. OMP.Entry.Next), otherwise, leave everything as is. |
| if (isa_and_nonnull<BranchInst>(OMP_Entry->getTerminator())) { |
| CopyEnd = OMP_Entry->splitBasicBlock(OMP_Entry->getTerminator(), |
| "copyin.not.master.end"); |
| OMP_Entry->getTerminator()->eraseFromParent(); |
| } else { |
| CopyEnd = |
| BasicBlock::Create(M.getContext(), "copyin.not.master.end", CurFn); |
| } |
| |
| Builder.SetInsertPoint(OMP_Entry); |
| Value *MasterPtr = Builder.CreatePtrToInt(MasterAddr, IntPtrTy); |
| Value *PrivatePtr = Builder.CreatePtrToInt(PrivateAddr, IntPtrTy); |
| Value *cmp = Builder.CreateICmpNE(MasterPtr, PrivatePtr); |
| Builder.CreateCondBr(cmp, CopyBegin, CopyEnd); |
| |
| Builder.SetInsertPoint(CopyBegin); |
| if (BranchtoEnd) |
| Builder.SetInsertPoint(Builder.CreateBr(CopyEnd)); |
| |
| return Builder.saveIP(); |
| } |
| |
| CallInst *OpenMPIRBuilder::CreateOMPAlloc(const LocationDescription &Loc, |
| Value *Size, Value *Allocator, |
| std::string Name) { |
| IRBuilder<>::InsertPointGuard IPG(Builder); |
| Builder.restoreIP(Loc.IP); |
| |
| Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); |
| Value *Ident = getOrCreateIdent(SrcLocStr); |
| Value *ThreadId = getOrCreateThreadID(Ident); |
| Value *Args[] = {ThreadId, Size, Allocator}; |
| |
| Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_alloc); |
| |
| return Builder.CreateCall(Fn, Args, Name); |
| } |
| |
| CallInst *OpenMPIRBuilder::CreateOMPFree(const LocationDescription &Loc, |
| Value *Addr, Value *Allocator, |
| std::string Name) { |
| IRBuilder<>::InsertPointGuard IPG(Builder); |
| Builder.restoreIP(Loc.IP); |
| |
| Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); |
| Value *Ident = getOrCreateIdent(SrcLocStr); |
| Value *ThreadId = getOrCreateThreadID(Ident); |
| Value *Args[] = {ThreadId, Addr, Allocator}; |
| Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_free); |
| return Builder.CreateCall(Fn, Args, Name); |
| } |
| |
| CallInst *OpenMPIRBuilder::CreateCachedThreadPrivate( |
| const LocationDescription &Loc, llvm::Value *Pointer, |
| llvm::ConstantInt *Size, const llvm::Twine &Name) { |
| IRBuilder<>::InsertPointGuard IPG(Builder); |
| Builder.restoreIP(Loc.IP); |
| |
| Constant *SrcLocStr = getOrCreateSrcLocStr(Loc); |
| Value *Ident = getOrCreateIdent(SrcLocStr); |
| Value *ThreadId = getOrCreateThreadID(Ident); |
| Constant *ThreadPrivateCache = |
| getOrCreateOMPInternalVariable(Int8PtrPtr, Name); |
| llvm::Value *Args[] = {Ident, ThreadId, Pointer, Size, ThreadPrivateCache}; |
| |
| Function *Fn = |
| getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_threadprivate_cached); |
| |
| return Builder.CreateCall(Fn, Args); |
| } |
| |
| std::string OpenMPIRBuilder::getNameWithSeparators(ArrayRef<StringRef> Parts, |
| StringRef FirstSeparator, |
| StringRef Separator) { |
| SmallString<128> Buffer; |
| llvm::raw_svector_ostream OS(Buffer); |
| StringRef Sep = FirstSeparator; |
| for (StringRef Part : Parts) { |
| OS << Sep << Part; |
| Sep = Separator; |
| } |
| return OS.str().str(); |
| } |
| |
| Constant *OpenMPIRBuilder::getOrCreateOMPInternalVariable( |
| llvm::Type *Ty, const llvm::Twine &Name, unsigned AddressSpace) { |
| // TODO: Replace the twine arg with stringref to get rid of the conversion |
| // logic. However This is taken from current implementation in clang as is. |
| // Since this method is used in many places exclusively for OMP internal use |
| // we will keep it as is for temporarily until we move all users to the |
| // builder and then, if possible, fix it everywhere in one go. |
| SmallString<256> Buffer; |
| llvm::raw_svector_ostream Out(Buffer); |
| Out << Name; |
| StringRef RuntimeName = Out.str(); |
| auto &Elem = *InternalVars.try_emplace(RuntimeName, nullptr).first; |
| if (Elem.second) { |
| assert(Elem.second->getType()->getPointerElementType() == Ty && |
| "OMP internal variable has different type than requested"); |
| } else { |
| // TODO: investigate the appropriate linkage type used for the global |
| // variable for possibly changing that to internal or private, or maybe |
| // create different versions of the function for different OMP internal |
| // variables. |
| Elem.second = new llvm::GlobalVariable( |
| M, Ty, /*IsConstant*/ false, llvm::GlobalValue::CommonLinkage, |
| llvm::Constant::getNullValue(Ty), Elem.first(), |
| /*InsertBefore=*/nullptr, llvm::GlobalValue::NotThreadLocal, |
| AddressSpace); |
| } |
| |
| return Elem.second; |
| } |
| |
| Value *OpenMPIRBuilder::getOMPCriticalRegionLock(StringRef CriticalName) { |
| std::string Prefix = Twine("gomp_critical_user_", CriticalName).str(); |
| std::string Name = getNameWithSeparators({Prefix, "var"}, ".", "."); |
| return getOrCreateOMPInternalVariable(KmpCriticalNameTy, Name); |
| } |
| |
| // Create all simple and struct types exposed by the runtime and remember |
| // the llvm::PointerTypes of them for easy access later. |
| void OpenMPIRBuilder::initializeTypes(Module &M) { |
| LLVMContext &Ctx = M.getContext(); |
| StructType *T; |
| #define OMP_TYPE(VarName, InitValue) VarName = InitValue; |
| #define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize) \ |
| VarName##Ty = ArrayType::get(ElemTy, ArraySize); \ |
| VarName##PtrTy = PointerType::getUnqual(VarName##Ty); |
| #define OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, ...) \ |
| VarName = FunctionType::get(ReturnType, {__VA_ARGS__}, IsVarArg); \ |
| VarName##Ptr = PointerType::getUnqual(VarName); |
| #define OMP_STRUCT_TYPE(VarName, StructName, ...) \ |
| T = M.getTypeByName(StructName); \ |
| if (!T) \ |
| T = StructType::create(Ctx, {__VA_ARGS__}, StructName); \ |
| VarName = T; \ |
| VarName##Ptr = PointerType::getUnqual(T); |
| #include "llvm/Frontend/OpenMP/OMPKinds.def" |
| } |
| |
| void OpenMPIRBuilder::OutlineInfo::collectBlocks( |
| SmallPtrSetImpl<BasicBlock *> &BlockSet, |
| SmallVectorImpl<BasicBlock *> &BlockVector) { |
| SmallVector<BasicBlock *, 32> Worklist; |
| BlockSet.insert(EntryBB); |
| BlockSet.insert(ExitBB); |
| |
| Worklist.push_back(EntryBB); |
| while (!Worklist.empty()) { |
| BasicBlock *BB = Worklist.pop_back_val(); |
| BlockVector.push_back(BB); |
| for (BasicBlock *SuccBB : successors(BB)) |
| if (BlockSet.insert(SuccBB).second) |
| Worklist.push_back(SuccBB); |
| } |
| } |
