| //===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===// |
| // |
| // The LLVM Compiler Infrastructure |
| // |
| // This file is distributed under the University of Illinois Open Source |
| // License. See LICENSE.TXT for details. |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "CGLoopInfo.h" |
| #include "clang/AST/ASTContext.h" |
| #include "clang/AST/Attr.h" |
| #include "clang/Sema/LoopHint.h" |
| #include "llvm/IR/BasicBlock.h" |
| #include "llvm/IR/Constants.h" |
| #include "llvm/IR/InstrTypes.h" |
| #include "llvm/IR/Instructions.h" |
| #include "llvm/IR/Metadata.h" |
| using namespace clang::CodeGen; |
| using namespace llvm; |
| |
| static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs, |
| llvm::DebugLoc Location) { |
| |
| if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 && |
| Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 && |
| Attrs.VectorizeEnable == LoopAttributes::Unspecified && |
| Attrs.UnrollEnable == LoopAttributes::Unspecified && |
| Attrs.DistributeEnable == LoopAttributes::Unspecified && |
| !Location) |
| return nullptr; |
| |
| SmallVector<Metadata *, 4> Args; |
| // Reserve operand 0 for loop id self reference. |
| auto TempNode = MDNode::getTemporary(Ctx, None); |
| Args.push_back(TempNode.get()); |
| |
| // If we have a valid debug location for the loop, add it. |
| if (Location) |
| Args.push_back(Location.getAsMDNode()); |
| |
| // Setting vectorize.width |
| if (Attrs.VectorizeWidth > 0) { |
| Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.width"), |
| ConstantAsMetadata::get(ConstantInt::get( |
| Type::getInt32Ty(Ctx), Attrs.VectorizeWidth))}; |
| Args.push_back(MDNode::get(Ctx, Vals)); |
| } |
| |
| // Setting interleave.count |
| if (Attrs.InterleaveCount > 0) { |
| Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.interleave.count"), |
| ConstantAsMetadata::get(ConstantInt::get( |
| Type::getInt32Ty(Ctx), Attrs.InterleaveCount))}; |
| Args.push_back(MDNode::get(Ctx, Vals)); |
| } |
| |
| // Setting interleave.count |
| if (Attrs.UnrollCount > 0) { |
| Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"), |
| ConstantAsMetadata::get(ConstantInt::get( |
| Type::getInt32Ty(Ctx), Attrs.UnrollCount))}; |
| Args.push_back(MDNode::get(Ctx, Vals)); |
| } |
| |
| // Setting vectorize.enable |
| if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) { |
| Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.enable"), |
| ConstantAsMetadata::get(ConstantInt::get( |
| Type::getInt1Ty(Ctx), (Attrs.VectorizeEnable == |
| LoopAttributes::Enable)))}; |
| Args.push_back(MDNode::get(Ctx, Vals)); |
| } |
| |
| // Setting unroll.full or unroll.disable |
| if (Attrs.UnrollEnable != LoopAttributes::Unspecified) { |
| std::string Name; |
| if (Attrs.UnrollEnable == LoopAttributes::Enable) |
| Name = "llvm.loop.unroll.enable"; |
| else if (Attrs.UnrollEnable == LoopAttributes::Full) |
| Name = "llvm.loop.unroll.full"; |
| else |
| Name = "llvm.loop.unroll.disable"; |
| Metadata *Vals[] = {MDString::get(Ctx, Name)}; |
| Args.push_back(MDNode::get(Ctx, Vals)); |
| } |
| |
| if (Attrs.DistributeEnable != LoopAttributes::Unspecified) { |
| Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"), |
| ConstantAsMetadata::get(ConstantInt::get( |
| Type::getInt1Ty(Ctx), (Attrs.DistributeEnable == |
| LoopAttributes::Enable)))}; |
| Args.push_back(MDNode::get(Ctx, Vals)); |
| } |
| |
| // Set the first operand to itself. |
| MDNode *LoopID = MDNode::get(Ctx, Args); |
| LoopID->replaceOperandWith(0, LoopID); |
| return LoopID; |
| } |
| |
| LoopAttributes::LoopAttributes(bool IsParallel) |
| : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified), |
| UnrollEnable(LoopAttributes::Unspecified), VectorizeWidth(0), |
| InterleaveCount(0), UnrollCount(0), |
| DistributeEnable(LoopAttributes::Unspecified) {} |
| |
| void LoopAttributes::clear() { |
| IsParallel = false; |
| VectorizeWidth = 0; |
| InterleaveCount = 0; |
| UnrollCount = 0; |
| VectorizeEnable = LoopAttributes::Unspecified; |
| UnrollEnable = LoopAttributes::Unspecified; |
| } |
| |
| LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs, |
| llvm::DebugLoc Location) |
| : LoopID(nullptr), Header(Header), Attrs(Attrs) { |
| LoopID = createMetadata(Header->getContext(), Attrs, Location); |
| } |
| |
| void LoopInfoStack::push(BasicBlock *Header, llvm::DebugLoc Location) { |
| Active.push_back(LoopInfo(Header, StagedAttrs, Location)); |
| // Clear the attributes so nested loops do not inherit them. |
| StagedAttrs.clear(); |
| } |
| |
| void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx, |
| ArrayRef<const clang::Attr *> Attrs, |
| llvm::DebugLoc Location) { |
| |
| // Identify loop hint attributes from Attrs. |
| for (const auto *Attr : Attrs) { |
| const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr); |
| const OpenCLUnrollHintAttr *OpenCLHint = |
| dyn_cast<OpenCLUnrollHintAttr>(Attr); |
| |
| // Skip non loop hint attributes |
| if (!LH && !OpenCLHint) { |
| continue; |
| } |
| |
| LoopHintAttr::OptionType Option = LoopHintAttr::Unroll; |
| LoopHintAttr::LoopHintState State = LoopHintAttr::Disable; |
| unsigned ValueInt = 1; |
| // Translate opencl_unroll_hint attribute argument to |
| // equivalent LoopHintAttr enums. |
| // OpenCL v2.0 s6.11.5: |
| // 0 - full unroll (no argument). |
| // 1 - disable unroll. |
| // other positive integer n - unroll by n. |
| if (OpenCLHint) { |
| ValueInt = OpenCLHint->getUnrollHint(); |
| if (ValueInt == 0) { |
| State = LoopHintAttr::Full; |
| } else if (ValueInt != 1) { |
| Option = LoopHintAttr::UnrollCount; |
| State = LoopHintAttr::Numeric; |
| } |
| } else if (LH) { |
| auto *ValueExpr = LH->getValue(); |
| if (ValueExpr) { |
| llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx); |
| ValueInt = ValueAPS.getSExtValue(); |
| } |
| |
| Option = LH->getOption(); |
| State = LH->getState(); |
| } |
| switch (State) { |
| case LoopHintAttr::Disable: |
| switch (Option) { |
| case LoopHintAttr::Vectorize: |
| // Disable vectorization by specifying a width of 1. |
| setVectorizeWidth(1); |
| break; |
| case LoopHintAttr::Interleave: |
| // Disable interleaving by speciyfing a count of 1. |
| setInterleaveCount(1); |
| break; |
| case LoopHintAttr::Unroll: |
| setUnrollState(LoopAttributes::Disable); |
| break; |
| case LoopHintAttr::Distribute: |
| setDistributeState(false); |
| break; |
| case LoopHintAttr::UnrollCount: |
| case LoopHintAttr::VectorizeWidth: |
| case LoopHintAttr::InterleaveCount: |
| llvm_unreachable("Options cannot be disabled."); |
| break; |
| } |
| break; |
| case LoopHintAttr::Enable: |
| switch (Option) { |
| case LoopHintAttr::Vectorize: |
| case LoopHintAttr::Interleave: |
| setVectorizeEnable(true); |
| break; |
| case LoopHintAttr::Unroll: |
| setUnrollState(LoopAttributes::Enable); |
| break; |
| case LoopHintAttr::Distribute: |
| setDistributeState(true); |
| break; |
| case LoopHintAttr::UnrollCount: |
| case LoopHintAttr::VectorizeWidth: |
| case LoopHintAttr::InterleaveCount: |
| llvm_unreachable("Options cannot enabled."); |
| break; |
| } |
| break; |
| case LoopHintAttr::AssumeSafety: |
| switch (Option) { |
| case LoopHintAttr::Vectorize: |
| case LoopHintAttr::Interleave: |
| // Apply "llvm.mem.parallel_loop_access" metadata to load/stores. |
| setParallel(true); |
| setVectorizeEnable(true); |
| break; |
| case LoopHintAttr::Unroll: |
| case LoopHintAttr::UnrollCount: |
| case LoopHintAttr::VectorizeWidth: |
| case LoopHintAttr::InterleaveCount: |
| case LoopHintAttr::Distribute: |
| llvm_unreachable("Options cannot be used to assume mem safety."); |
| break; |
| } |
| break; |
| case LoopHintAttr::Full: |
| switch (Option) { |
| case LoopHintAttr::Unroll: |
| setUnrollState(LoopAttributes::Full); |
| break; |
| case LoopHintAttr::Vectorize: |
| case LoopHintAttr::Interleave: |
| case LoopHintAttr::UnrollCount: |
| case LoopHintAttr::VectorizeWidth: |
| case LoopHintAttr::InterleaveCount: |
| case LoopHintAttr::Distribute: |
| llvm_unreachable("Options cannot be used with 'full' hint."); |
| break; |
| } |
| break; |
| case LoopHintAttr::Numeric: |
| switch (Option) { |
| case LoopHintAttr::VectorizeWidth: |
| setVectorizeWidth(ValueInt); |
| break; |
| case LoopHintAttr::InterleaveCount: |
| setInterleaveCount(ValueInt); |
| break; |
| case LoopHintAttr::UnrollCount: |
| setUnrollCount(ValueInt); |
| break; |
| case LoopHintAttr::Unroll: |
| case LoopHintAttr::Vectorize: |
| case LoopHintAttr::Interleave: |
| case LoopHintAttr::Distribute: |
| llvm_unreachable("Options cannot be assigned a value."); |
| break; |
| } |
| break; |
| } |
| } |
| |
| /// Stage the attributes. |
| push(Header, Location); |
| } |
| |
| void LoopInfoStack::pop() { |
| assert(!Active.empty() && "No active loops to pop"); |
| Active.pop_back(); |
| } |
| |
| void LoopInfoStack::InsertHelper(Instruction *I) const { |
| if (!hasInfo()) |
| return; |
| |
| const LoopInfo &L = getInfo(); |
| if (!L.getLoopID()) |
| return; |
| |
| if (TerminatorInst *TI = dyn_cast<TerminatorInst>(I)) { |
| for (unsigned i = 0, ie = TI->getNumSuccessors(); i < ie; ++i) |
| if (TI->getSuccessor(i) == L.getHeader()) { |
| TI->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID()); |
| break; |
| } |
| return; |
| } |
| |
| if (L.getAttributes().IsParallel && I->mayReadOrWriteMemory()) |
| I->setMetadata("llvm.mem.parallel_loop_access", L.getLoopID()); |
| } |