| //===--- CGExpr.cpp - Emit LLVM Code from Expressions ---------------------===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| // |
| // This contains code to emit Expr nodes as LLVM code. |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "CGCXXABI.h" |
| #include "CGCall.h" |
| #include "CGCleanup.h" |
| #include "CGDebugInfo.h" |
| #include "CGObjCRuntime.h" |
| #include "CGOpenMPRuntime.h" |
| #include "CGRecordLayout.h" |
| #include "CodeGenFunction.h" |
| #include "CodeGenModule.h" |
| #include "ConstantEmitter.h" |
| #include "TargetInfo.h" |
| #include "clang/AST/ASTContext.h" |
| #include "clang/AST/Attr.h" |
| #include "clang/AST/DeclObjC.h" |
| #include "clang/AST/NSAPI.h" |
| #include "clang/Basic/Builtins.h" |
| #include "clang/Basic/CodeGenOptions.h" |
| #include "clang/Basic/SourceManager.h" |
| #include "llvm/ADT/Hashing.h" |
| #include "llvm/ADT/StringExtras.h" |
| #include "llvm/IR/DataLayout.h" |
| #include "llvm/IR/Intrinsics.h" |
| #include "llvm/IR/LLVMContext.h" |
| #include "llvm/IR/MDBuilder.h" |
| #include "llvm/Support/ConvertUTF.h" |
| #include "llvm/Support/MathExtras.h" |
| #include "llvm/Support/Path.h" |
| #include "llvm/Transforms/Utils/SanitizerStats.h" |
| |
| #include <string> |
| |
| using namespace clang; |
| using namespace CodeGen; |
| |
| //===--------------------------------------------------------------------===// |
| // Miscellaneous Helper Methods |
| //===--------------------------------------------------------------------===// |
| |
| llvm::Value *CodeGenFunction::EmitCastToVoidPtr(llvm::Value *value) { |
| unsigned addressSpace = |
| cast<llvm::PointerType>(value->getType())->getAddressSpace(); |
| |
| llvm::PointerType *destType = Int8PtrTy; |
| if (addressSpace) |
| destType = llvm::Type::getInt8PtrTy(getLLVMContext(), addressSpace); |
| |
| if (value->getType() == destType) return value; |
| return Builder.CreateBitCast(value, destType); |
| } |
| |
| /// CreateTempAlloca - This creates a alloca and inserts it into the entry |
| /// block. |
| Address CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty, |
| CharUnits Align, |
| const Twine &Name, |
| llvm::Value *ArraySize) { |
| auto Alloca = CreateTempAlloca(Ty, Name, ArraySize); |
| Alloca->setAlignment(Align.getAsAlign()); |
| return Address(Alloca, Align); |
| } |
| |
| /// CreateTempAlloca - This creates a alloca and inserts it into the entry |
| /// block. The alloca is casted to default address space if necessary. |
| Address CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align, |
| const Twine &Name, |
| llvm::Value *ArraySize, |
| Address *AllocaAddr) { |
| auto Alloca = CreateTempAllocaWithoutCast(Ty, Align, Name, ArraySize); |
| if (AllocaAddr) |
| *AllocaAddr = Alloca; |
| llvm::Value *V = Alloca.getPointer(); |
| // Alloca always returns a pointer in alloca address space, which may |
| // be different from the type defined by the language. For example, |
| // in C++ the auto variables are in the default address space. Therefore |
| // cast alloca to the default address space when necessary. |
| if (getASTAllocaAddressSpace() != LangAS::Default) { |
| auto DestAddrSpace = getContext().getTargetAddressSpace(LangAS::Default); |
| llvm::IRBuilderBase::InsertPointGuard IPG(Builder); |
| // When ArraySize is nullptr, alloca is inserted at AllocaInsertPt, |
| // otherwise alloca is inserted at the current insertion point of the |
| // builder. |
| if (!ArraySize) |
| Builder.SetInsertPoint(AllocaInsertPt); |
| V = getTargetHooks().performAddrSpaceCast( |
| *this, V, getASTAllocaAddressSpace(), LangAS::Default, |
| Ty->getPointerTo(DestAddrSpace), /*non-null*/ true); |
| } |
| |
| return Address(V, Align); |
| } |
| |
| /// CreateTempAlloca - This creates an alloca and inserts it into the entry |
| /// block if \p ArraySize is nullptr, otherwise inserts it at the current |
| /// insertion point of the builder. |
| llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, |
| const Twine &Name, |
| llvm::Value *ArraySize) { |
| if (ArraySize) |
| return Builder.CreateAlloca(Ty, ArraySize, Name); |
| return new llvm::AllocaInst(Ty, CGM.getDataLayout().getAllocaAddrSpace(), |
| ArraySize, Name, AllocaInsertPt); |
| } |
| |
| /// CreateDefaultAlignTempAlloca - This creates an alloca with the |
| /// default alignment of the corresponding LLVM type, which is *not* |
| /// guaranteed to be related in any way to the expected alignment of |
| /// an AST type that might have been lowered to Ty. |
| Address CodeGenFunction::CreateDefaultAlignTempAlloca(llvm::Type *Ty, |
| const Twine &Name) { |
| CharUnits Align = |
| CharUnits::fromQuantity(CGM.getDataLayout().getABITypeAlignment(Ty)); |
| return CreateTempAlloca(Ty, Align, Name); |
| } |
| |
| void CodeGenFunction::InitTempAlloca(Address Var, llvm::Value *Init) { |
| auto *Alloca = Var.getPointer(); |
| assert(isa<llvm::AllocaInst>(Alloca) || |
| (isa<llvm::AddrSpaceCastInst>(Alloca) && |
| isa<llvm::AllocaInst>( |
| cast<llvm::AddrSpaceCastInst>(Alloca)->getPointerOperand()))); |
| |
| auto *Store = new llvm::StoreInst(Init, Alloca, /*volatile*/ false, |
| Var.getAlignment().getAsAlign()); |
| llvm::BasicBlock *Block = AllocaInsertPt->getParent(); |
| Block->getInstList().insertAfter(AllocaInsertPt->getIterator(), Store); |
| } |
| |
| Address CodeGenFunction::CreateIRTemp(QualType Ty, const Twine &Name) { |
| CharUnits Align = getContext().getTypeAlignInChars(Ty); |
| return CreateTempAlloca(ConvertType(Ty), Align, Name); |
| } |
| |
| Address CodeGenFunction::CreateMemTemp(QualType Ty, const Twine &Name, |
| Address *Alloca) { |
| // FIXME: Should we prefer the preferred type alignment here? |
| return CreateMemTemp(Ty, getContext().getTypeAlignInChars(Ty), Name, Alloca); |
| } |
| |
| Address CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align, |
| const Twine &Name, Address *Alloca) { |
| Address Result = CreateTempAlloca(ConvertTypeForMem(Ty), Align, Name, |
| /*ArraySize=*/nullptr, Alloca); |
| |
| if (Ty->isConstantMatrixType()) { |
| auto *ArrayTy = cast<llvm::ArrayType>(Result.getType()->getElementType()); |
| auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(), |
| ArrayTy->getNumElements()); |
| |
| Result = Address( |
| Builder.CreateBitCast(Result.getPointer(), VectorTy->getPointerTo()), |
| Result.getAlignment()); |
| } |
| return Result; |
| } |
| |
| Address CodeGenFunction::CreateMemTempWithoutCast(QualType Ty, CharUnits Align, |
| const Twine &Name) { |
| return CreateTempAllocaWithoutCast(ConvertTypeForMem(Ty), Align, Name); |
| } |
| |
| Address CodeGenFunction::CreateMemTempWithoutCast(QualType Ty, |
| const Twine &Name) { |
| return CreateMemTempWithoutCast(Ty, getContext().getTypeAlignInChars(Ty), |
| Name); |
| } |
| |
| /// EvaluateExprAsBool - Perform the usual unary conversions on the specified |
| /// expression and compare the result against zero, returning an Int1Ty value. |
| llvm::Value *CodeGenFunction::EvaluateExprAsBool(const Expr *E) { |
| PGO.setCurrentStmt(E); |
| if (const MemberPointerType *MPT = E->getType()->getAs<MemberPointerType>()) { |
| llvm::Value *MemPtr = EmitScalarExpr(E); |
| return CGM.getCXXABI().EmitMemberPointerIsNotNull(*this, MemPtr, MPT); |
| } |
| |
| QualType BoolTy = getContext().BoolTy; |
| SourceLocation Loc = E->getExprLoc(); |
| if (!E->getType()->isAnyComplexType()) |
| return EmitScalarConversion(EmitScalarExpr(E), E->getType(), BoolTy, Loc); |
| |
| return EmitComplexToScalarConversion(EmitComplexExpr(E), E->getType(), BoolTy, |
| Loc); |
| } |
| |
| /// EmitIgnoredExpr - Emit code to compute the specified expression, |
| /// ignoring the result. |
| void CodeGenFunction::EmitIgnoredExpr(const Expr *E) { |
| if (E->isRValue()) |
| return (void) EmitAnyExpr(E, AggValueSlot::ignored(), true); |
| |
| // Just emit it as an l-value and drop the result. |
| EmitLValue(E); |
| } |
| |
| /// EmitAnyExpr - Emit code to compute the specified expression which |
| /// can have any type. The result is returned as an RValue struct. |
| /// If this is an aggregate expression, AggSlot indicates where the |
| /// result should be returned. |
| RValue CodeGenFunction::EmitAnyExpr(const Expr *E, |
| AggValueSlot aggSlot, |
| bool ignoreResult) { |
| switch (getEvaluationKind(E->getType())) { |
| case TEK_Scalar: |
| return RValue::get(EmitScalarExpr(E, ignoreResult)); |
| case TEK_Complex: |
| return RValue::getComplex(EmitComplexExpr(E, ignoreResult, ignoreResult)); |
| case TEK_Aggregate: |
| if (!ignoreResult && aggSlot.isIgnored()) |
| aggSlot = CreateAggTemp(E->getType(), "agg-temp"); |
| EmitAggExpr(E, aggSlot); |
| return aggSlot.asRValue(); |
| } |
| llvm_unreachable("bad evaluation kind"); |
| } |
| |
| /// EmitAnyExprToTemp - Similar to EmitAnyExpr(), however, the result will |
| /// always be accessible even if no aggregate location is provided. |
| RValue CodeGenFunction::EmitAnyExprToTemp(const Expr *E) { |
| AggValueSlot AggSlot = AggValueSlot::ignored(); |
| |
| if (hasAggregateEvaluationKind(E->getType())) |
| AggSlot = CreateAggTemp(E->getType(), "agg.tmp"); |
| return EmitAnyExpr(E, AggSlot); |
| } |
| |
| /// EmitAnyExprToMem - Evaluate an expression into a given memory |
| /// location. |
| void CodeGenFunction::EmitAnyExprToMem(const Expr *E, |
| Address Location, |
| Qualifiers Quals, |
| bool IsInit) { |
| // FIXME: This function should take an LValue as an argument. |
| switch (getEvaluationKind(E->getType())) { |
| case TEK_Complex: |
| EmitComplexExprIntoLValue(E, MakeAddrLValue(Location, E->getType()), |
| /*isInit*/ false); |
| return; |
| |
| case TEK_Aggregate: { |
| EmitAggExpr(E, AggValueSlot::forAddr(Location, Quals, |
| AggValueSlot::IsDestructed_t(IsInit), |
| AggValueSlot::DoesNotNeedGCBarriers, |
| AggValueSlot::IsAliased_t(!IsInit), |
| AggValueSlot::MayOverlap)); |
| return; |
| } |
| |
| case TEK_Scalar: { |
| RValue RV = RValue::get(EmitScalarExpr(E, /*Ignore*/ false)); |
| LValue LV = MakeAddrLValue(Location, E->getType()); |
| EmitStoreThroughLValue(RV, LV); |
| return; |
| } |
| } |
| llvm_unreachable("bad evaluation kind"); |
| } |
| |
| static void |
| pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M, |
| const Expr *E, Address ReferenceTemporary) { |
| // Objective-C++ ARC: |
| // If we are binding a reference to a temporary that has ownership, we |
| // need to perform retain/release operations on the temporary. |
| // |
| // FIXME: This should be looking at E, not M. |
| if (auto Lifetime = M->getType().getObjCLifetime()) { |
| switch (Lifetime) { |
| case Qualifiers::OCL_None: |
| case Qualifiers::OCL_ExplicitNone: |
| // Carry on to normal cleanup handling. |
| break; |
| |
| case Qualifiers::OCL_Autoreleasing: |
| // Nothing to do; cleaned up by an autorelease pool. |
| return; |
| |
| case Qualifiers::OCL_Strong: |
| case Qualifiers::OCL_Weak: |
| switch (StorageDuration Duration = M->getStorageDuration()) { |
| case SD_Static: |
| // Note: we intentionally do not register a cleanup to release |
| // the object on program termination. |
| return; |
| |
| case SD_Thread: |
| // FIXME: We should probably register a cleanup in this case. |
| return; |
| |
| case SD_Automatic: |
| case SD_FullExpression: |
| CodeGenFunction::Destroyer *Destroy; |
| CleanupKind CleanupKind; |
| if (Lifetime == Qualifiers::OCL_Strong) { |
| const ValueDecl *VD = M->getExtendingDecl(); |
| bool Precise = |
| VD && isa<VarDecl>(VD) && VD->hasAttr<ObjCPreciseLifetimeAttr>(); |
| CleanupKind = CGF.getARCCleanupKind(); |
| Destroy = Precise ? &CodeGenFunction::destroyARCStrongPrecise |
| : &CodeGenFunction::destroyARCStrongImprecise; |
| } else { |
| // __weak objects always get EH cleanups; otherwise, exceptions |
| // could cause really nasty crashes instead of mere leaks. |
| CleanupKind = NormalAndEHCleanup; |
| Destroy = &CodeGenFunction::destroyARCWeak; |
| } |
| if (Duration == SD_FullExpression) |
| CGF.pushDestroy(CleanupKind, ReferenceTemporary, |
| M->getType(), *Destroy, |
| CleanupKind & EHCleanup); |
| else |
| CGF.pushLifetimeExtendedDestroy(CleanupKind, ReferenceTemporary, |
| M->getType(), |
| *Destroy, CleanupKind & EHCleanup); |
| return; |
| |
| case SD_Dynamic: |
| llvm_unreachable("temporary cannot have dynamic storage duration"); |
| } |
| llvm_unreachable("unknown storage duration"); |
| } |
| } |
| |
| CXXDestructorDecl *ReferenceTemporaryDtor = nullptr; |
| if (const RecordType *RT = |
| E->getType()->getBaseElementTypeUnsafe()->getAs<RecordType>()) { |
| // Get the destructor for the reference temporary. |
| auto *ClassDecl = cast<CXXRecordDecl>(RT->getDecl()); |
| if (!ClassDecl->hasTrivialDestructor()) |
| ReferenceTemporaryDtor = ClassDecl->getDestructor(); |
| } |
| |
| if (!ReferenceTemporaryDtor) |
| return; |
| |
| // Call the destructor for the temporary. |
| switch (M->getStorageDuration()) { |
| case SD_Static: |
| case SD_Thread: { |
| llvm::FunctionCallee CleanupFn; |
| llvm::Constant *CleanupArg; |
| if (E->getType()->isArrayType()) { |
| CleanupFn = CodeGenFunction(CGF.CGM).generateDestroyHelper( |
| ReferenceTemporary, E->getType(), |
| CodeGenFunction::destroyCXXObject, CGF.getLangOpts().Exceptions, |
| dyn_cast_or_null<VarDecl>(M->getExtendingDecl())); |
| CleanupArg = llvm::Constant::getNullValue(CGF.Int8PtrTy); |
| } else { |
| CleanupFn = CGF.CGM.getAddrAndTypeOfCXXStructor( |
| GlobalDecl(ReferenceTemporaryDtor, Dtor_Complete)); |
| CleanupArg = cast<llvm::Constant>(ReferenceTemporary.getPointer()); |
| } |
| CGF.CGM.getCXXABI().registerGlobalDtor( |
| CGF, *cast<VarDecl>(M->getExtendingDecl()), CleanupFn, CleanupArg); |
| break; |
| } |
| |
| case SD_FullExpression: |
| CGF.pushDestroy(NormalAndEHCleanup, ReferenceTemporary, E->getType(), |
| CodeGenFunction::destroyCXXObject, |
| CGF.getLangOpts().Exceptions); |
| break; |
| |
| case SD_Automatic: |
| CGF.pushLifetimeExtendedDestroy(NormalAndEHCleanup, |
| ReferenceTemporary, E->getType(), |
| CodeGenFunction::destroyCXXObject, |
| CGF.getLangOpts().Exceptions); |
| break; |
| |
| case SD_Dynamic: |
| llvm_unreachable("temporary cannot have dynamic storage duration"); |
| } |
| } |
| |
| static Address createReferenceTemporary(CodeGenFunction &CGF, |
| const MaterializeTemporaryExpr *M, |
| const Expr *Inner, |
| Address *Alloca = nullptr) { |
| auto &TCG = CGF.getTargetHooks(); |
| switch (M->getStorageDuration()) { |
| case SD_FullExpression: |
| case SD_Automatic: { |
| // If we have a constant temporary array or record try to promote it into a |
| // constant global under the same rules a normal constant would've been |
| // promoted. This is easier on the optimizer and generally emits fewer |
| // instructions. |
| QualType Ty = Inner->getType(); |
| if (CGF.CGM.getCodeGenOpts().MergeAllConstants && |
| (Ty->isArrayType() || Ty->isRecordType()) && |
| CGF.CGM.isTypeConstant(Ty, true)) |
| if (auto Init = ConstantEmitter(CGF).tryEmitAbstract(Inner, Ty)) { |
| if (auto AddrSpace = CGF.getTarget().getConstantAddressSpace()) { |
| auto AS = AddrSpace.getValue(); |
| auto *GV = new llvm::GlobalVariable( |
| CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true, |
| llvm::GlobalValue::PrivateLinkage, Init, ".ref.tmp", nullptr, |
| llvm::GlobalValue::NotThreadLocal, |
| CGF.getContext().getTargetAddressSpace(AS)); |
| CharUnits alignment = CGF.getContext().getTypeAlignInChars(Ty); |
| GV->setAlignment(alignment.getAsAlign()); |
| llvm::Constant *C = GV; |
| if (AS != LangAS::Default) |
| C = TCG.performAddrSpaceCast( |
| CGF.CGM, GV, AS, LangAS::Default, |
| GV->getValueType()->getPointerTo( |
| CGF.getContext().getTargetAddressSpace(LangAS::Default))); |
| // FIXME: Should we put the new global into a COMDAT? |
| return Address(C, alignment); |
| } |
| } |
| return CGF.CreateMemTemp(Ty, "ref.tmp", Alloca); |
| } |
| case SD_Thread: |
| case SD_Static: |
| return CGF.CGM.GetAddrOfGlobalTemporary(M, Inner); |
| |
| case SD_Dynamic: |
| llvm_unreachable("temporary can't have dynamic storage duration"); |
| } |
| llvm_unreachable("unknown storage duration"); |
| } |
| |
| /// Helper method to check if the underlying ABI is AAPCS |
| static bool isAAPCS(const TargetInfo &TargetInfo) { |
| return TargetInfo.getABI().startswith("aapcs"); |
| } |
| |
| LValue CodeGenFunction:: |
| EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) { |
| const Expr *E = M->getSubExpr(); |
| |
| assert((!M->getExtendingDecl() || !isa<VarDecl>(M->getExtendingDecl()) || |
| !cast<VarDecl>(M->getExtendingDecl())->isARCPseudoStrong()) && |
| "Reference should never be pseudo-strong!"); |
| |
| // FIXME: ideally this would use EmitAnyExprToMem, however, we cannot do so |
| // as that will cause the lifetime adjustment to be lost for ARC |
| auto ownership = M->getType().getObjCLifetime(); |
| if (ownership != Qualifiers::OCL_None && |
| ownership != Qualifiers::OCL_ExplicitNone) { |
| Address Object = createReferenceTemporary(*this, M, E); |
| if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object.getPointer())) { |
| Object = Address(llvm::ConstantExpr::getBitCast(Var, |
| ConvertTypeForMem(E->getType()) |
| ->getPointerTo(Object.getAddressSpace())), |
| Object.getAlignment()); |
| |
| // createReferenceTemporary will promote the temporary to a global with a |
| // constant initializer if it can. It can only do this to a value of |
| // ARC-manageable type if the value is global and therefore "immune" to |
| // ref-counting operations. Therefore we have no need to emit either a |
| // dynamic initialization or a cleanup and we can just return the address |
| // of the temporary. |
| if (Var->hasInitializer()) |
| return MakeAddrLValue(Object, M->getType(), AlignmentSource::Decl); |
| |
| Var->setInitializer(CGM.EmitNullConstant(E->getType())); |
| } |
| LValue RefTempDst = MakeAddrLValue(Object, M->getType(), |
| AlignmentSource::Decl); |
| |
| switch (getEvaluationKind(E->getType())) { |
| default: llvm_unreachable("expected scalar or aggregate expression"); |
| case TEK_Scalar: |
| EmitScalarInit(E, M->getExtendingDecl(), RefTempDst, false); |
| break; |
| case TEK_Aggregate: { |
| EmitAggExpr(E, AggValueSlot::forAddr(Object, |
| E->getType().getQualifiers(), |
| AggValueSlot::IsDestructed, |
| AggValueSlot::DoesNotNeedGCBarriers, |
| AggValueSlot::IsNotAliased, |
| AggValueSlot::DoesNotOverlap)); |
| break; |
| } |
| } |
| |
| pushTemporaryCleanup(*this, M, E, Object); |
| return RefTempDst; |
| } |
| |
| SmallVector<const Expr *, 2> CommaLHSs; |
| SmallVector<SubobjectAdjustment, 2> Adjustments; |
| E = E->skipRValueSubobjectAdjustments(CommaLHSs, Adjustments); |
| |
| for (const auto &Ignored : CommaLHSs) |
| EmitIgnoredExpr(Ignored); |
| |
| if (const auto *opaque = dyn_cast<OpaqueValueExpr>(E)) { |
| if (opaque->getType()->isRecordType()) { |
| assert(Adjustments.empty()); |
| return EmitOpaqueValueLValue(opaque); |
| } |
| } |
| |
| // Create and initialize the reference temporary. |
| Address Alloca = Address::invalid(); |
| Address Object = createReferenceTemporary(*this, M, E, &Alloca); |
| if (auto *Var = dyn_cast<llvm::GlobalVariable>( |
| Object.getPointer()->stripPointerCasts())) { |
| Object = Address(llvm::ConstantExpr::getBitCast( |
| cast<llvm::Constant>(Object.getPointer()), |
| ConvertTypeForMem(E->getType())->getPointerTo()), |
| Object.getAlignment()); |
| // If the temporary is a global and has a constant initializer or is a |
| // constant temporary that we promoted to a global, we may have already |
| // initialized it. |
| if (!Var->hasInitializer()) { |
| Var->setInitializer(CGM.EmitNullConstant(E->getType())); |
| EmitAnyExprToMem(E, Object, Qualifiers(), /*IsInit*/true); |
| } |
| } else { |
| switch (M->getStorageDuration()) { |
| case SD_Automatic: |
| if (auto *Size = EmitLifetimeStart( |
| CGM.getDataLayout().getTypeAllocSize(Alloca.getElementType()), |
| Alloca.getPointer())) { |
| pushCleanupAfterFullExpr<CallLifetimeEnd>(NormalEHLifetimeMarker, |
| Alloca, Size); |
| } |
| break; |
| |
| case SD_FullExpression: { |
| if (!ShouldEmitLifetimeMarkers) |
| break; |
| |
| // Avoid creating a conditional cleanup just to hold an llvm.lifetime.end |
| // marker. Instead, start the lifetime of a conditional temporary earlier |
| // so that it's unconditional. Don't do this with sanitizers which need |
| // more precise lifetime marks. |
| ConditionalEvaluation *OldConditional = nullptr; |
| CGBuilderTy::InsertPoint OldIP; |
| if (isInConditionalBranch() && !E->getType().isDestructedType() && |
| !SanOpts.has(SanitizerKind::HWAddress) && |
| !SanOpts.has(SanitizerKind::Memory) && |
| !CGM.getCodeGenOpts().SanitizeAddressUseAfterScope) { |
| OldConditional = OutermostConditional; |
| OutermostConditional = nullptr; |
| |
| OldIP = Builder.saveIP(); |
| llvm::BasicBlock *Block = OldConditional->getStartingBlock(); |
| Builder.restoreIP(CGBuilderTy::InsertPoint( |
| Block, llvm::BasicBlock::iterator(Block->back()))); |
| } |
| |
| if (auto *Size = EmitLifetimeStart( |
| CGM.getDataLayout().getTypeAllocSize(Alloca.getElementType()), |
| Alloca.getPointer())) { |
| pushFullExprCleanup<CallLifetimeEnd>(NormalEHLifetimeMarker, Alloca, |
| Size); |
| } |
| |
| if (OldConditional) { |
| OutermostConditional = OldConditional; |
| Builder.restoreIP(OldIP); |
| } |
| break; |
| } |
| |
| default: |
| break; |
| } |
| EmitAnyExprToMem(E, Object, Qualifiers(), /*IsInit*/true); |
| } |
| pushTemporaryCleanup(*this, M, E, Object); |
| |
| // Perform derived-to-base casts and/or field accesses, to get from the |
| // temporary object we created (and, potentially, for which we extended |
| // the lifetime) to the subobject we're binding the reference to. |
| for (unsigned I = Adjustments.size(); I != 0; --I) { |
| SubobjectAdjustment &Adjustment = Adjustments[I-1]; |
| switch (Adjustment.Kind) { |
| case SubobjectAdjustment::DerivedToBaseAdjustment: |
| Object = |
| GetAddressOfBaseClass(Object, Adjustment.DerivedToBase.DerivedClass, |
| Adjustment.DerivedToBase.BasePath->path_begin(), |
| Adjustment.DerivedToBase.BasePath->path_end(), |
| /*NullCheckValue=*/ false, E->getExprLoc()); |
| break; |
| |
| case SubobjectAdjustment::FieldAdjustment: { |
| LValue LV = MakeAddrLValue(Object, E->getType(), AlignmentSource::Decl); |
| LV = EmitLValueForField(LV, Adjustment.Field); |
| assert(LV.isSimple() && |
| "materialized temporary field is not a simple lvalue"); |
| Object = LV.getAddress(*this); |
| break; |
| } |
| |
| case SubobjectAdjustment::MemberPointerAdjustment: { |
| llvm::Value *Ptr = EmitScalarExpr(Adjustment.Ptr.RHS); |
| Object = EmitCXXMemberDataPointerAddress(E, Object, Ptr, |
| Adjustment.Ptr.MPT); |
| break; |
| } |
| } |
| } |
| |
| return MakeAddrLValue(Object, M->getType(), AlignmentSource::Decl); |
| } |
| |
| RValue |
| CodeGenFunction::EmitReferenceBindingToExpr(const Expr *E) { |
| // Emit the expression as an lvalue. |
| LValue LV = EmitLValue(E); |
| assert(LV.isSimple()); |
| llvm::Value *Value = LV.getPointer(*this); |
| |
| if (sanitizePerformTypeCheck() && !E->getType()->isFunctionType()) { |
| // C++11 [dcl.ref]p5 (as amended by core issue 453): |
| // If a glvalue to which a reference is directly bound designates neither |
| // an existing object or function of an appropriate type nor a region of |
| // storage of suitable size and alignment to contain an object of the |
| // reference's type, the behavior is undefined. |
| QualType Ty = E->getType(); |
| EmitTypeCheck(TCK_ReferenceBinding, E->getExprLoc(), Value, Ty); |
| } |
| |
| return RValue::get(Value); |
| } |
| |
| |
| /// getAccessedFieldNo - Given an encoded value and a result number, return the |
| /// input field number being accessed. |
| unsigned CodeGenFunction::getAccessedFieldNo(unsigned Idx, |
| const llvm::Constant *Elts) { |
| return cast<llvm::ConstantInt>(Elts->getAggregateElement(Idx)) |
| ->getZExtValue(); |
| } |
| |
| /// Emit the hash_16_bytes function from include/llvm/ADT/Hashing.h. |
| static llvm::Value *emitHash16Bytes(CGBuilderTy &Builder, llvm::Value *Low, |
| llvm::Value *High) { |
| llvm::Value *KMul = Builder.getInt64(0x9ddfea08eb382d69ULL); |
| llvm::Value *K47 = Builder.getInt64(47); |
| llvm::Value *A0 = Builder.CreateMul(Builder.CreateXor(Low, High), KMul); |
| llvm::Value *A1 = Builder.CreateXor(Builder.CreateLShr(A0, K47), A0); |
| llvm::Value *B0 = Builder.CreateMul(Builder.CreateXor(High, A1), KMul); |
| llvm::Value *B1 = Builder.CreateXor(Builder.CreateLShr(B0, K47), B0); |
| return Builder.CreateMul(B1, KMul); |
| } |
| |
| bool CodeGenFunction::isNullPointerAllowed(TypeCheckKind TCK) { |
| return TCK == TCK_DowncastPointer || TCK == TCK_Upcast || |
| TCK == TCK_UpcastToVirtualBase || TCK == TCK_DynamicOperation; |
| } |
| |
| bool CodeGenFunction::isVptrCheckRequired(TypeCheckKind TCK, QualType Ty) { |
| CXXRecordDecl *RD = Ty->getAsCXXRecordDecl(); |
| return (RD && RD->hasDefinition() && RD->isDynamicClass()) && |
| (TCK == TCK_MemberAccess || TCK == TCK_MemberCall || |
| TCK == TCK_DowncastPointer || TCK == TCK_DowncastReference || |
| TCK == TCK_UpcastToVirtualBase || TCK == TCK_DynamicOperation); |
| } |
| |
| bool CodeGenFunction::sanitizePerformTypeCheck() const { |
| return SanOpts.has(SanitizerKind::Null) | |
| SanOpts.has(SanitizerKind::Alignment) | |
| SanOpts.has(SanitizerKind::ObjectSize) | |
| SanOpts.has(SanitizerKind::Vptr); |
| } |
| |
| void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, |
| llvm::Value *Ptr, QualType Ty, |
| CharUnits Alignment, |
| SanitizerSet SkippedChecks, |
| llvm::Value *ArraySize) { |
| if (!sanitizePerformTypeCheck()) |
| return; |
| |
| // Don't check pointers outside the default address space. The null check |
| // isn't correct, the object-size check isn't supported by LLVM, and we can't |
| // communicate the addresses to the runtime handler for the vptr check. |
| if (Ptr->getType()->getPointerAddressSpace()) |
| return; |
| |
| // Don't check pointers to volatile data. The behavior here is implementation- |
| // defined. |
| if (Ty.isVolatileQualified()) |
| return; |
| |
| SanitizerScope SanScope(this); |
| |
| SmallVector<std::pair<llvm::Value *, SanitizerMask>, 3> Checks; |
| llvm::BasicBlock *Done = nullptr; |
| |
| // Quickly determine whether we have a pointer to an alloca. It's possible |
| // to skip null checks, and some alignment checks, for these pointers. This |
| // can reduce compile-time significantly. |
| auto PtrToAlloca = dyn_cast<llvm::AllocaInst>(Ptr->stripPointerCasts()); |
| |
| llvm::Value *True = llvm::ConstantInt::getTrue(getLLVMContext()); |
| llvm::Value *IsNonNull = nullptr; |
| bool IsGuaranteedNonNull = |
| SkippedChecks.has(SanitizerKind::Null) || PtrToAlloca; |
| bool AllowNullPointers = isNullPointerAllowed(TCK); |
| if ((SanOpts.has(SanitizerKind::Null) || AllowNullPointers) && |
| !IsGuaranteedNonNull) { |
| // The glvalue must not be an empty glvalue. |
| IsNonNull = Builder.CreateIsNotNull(Ptr); |
| |
| // The IR builder can constant-fold the null check if the pointer points to |
| // a constant. |
| IsGuaranteedNonNull = IsNonNull == True; |
| |
| // Skip the null check if the pointer is known to be non-null. |
| if (!IsGuaranteedNonNull) { |
| if (AllowNullPointers) { |
| // When performing pointer casts, it's OK if the value is null. |
| // Skip the remaining checks in that case. |
| Done = createBasicBlock("null"); |
| llvm::BasicBlock *Rest = createBasicBlock("not.null"); |
| Builder.CreateCondBr(IsNonNull, Rest, Done); |
| EmitBlock(Rest); |
| } else { |
| Checks.push_back(std::make_pair(IsNonNull, SanitizerKind::Null)); |
| } |
| } |
| } |
| |
| if (SanOpts.has(SanitizerKind::ObjectSize) && |
| !SkippedChecks.has(SanitizerKind::ObjectSize) && |
| !Ty->isIncompleteType()) { |
| uint64_t TySize = CGM.getMinimumObjectSize(Ty).getQuantity(); |
| llvm::Value *Size = llvm::ConstantInt::get(IntPtrTy, TySize); |
| if (ArraySize) |
| Size = Builder.CreateMul(Size, ArraySize); |
| |
| // Degenerate case: new X[0] does not need an objectsize check. |
| llvm::Constant *ConstantSize = dyn_cast<llvm::Constant>(Size); |
| if (!ConstantSize || !ConstantSize->isNullValue()) { |
| // The glvalue must refer to a large enough storage region. |
| // FIXME: If Address Sanitizer is enabled, insert dynamic instrumentation |
| // to check this. |
| // FIXME: Get object address space |
| llvm::Type *Tys[2] = { IntPtrTy, Int8PtrTy }; |
| llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::objectsize, Tys); |
| llvm::Value *Min = Builder.getFalse(); |
| llvm::Value *NullIsUnknown = Builder.getFalse(); |
| llvm::Value *Dynamic = Builder.getFalse(); |
| llvm::Value *CastAddr = Builder.CreateBitCast(Ptr, Int8PtrTy); |
| llvm::Value *LargeEnough = Builder.CreateICmpUGE( |
| Builder.CreateCall(F, {CastAddr, Min, NullIsUnknown, Dynamic}), Size); |
| Checks.push_back(std::make_pair(LargeEnough, SanitizerKind::ObjectSize)); |
| } |
| } |
| |
| uint64_t AlignVal = 0; |
| llvm::Value *PtrAsInt = nullptr; |
| |
| if (SanOpts.has(SanitizerKind::Alignment) && |
| !SkippedChecks.has(SanitizerKind::Alignment)) { |
| AlignVal = Alignment.getQuantity(); |
| if (!Ty->isIncompleteType() && !AlignVal) |
| AlignVal = CGM.getNaturalTypeAlignment(Ty, nullptr, nullptr, |
| /*ForPointeeType=*/true) |
| .getQuantity(); |
| |
| // The glvalue must be suitably aligned. |
| if (AlignVal > 1 && |
| (!PtrToAlloca || PtrToAlloca->getAlignment() < AlignVal)) { |
| PtrAsInt = Builder.CreatePtrToInt(Ptr, IntPtrTy); |
| llvm::Value *Align = Builder.CreateAnd( |
| PtrAsInt, llvm::ConstantInt::get(IntPtrTy, AlignVal - 1)); |
| llvm::Value *Aligned = |
| Builder.CreateICmpEQ(Align, llvm::ConstantInt::get(IntPtrTy, 0)); |
| if (Aligned != True) |
| Checks.push_back(std::make_pair(Aligned, SanitizerKind::Alignment)); |
| } |
| } |
| |
| if (Checks.size() > 0) { |
| // Make sure we're not losing information. Alignment needs to be a power of |
| // 2 |
| assert(!AlignVal || (uint64_t)1 << llvm::Log2_64(AlignVal) == AlignVal); |
| llvm::Constant *StaticData[] = { |
| EmitCheckSourceLocation(Loc), EmitCheckTypeDescriptor(Ty), |
| llvm::ConstantInt::get(Int8Ty, AlignVal ? llvm::Log2_64(AlignVal) : 1), |
| llvm::ConstantInt::get(Int8Ty, TCK)}; |
| EmitCheck(Checks, SanitizerHandler::TypeMismatch, StaticData, |
| PtrAsInt ? PtrAsInt : Ptr); |
| } |
| |
| // If possible, check that the vptr indicates that there is a subobject of |
| // type Ty at offset zero within this object. |
| // |
| // C++11 [basic.life]p5,6: |
| // [For storage which does not refer to an object within its lifetime] |
| // The program has undefined behavior if: |
| // -- the [pointer or glvalue] is used to access a non-static data member |
| // or call a non-static member function |
| if (SanOpts.has(SanitizerKind::Vptr) && |
| !SkippedChecks.has(SanitizerKind::Vptr) && isVptrCheckRequired(TCK, Ty)) { |
| // Ensure that the pointer is non-null before loading it. If there is no |
| // compile-time guarantee, reuse the run-time null check or emit a new one. |
| if (!IsGuaranteedNonNull) { |
| if (!IsNonNull) |
| IsNonNull = Builder.CreateIsNotNull(Ptr); |
| if (!Done) |
| Done = createBasicBlock("vptr.null"); |
| llvm::BasicBlock *VptrNotNull = createBasicBlock("vptr.not.null"); |
| Builder.CreateCondBr(IsNonNull, VptrNotNull, Done); |
| EmitBlock(VptrNotNull); |
| } |
| |
| // Compute a hash of the mangled name of the type. |
| // |
| // FIXME: This is not guaranteed to be deterministic! Move to a |
| // fingerprinting mechanism once LLVM provides one. For the time |
| // being the implementation happens to be deterministic. |
| SmallString<64> MangledName; |
| llvm::raw_svector_ostream Out(MangledName); |
| CGM.getCXXABI().getMangleContext().mangleCXXRTTI(Ty.getUnqualifiedType(), |
| Out); |
| |
| // Blacklist based on the mangled type. |
| if (!CGM.getContext().getSanitizerBlacklist().isBlacklistedType( |
| SanitizerKind::Vptr, Out.str())) { |
| llvm::hash_code TypeHash = hash_value(Out.str()); |
| |
| // Load the vptr, and compute hash_16_bytes(TypeHash, vptr). |
| llvm::Value *Low = llvm::ConstantInt::get(Int64Ty, TypeHash); |
| llvm::Type *VPtrTy = llvm::PointerType::get(IntPtrTy, 0); |
| Address VPtrAddr(Builder.CreateBitCast(Ptr, VPtrTy), getPointerAlign()); |
| llvm::Value *VPtrVal = Builder.CreateLoad(VPtrAddr); |
| llvm::Value *High = Builder.CreateZExt(VPtrVal, Int64Ty); |
| |
| llvm::Value *Hash = emitHash16Bytes(Builder, Low, High); |
| Hash = Builder.CreateTrunc(Hash, IntPtrTy); |
| |
| // Look the hash up in our cache. |
| const int CacheSize = 128; |
| llvm::Type *HashTable = llvm::ArrayType::get(IntPtrTy, CacheSize); |
| llvm::Value *Cache = CGM.CreateRuntimeVariable(HashTable, |
| "__ubsan_vptr_type_cache"); |
| llvm::Value *Slot = Builder.CreateAnd(Hash, |
| llvm::ConstantInt::get(IntPtrTy, |
| CacheSize-1)); |
| llvm::Value *Indices[] = { Builder.getInt32(0), Slot }; |
| llvm::Value *CacheVal = |
| Builder.CreateAlignedLoad(Builder.CreateInBoundsGEP(Cache, Indices), |
| getPointerAlign()); |
| |
| // If the hash isn't in the cache, call a runtime handler to perform the |
| // hard work of checking whether the vptr is for an object of the right |
| // type. This will either fill in the cache and return, or produce a |
| // diagnostic. |
| llvm::Value *EqualHash = Builder.CreateICmpEQ(CacheVal, Hash); |
| llvm::Constant *StaticData[] = { |
| EmitCheckSourceLocation(Loc), |
| EmitCheckTypeDescriptor(Ty), |
| CGM.GetAddrOfRTTIDescriptor(Ty.getUnqualifiedType()), |
| llvm::ConstantInt::get(Int8Ty, TCK) |
| }; |
| llvm::Value *DynamicData[] = { Ptr, Hash }; |
| EmitCheck(std::make_pair(EqualHash, SanitizerKind::Vptr), |
| SanitizerHandler::DynamicTypeCacheMiss, StaticData, |
| DynamicData); |
| } |
| } |
| |
| if (Done) { |
| Builder.CreateBr(Done); |
| EmitBlock(Done); |
| } |
| } |
| |
| /// Determine whether this expression refers to a flexible array member in a |
| /// struct. We disable array bounds checks for such members. |
| static bool isFlexibleArrayMemberExpr(const Expr *E) { |
| // For compatibility with existing code, we treat arrays of length 0 or |
| // 1 as flexible array members. |
| // FIXME: This is inconsistent with the warning code in SemaChecking. Unify |
| // the two mechanisms. |
| const ArrayType *AT = E->getType()->castAsArrayTypeUnsafe(); |
| if (const auto *CAT = dyn_cast<ConstantArrayType>(AT)) { |
| // FIXME: Sema doesn't treat [1] as a flexible array member if the bound |
| // was produced by macro expansion. |
| if (CAT->getSize().ugt(1)) |
| return false; |
| } else if (!isa<IncompleteArrayType>(AT)) |
| return false; |
| |
| E = E->IgnoreParens(); |
| |
| // A flexible array member must be the last member in the class. |
| if (const auto *ME = dyn_cast<MemberExpr>(E)) { |
| // FIXME: If the base type of the member expr is not FD->getParent(), |
| // this should not be treated as a flexible array member access. |
| if (const auto *FD = dyn_cast<FieldDecl>(ME->getMemberDecl())) { |
| // FIXME: Sema doesn't treat a T[1] union member as a flexible array |
| // member, only a T[0] or T[] member gets that treatment. |
| if (FD->getParent()->isUnion()) |
| return true; |
| RecordDecl::field_iterator FI( |
| DeclContext::decl_iterator(const_cast<FieldDecl *>(FD))); |
| return ++FI == FD->getParent()->field_end(); |
| } |
| } else if (const auto *IRE = dyn_cast<ObjCIvarRefExpr>(E)) { |
| return IRE->getDecl()->getNextIvar() == nullptr; |
| } |
| |
| return false; |
| } |
| |
| llvm::Value *CodeGenFunction::LoadPassedObjectSize(const Expr *E, |
| QualType EltTy) { |
| ASTContext &C = getContext(); |
| uint64_t EltSize = C.getTypeSizeInChars(EltTy).getQuantity(); |
| if (!EltSize) |
| return nullptr; |
| |
| auto *ArrayDeclRef = dyn_cast<DeclRefExpr>(E->IgnoreParenImpCasts()); |
| if (!ArrayDeclRef) |
| return nullptr; |
| |
| auto *ParamDecl = dyn_cast<ParmVarDecl>(ArrayDeclRef->getDecl()); |
| if (!ParamDecl) |
| return nullptr; |
| |
| auto *POSAttr = ParamDecl->getAttr<PassObjectSizeAttr>(); |
| if (!POSAttr) |
| return nullptr; |
| |
| // Don't load the size if it's a lower bound. |
| int POSType = POSAttr->getType(); |
| if (POSType != 0 && POSType != 1) |
| return nullptr; |
| |
| // Find the implicit size parameter. |
| auto PassedSizeIt = SizeArguments.find(ParamDecl); |
| if (PassedSizeIt == SizeArguments.end()) |
| return nullptr; |
| |
| const ImplicitParamDecl *PassedSizeDecl = PassedSizeIt->second; |
| assert(LocalDeclMap.count(PassedSizeDecl) && "Passed size not loadable"); |
| Address AddrOfSize = LocalDeclMap.find(PassedSizeDecl)->second; |
| llvm::Value *SizeInBytes = EmitLoadOfScalar(AddrOfSize, /*Volatile=*/false, |
| C.getSizeType(), E->getExprLoc()); |
| llvm::Value *SizeOfElement = |
| llvm::ConstantInt::get(SizeInBytes->getType(), EltSize); |
| return Builder.CreateUDiv(SizeInBytes, SizeOfElement); |
| } |
| |
| /// If Base is known to point to the start of an array, return the length of |
| /// that array. Return 0 if the length cannot be determined. |
| static llvm::Value *getArrayIndexingBound( |
| CodeGenFunction &CGF, const Expr *Base, QualType &IndexedType) { |
| // For the vector indexing extension, the bound is the number of elements. |
| if (const VectorType *VT = Base->getType()->getAs<VectorType>()) { |
| IndexedType = Base->getType(); |
| return CGF.Builder.getInt32(VT->getNumElements()); |
| } |
| |
| Base = Base->IgnoreParens(); |
| |
| if (const auto *CE = dyn_cast<CastExpr>(Base)) { |
| if (CE->getCastKind() == CK_ArrayToPointerDecay && |
| !isFlexibleArrayMemberExpr(CE->getSubExpr())) { |
| IndexedType = CE->getSubExpr()->getType(); |
| const ArrayType *AT = IndexedType->castAsArrayTypeUnsafe(); |
| if (const auto *CAT = dyn_cast<ConstantArrayType>(AT)) |
| return CGF.Builder.getInt(CAT->getSize()); |
| else if (const auto *VAT = dyn_cast<VariableArrayType>(AT)) |
| return CGF.getVLASize(VAT).NumElts; |
| // Ignore pass_object_size here. It's not applicable on decayed pointers. |
| } |
| } |
| |
| QualType EltTy{Base->getType()->getPointeeOrArrayElementType(), 0}; |
| if (llvm::Value *POS = CGF.LoadPassedObjectSize(Base, EltTy)) { |
| IndexedType = Base->getType(); |
| return POS; |
| } |
| |
| return nullptr; |
| } |
| |
| void CodeGenFunction::EmitBoundsCheck(const Expr *E, const Expr *Base, |
| llvm::Value *Index, QualType IndexType, |
| bool Accessed) { |
| assert(SanOpts.has(SanitizerKind::ArrayBounds) && |
| "should not be called unless adding bounds checks"); |
| SanitizerScope SanScope(this); |
| |
| QualType IndexedType; |
| llvm::Value *Bound = getArrayIndexingBound(*this, Base, IndexedType); |
| if (!Bound) |
| return; |
| |
| bool IndexSigned = IndexType->isSignedIntegerOrEnumerationType(); |
| llvm::Value *IndexVal = Builder.CreateIntCast(Index, SizeTy, IndexSigned); |
| llvm::Value *BoundVal = Builder.CreateIntCast(Bound, SizeTy, false); |
| |
| llvm::Constant *StaticData[] = { |
| EmitCheckSourceLocation(E->getExprLoc()), |
| EmitCheckTypeDescriptor(IndexedType), |
| EmitCheckTypeDescriptor(IndexType) |
| }; |
| llvm::Value *Check = Accessed ? Builder.CreateICmpULT(IndexVal, BoundVal) |
| : Builder.CreateICmpULE(IndexVal, BoundVal); |
| EmitCheck(std::make_pair(Check, SanitizerKind::ArrayBounds), |
| SanitizerHandler::OutOfBounds, StaticData, Index); |
| } |
| |
| |
| CodeGenFunction::ComplexPairTy CodeGenFunction:: |
| EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV, |
| bool isInc, bool isPre) { |
| ComplexPairTy InVal = EmitLoadOfComplex(LV, E->getExprLoc()); |
| |
| llvm::Value *NextVal; |
| if (isa<llvm::IntegerType>(InVal.first->getType())) { |
| uint64_t AmountVal = isInc ? 1 : -1; |
| NextVal = llvm::ConstantInt::get(InVal.first->getType(), AmountVal, true); |
| |
| // Add the inc/dec to the real part. |
| NextVal = Builder.CreateAdd(InVal.first, NextVal, isInc ? "inc" : "dec"); |
| } else { |
| QualType ElemTy = E->getType()->castAs<ComplexType>()->getElementType(); |
| llvm::APFloat FVal(getContext().getFloatTypeSemantics(ElemTy), 1); |
| if (!isInc) |
| FVal.changeSign(); |
| NextVal = llvm::ConstantFP::get(getLLVMContext(), FVal); |
| |
| // Add the inc/dec to the real part. |
| NextVal = Builder.CreateFAdd(InVal.first, NextVal, isInc ? "inc" : "dec"); |
| } |
| |
| ComplexPairTy IncVal(NextVal, InVal.second); |
| |
| // Store the updated result through the lvalue. |
| EmitStoreOfComplex(IncVal, LV, /*init*/ false); |
| if (getLangOpts().OpenMP) |
| CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(*this, |
| E->getSubExpr()); |
| |
| // If this is a postinc, return the value read from memory, otherwise use the |
| // updated value. |
| return isPre ? IncVal : InVal; |
| } |
| |
| void CodeGenModule::EmitExplicitCastExprType(const ExplicitCastExpr *E, |
| CodeGenFunction *CGF) { |
| // Bind VLAs in the cast type. |
| if (CGF && E->getType()->isVariablyModifiedType()) |
| CGF->EmitVariablyModifiedType(E->getType()); |
| |
| if (CGDebugInfo *DI = getModuleDebugInfo()) |
| DI->EmitExplicitCastType(E->getType()); |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // LValue Expression Emission |
| //===----------------------------------------------------------------------===// |
| |
| /// EmitPointerWithAlignment - Given an expression of pointer type, try to |
| /// derive a more accurate bound on the alignment of the pointer. |
| Address CodeGenFunction::EmitPointerWithAlignment(const Expr *E, |
| LValueBaseInfo *BaseInfo, |
| TBAAAccessInfo *TBAAInfo) { |
| // We allow this with ObjC object pointers because of fragile ABIs. |
| assert(E->getType()->isPointerType() || |
| E->getType()->isObjCObjectPointerType()); |
| E = E->IgnoreParens(); |
| |
| // Casts: |
| if (const CastExpr *CE = dyn_cast<CastExpr>(E)) { |
| if (const auto *ECE = dyn_cast<ExplicitCastExpr>(CE)) |
| CGM.EmitExplicitCastExprType(ECE, this); |
| |
| switch (CE->getCastKind()) { |
| // Non-converting casts (but not C's implicit conversion from void*). |
| case CK_BitCast: |
| case CK_NoOp: |
| case CK_AddressSpaceConversion: |
| if (auto PtrTy = CE->getSubExpr()->getType()->getAs<PointerType>()) { |
| if (PtrTy->getPointeeType()->isVoidType()) |
| break; |
| |
| LValueBaseInfo InnerBaseInfo; |
| TBAAAccessInfo InnerTBAAInfo; |
| Address Addr = EmitPointerWithAlignment(CE->getSubExpr(), |
| &InnerBaseInfo, |
| &InnerTBAAInfo); |
| if (BaseInfo) *BaseInfo = InnerBaseInfo; |
| if (TBAAInfo) *TBAAInfo = InnerTBAAInfo; |
| |
| if (isa<ExplicitCastExpr>(CE)) { |
| LValueBaseInfo TargetTypeBaseInfo; |
| TBAAAccessInfo TargetTypeTBAAInfo; |
| CharUnits Align = CGM.getNaturalPointeeTypeAlignment( |
| E->getType(), &TargetTypeBaseInfo, &TargetTypeTBAAInfo); |
| if (TBAAInfo) |
| *TBAAInfo = CGM.mergeTBAAInfoForCast(*TBAAInfo, |
| TargetTypeTBAAInfo); |
| // If the source l-value is opaque, honor the alignment of the |
| // casted-to type. |
| if (InnerBaseInfo.getAlignmentSource() != AlignmentSource::Decl) { |
| if (BaseInfo) |
| BaseInfo->mergeForCast(TargetTypeBaseInfo); |
| Addr = Address(Addr.getPointer(), Align); |
| } |
| } |
| |
| if (SanOpts.has(SanitizerKind::CFIUnrelatedCast) && |
| CE->getCastKind() == CK_BitCast) { |
| if (auto PT = E->getType()->getAs<PointerType>()) |
| EmitVTablePtrCheckForCast(PT->getPointeeType(), Addr.getPointer(), |
| /*MayBeNull=*/true, |
| CodeGenFunction::CFITCK_UnrelatedCast, |
| CE->getBeginLoc()); |
| } |
| return CE->getCastKind() != CK_AddressSpaceConversion |
| ? Builder.CreateBitCast(Addr, ConvertType(E->getType())) |
| : Builder.CreateAddrSpaceCast(Addr, |
| ConvertType(E->getType())); |
| } |
| break; |
| |
| // Array-to-pointer decay. |
| case CK_ArrayToPointerDecay: |
| return EmitArrayToPointerDecay(CE->getSubExpr(), BaseInfo, TBAAInfo); |
| |
| // Derived-to-base conversions. |
| case CK_UncheckedDerivedToBase: |
| case CK_DerivedToBase: { |
| // TODO: Support accesses to members of base classes in TBAA. For now, we |
| // conservatively pretend that the complete object is of the base class |
| // type. |
| if (TBAAInfo) |
| *TBAAInfo = CGM.getTBAAAccessInfo(E->getType()); |
| Address Addr = EmitPointerWithAlignment(CE->getSubExpr(), BaseInfo); |
| auto Derived = CE->getSubExpr()->getType()->getPointeeCXXRecordDecl(); |
| return GetAddressOfBaseClass(Addr, Derived, |
| CE->path_begin(), CE->path_end(), |
| ShouldNullCheckClassCastValue(CE), |
| CE->getExprLoc()); |
| } |
| |
| // TODO: Is there any reason to treat base-to-derived conversions |
| // specially? |
| default: |
| break; |
| } |
| } |
| |
| // Unary &. |
| if (const UnaryOperator *UO = dyn_cast<UnaryOperator>(E)) { |
| if (UO->getOpcode() == UO_AddrOf) { |
| LValue LV = EmitLValue(UO->getSubExpr()); |
| if (BaseInfo) *BaseInfo = LV.getBaseInfo(); |
| if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo(); |
| return LV.getAddress(*this); |
| } |
| } |
| |
| // TODO: conditional operators, comma. |
| |
| // Otherwise, use the alignment of the type. |
| CharUnits Align = |
| CGM.getNaturalPointeeTypeAlignment(E->getType(), BaseInfo, TBAAInfo); |
| return Address(EmitScalarExpr(E), Align); |
| } |
| |
| RValue CodeGenFunction::GetUndefRValue(QualType Ty) { |
| if (Ty->isVoidType()) |
| return RValue::get(nullptr); |
| |
| switch (getEvaluationKind(Ty)) { |
| case TEK_Complex: { |
| llvm::Type *EltTy = |
| ConvertType(Ty->castAs<ComplexType>()->getElementType()); |
| llvm::Value *U = llvm::UndefValue::get(EltTy); |
| return RValue::getComplex(std::make_pair(U, U)); |
| } |
| |
| // If this is a use of an undefined aggregate type, the aggregate must have an |
| // identifiable address. Just because the contents of the value are undefined |
| // doesn't mean that the address can't be taken and compared. |
| case TEK_Aggregate: { |
| Address DestPtr = CreateMemTemp(Ty, "undef.agg.tmp"); |
| return RValue::getAggregate(DestPtr); |
| } |
| |
| case TEK_Scalar: |
| return RValue::get(llvm::UndefValue::get(ConvertType(Ty))); |
| } |
| llvm_unreachable("bad evaluation kind"); |
| } |
| |
| RValue CodeGenFunction::EmitUnsupportedRValue(const Expr *E, |
| const char *Name) { |
| ErrorUnsupported(E, Name); |
| return GetUndefRValue(E->getType()); |
| } |
| |
| LValue CodeGenFunction::EmitUnsupportedLValue(const Expr *E, |
| const char *Name) { |
| ErrorUnsupported(E, Name); |
| llvm::Type *Ty = llvm::PointerType::getUnqual(ConvertType(E->getType())); |
| return MakeAddrLValue(Address(llvm::UndefValue::get(Ty), CharUnits::One()), |
| E->getType()); |
| } |
| |
| bool CodeGenFunction::IsWrappedCXXThis(const Expr *Obj) { |
| const Expr *Base = Obj; |
| while (!isa<CXXThisExpr>(Base)) { |
| // The result of a dynamic_cast can be null. |
| if (isa<CXXDynamicCastExpr>(Base)) |
| return false; |
| |
| if (const auto *CE = dyn_cast<CastExpr>(Base)) { |
| Base = CE->getSubExpr(); |
| } else if (const auto *PE = dyn_cast<ParenExpr>(Base)) { |
| Base = PE->getSubExpr(); |
| } else if (const auto *UO = dyn_cast<UnaryOperator>(Base)) { |
| if (UO->getOpcode() == UO_Extension) |
| Base = UO->getSubExpr(); |
| else |
| return false; |
| } else { |
| return false; |
| } |
| } |
| return true; |
| } |
| |
| LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) { |
| LValue LV; |
| if (SanOpts.has(SanitizerKind::ArrayBounds) && isa<ArraySubscriptExpr>(E)) |
| LV = EmitArraySubscriptExpr(cast<ArraySubscriptExpr>(E), /*Accessed*/true); |
| else |
| LV = EmitLValue(E); |
| if (!isa<DeclRefExpr>(E) && !LV.isBitField() && LV.isSimple()) { |
| SanitizerSet SkippedChecks; |
| if (const auto *ME = dyn_cast<MemberExpr>(E)) { |
| bool IsBaseCXXThis = IsWrappedCXXThis(ME->getBase()); |
| if (IsBaseCXXThis) |
| SkippedChecks.set(SanitizerKind::Alignment, true); |
| if (IsBaseCXXThis || isa<DeclRefExpr>(ME->getBase())) |
| SkippedChecks.set(SanitizerKind::Null, true); |
| } |
| EmitTypeCheck(TCK, E->getExprLoc(), LV.getPointer(*this), E->getType(), |
| LV.getAlignment(), SkippedChecks); |
| } |
| return LV; |
| } |
| |
| /// EmitLValue - Emit code to compute a designator that specifies the location |
| /// of the expression. |
| /// |
| /// This can return one of two things: a simple address or a bitfield reference. |
| /// In either case, the LLVM Value* in the LValue structure is guaranteed to be |
| /// an LLVM pointer type. |
| /// |
| /// If this returns a bitfield reference, nothing about the pointee type of the |
| /// LLVM value is known: For example, it may not be a pointer to an integer. |
| /// |
| /// If this returns a normal address, and if the lvalue's C type is fixed size, |
| /// this method guarantees that the returned pointer type will point to an LLVM |
| /// type of the same size of the lvalue's type. If the lvalue has a variable |
| /// length type, this is not possible. |
| /// |
| LValue CodeGenFunction::EmitLValue(const Expr *E) { |
| ApplyDebugLocation DL(*this, E); |
| switch (E->getStmtClass()) { |
| default: return EmitUnsupportedLValue(E, "l-value expression"); |
| |
| case Expr::ObjCPropertyRefExprClass: |
| llvm_unreachable("cannot emit a property reference directly"); |
| |
| case Expr::ObjCSelectorExprClass: |
| return EmitObjCSelectorLValue(cast<ObjCSelectorExpr>(E)); |
| case Expr::ObjCIsaExprClass: |
| return EmitObjCIsaExpr(cast<ObjCIsaExpr>(E)); |
| case Expr::BinaryOperatorClass: |
| return EmitBinaryOperatorLValue(cast<BinaryOperator>(E)); |
| case Expr::CompoundAssignOperatorClass: { |
| QualType Ty = E->getType(); |
| if (const AtomicType *AT = Ty->getAs<AtomicType>()) |
| Ty = AT->getValueType(); |
| if (!Ty->isAnyComplexType()) |
| return EmitCompoundAssignmentLValue(cast<CompoundAssignOperator>(E)); |
| return EmitComplexCompoundAssignmentLValue(cast<CompoundAssignOperator>(E)); |
| } |
| case Expr::CallExprClass: |
| case Expr::CXXMemberCallExprClass: |
| case Expr::CXXOperatorCallExprClass: |
| case Expr::UserDefinedLiteralClass: |
| return EmitCallExprLValue(cast<CallExpr>(E)); |
| case Expr::CXXRewrittenBinaryOperatorClass: |
| return EmitLValue(cast<CXXRewrittenBinaryOperator>(E)->getSemanticForm()); |
| case Expr::VAArgExprClass: |
| return EmitVAArgExprLValue(cast<VAArgExpr>(E)); |
| case Expr::DeclRefExprClass: |
| return EmitDeclRefLValue(cast<DeclRefExpr>(E)); |
| case Expr::ConstantExprClass: { |
| const ConstantExpr *CE = cast<ConstantExpr>(E); |
| if (llvm::Value *Result = ConstantEmitter(*this).tryEmitConstantExpr(CE)) { |
| QualType RetType = cast<CallExpr>(CE->getSubExpr()->IgnoreImplicit()) |
| ->getCallReturnType(getContext()); |
| return MakeNaturalAlignAddrLValue(Result, RetType); |
| } |
| return EmitLValue(cast<ConstantExpr>(E)->getSubExpr()); |
| } |
| case Expr::ParenExprClass: |
| return EmitLValue(cast<ParenExpr>(E)->getSubExpr()); |
| case Expr::GenericSelectionExprClass: |
| return EmitLValue(cast<GenericSelectionExpr>(E)->getResultExpr()); |
| case Expr::PredefinedExprClass: |
| return EmitPredefinedLValue(cast<PredefinedExpr>(E)); |
| case Expr::StringLiteralClass: |
| return EmitStringLiteralLValue(cast<StringLiteral>(E)); |
| case Expr::ObjCEncodeExprClass: |
| return EmitObjCEncodeExprLValue(cast<ObjCEncodeExpr>(E)); |
| case Expr::PseudoObjectExprClass: |
| return EmitPseudoObjectLValue(cast<PseudoObjectExpr>(E)); |
| case Expr::InitListExprClass: |
| return EmitInitListLValue(cast<InitListExpr>(E)); |
| case Expr::CXXTemporaryObjectExprClass: |
| case Expr::CXXConstructExprClass: |
| return EmitCXXConstructLValue(cast<CXXConstructExpr>(E)); |
| case Expr::CXXBindTemporaryExprClass: |
| return EmitCXXBindTemporaryLValue(cast<CXXBindTemporaryExpr>(E)); |
| case Expr::CXXUuidofExprClass: |
| return EmitCXXUuidofLValue(cast<CXXUuidofExpr>(E)); |
| case Expr::LambdaExprClass: |
| return EmitAggExprToLValue(E); |
| |
| case Expr::ExprWithCleanupsClass: { |
| const auto *cleanups = cast<ExprWithCleanups>(E); |
| RunCleanupsScope Scope(*this); |
| LValue LV = EmitLValue(cleanups->getSubExpr()); |
| if (LV.isSimple()) { |
| // Defend against branches out of gnu statement expressions surrounded by |
| // cleanups. |
| llvm::Value *V = LV.getPointer(*this); |
| Scope.ForceCleanup({&V}); |
| return LValue::MakeAddr(Address(V, LV.getAlignment()), LV.getType(), |
| getContext(), LV.getBaseInfo(), LV.getTBAAInfo()); |
| } |
| // FIXME: Is it possible to create an ExprWithCleanups that produces a |
| // bitfield lvalue or some other non-simple lvalue? |
| return LV; |
| } |
| |
| case Expr::CXXDefaultArgExprClass: { |
| auto *DAE = cast<CXXDefaultArgExpr>(E); |
| CXXDefaultArgExprScope Scope(*this, DAE); |
| return EmitLValue(DAE->getExpr()); |
| } |
| case Expr::CXXDefaultInitExprClass: { |
| auto *DIE = cast<CXXDefaultInitExpr>(E); |
| CXXDefaultInitExprScope Scope(*this, DIE); |
| return EmitLValue(DIE->getExpr()); |
| } |
| case Expr::CXXTypeidExprClass: |
| return EmitCXXTypeidLValue(cast<CXXTypeidExpr>(E)); |
| |
| case Expr::ObjCMessageExprClass: |
| return EmitObjCMessageExprLValue(cast<ObjCMessageExpr>(E)); |
| case Expr::ObjCIvarRefExprClass: |
| return EmitObjCIvarRefLValue(cast<ObjCIvarRefExpr>(E)); |
| case Expr::StmtExprClass: |
| return EmitStmtExprLValue(cast<StmtExpr>(E)); |
| case Expr::UnaryOperatorClass: |
| return EmitUnaryOpLValue(cast<UnaryOperator>(E)); |
| case Expr::ArraySubscriptExprClass: |
| return EmitArraySubscriptExpr(cast<ArraySubscriptExpr>(E)); |
| case Expr::MatrixSubscriptExprClass: |
| return EmitMatrixSubscriptExpr(cast<MatrixSubscriptExpr>(E)); |
| case Expr::OMPArraySectionExprClass: |
| return EmitOMPArraySectionExpr(cast<OMPArraySectionExpr>(E)); |
| case Expr::ExtVectorElementExprClass: |
| return EmitExtVectorElementExpr(cast<ExtVectorElementExpr>(E)); |
| case Expr::MemberExprClass: |
| return EmitMemberExpr(cast<MemberExpr>(E)); |
| case Expr::CompoundLiteralExprClass: |
| return EmitCompoundLiteralLValue(cast<CompoundLiteralExpr>(E)); |
| case Expr::ConditionalOperatorClass: |
| return EmitConditionalOperatorLValue(cast<ConditionalOperator>(E)); |
| case Expr::BinaryConditionalOperatorClass: |
| return EmitConditionalOperatorLValue(cast<BinaryConditionalOperator>(E)); |
| case Expr::ChooseExprClass: |
| return EmitLValue(cast<ChooseExpr>(E)->getChosenSubExpr()); |
| case Expr::OpaqueValueExprClass: |
| return EmitOpaqueValueLValue(cast<OpaqueValueExpr>(E)); |
| case Expr::SubstNonTypeTemplateParmExprClass: |
| return EmitLValue(cast<SubstNonTypeTemplateParmExpr>(E)->getReplacement()); |
| case Expr::ImplicitCastExprClass: |
| case Expr::CStyleCastExprClass: |
| case Expr::CXXFunctionalCastExprClass: |
| case Expr::CXXStaticCastExprClass: |
| case Expr::CXXDynamicCastExprClass: |
| case Expr::CXXReinterpretCastExprClass: |
| case Expr::CXXConstCastExprClass: |
| case Expr::CXXAddrspaceCastExprClass: |
| case Expr::ObjCBridgedCastExprClass: |
| return EmitCastLValue(cast<CastExpr>(E)); |
| |
| case Expr::MaterializeTemporaryExprClass: |
| return EmitMaterializeTemporaryExpr(cast<MaterializeTemporaryExpr>(E)); |
| |
| case Expr::CoawaitExprClass: |
| return EmitCoawaitLValue(cast<CoawaitExpr>(E)); |
| case Expr::CoyieldExprClass: |
| return EmitCoyieldLValue(cast<CoyieldExpr>(E)); |
| } |
| } |
| |
| /// Given an object of the given canonical type, can we safely copy a |
| /// value out of it based on its initializer? |
| static bool isConstantEmittableObjectType(QualType type) { |
| assert(type.isCanonical()); |
| assert(!type->isReferenceType()); |
| |
| // Must be const-qualified but non-volatile. |
| Qualifiers qs = type.getLocalQualifiers(); |
| if (!qs.hasConst() || qs.hasVolatile()) return false; |
| |
| // Otherwise, all object types satisfy this except C++ classes with |
| // mutable subobjects or non-trivial copy/destroy behavior. |
| if (const auto *RT = dyn_cast<RecordType>(type)) |
| if (const auto *RD = dyn_cast<CXXRecordDecl>(RT->getDecl())) |
| if (RD->hasMutableFields() || !RD->isTrivial()) |
| return false; |
| |
| return true; |
| } |
| |
| /// Can we constant-emit a load of a reference to a variable of the |
| /// given type? This is different from predicates like |
| /// Decl::mightBeUsableInConstantExpressions because we do want it to apply |
| /// in situations that don't necessarily satisfy the language's rules |
| /// for this (e.g. C++'s ODR-use rules). For example, we want to able |
| /// to do this with const float variables even if those variables |
| /// aren't marked 'constexpr'. |
| enum ConstantEmissionKind { |
| CEK_None, |
| CEK_AsReferenceOnly, |
| CEK_AsValueOrReference, |
| CEK_AsValueOnly |
| }; |
| static ConstantEmissionKind checkVarTypeForConstantEmission(QualType type) { |
| type = type.getCanonicalType(); |
| if (const auto *ref = dyn_cast<ReferenceType>(type)) { |
| if (isConstantEmittableObjectType(ref->getPointeeType())) |
| return CEK_AsValueOrReference; |
| return CEK_AsReferenceOnly; |
| } |
| if (isConstantEmittableObjectType(type)) |
| return CEK_AsValueOnly; |
| return CEK_None; |
| } |
| |
| /// Try to emit a reference to the given value without producing it as |
| /// an l-value. This is just an optimization, but it avoids us needing |
| /// to emit global copies of variables if they're named without triggering |
| /// a formal use in a context where we can't emit a direct reference to them, |
| /// for instance if a block or lambda or a member of a local class uses a |
| /// const int variable or constexpr variable from an enclosing function. |
| CodeGenFunction::ConstantEmission |
| CodeGenFunction::tryEmitAsConstant(DeclRefExpr *refExpr) { |
| ValueDecl *value = refExpr->getDecl(); |
| |
| // The value needs to be an enum constant or a constant variable. |
| ConstantEmissionKind CEK; |
| if (isa<ParmVarDecl>(value)) { |
| CEK = CEK_None; |
| } else if (auto *var = dyn_cast<VarDecl>(value)) { |
| CEK = checkVarTypeForConstantEmission(var->getType()); |
| } else if (isa<EnumConstantDecl>(value)) { |
| CEK = CEK_AsValueOnly; |
| } else { |
| CEK = CEK_None; |
| } |
| if (CEK == CEK_None) return ConstantEmission(); |
| |
| Expr::EvalResult result; |
| bool resultIsReference; |
| QualType resultType; |
| |
| // It's best to evaluate all the way as an r-value if that's permitted. |
| if (CEK != CEK_AsReferenceOnly && |
| refExpr->EvaluateAsRValue(result, getContext())) { |
| resultIsReference = false; |
| resultType = refExpr->getType(); |
| |
| // Otherwise, try to evaluate as an l-value. |
| } else if (CEK != CEK_AsValueOnly && |
| refExpr->EvaluateAsLValue(result, getContext())) { |
| resultIsReference = true; |
| resultType = value->getType(); |
| |
| // Failure. |
| } else { |
| return ConstantEmission(); |
| } |
| |
| // In any case, if the initializer has side-effects, abandon ship. |
| if (result.HasSideEffects) |
| return ConstantEmission(); |
| |
| // Emit as a constant. |
| auto C = ConstantEmitter(*this).emitAbstract(refExpr->getLocation(), |
| result.Val, resultType); |
| |
| // Make sure we emit a debug reference to the global variable. |
| // This should probably fire even for |
| if (isa<VarDecl>(value)) { |
| if (!getContext().DeclMustBeEmitted(cast<VarDecl>(value))) |
| EmitDeclRefExprDbgValue(refExpr, result.Val); |
| } else { |
| assert(isa<EnumConstantDecl>(value)); |
| EmitDeclRefExprDbgValue(refExpr, result.Val); |
| } |
| |
| // If we emitted a reference constant, we need to dereference that. |
| if (resultIsReference) |
| return ConstantEmission::forReference(C); |
| |
| return ConstantEmission::forValue(C); |
| } |
| |
| static DeclRefExpr *tryToConvertMemberExprToDeclRefExpr(CodeGenFunction &CGF, |
| const MemberExpr *ME) { |
| if (auto *VD = dyn_cast<VarDecl>(ME->getMemberDecl())) { |
| // Try to emit static variable member expressions as DREs. |
| return DeclRefExpr::Create( |
| CGF.getContext(), NestedNameSpecifierLoc(), SourceLocation(), VD, |
| /*RefersToEnclosingVariableOrCapture=*/false, ME->getExprLoc(), |
| ME->getType(), ME->getValueKind(), nullptr, nullptr, ME->isNonOdrUse()); |
| } |
| return nullptr; |
| } |
| |
| CodeGenFunction::ConstantEmission |
| CodeGenFunction::tryEmitAsConstant(const MemberExpr *ME) { |
| if (DeclRefExpr *DRE = tryToConvertMemberExprToDeclRefExpr(*this, ME)) |
| return tryEmitAsConstant(DRE); |
| return ConstantEmission(); |
| } |
| |
| llvm::Value *CodeGenFunction::emitScalarConstant( |
| const CodeGenFunction::ConstantEmission &Constant, Expr *E) { |
| assert(Constant && "not a constant"); |
| if (Constant.isReference()) |
| return EmitLoadOfLValue(Constant.getReferenceLValue(*this, E), |
| E->getExprLoc()) |
| .getScalarVal(); |
| return Constant.getValue(); |
| } |
| |
| llvm::Value *CodeGenFunction::EmitLoadOfScalar(LValue lvalue, |
| SourceLocation Loc) { |
| return EmitLoadOfScalar(lvalue.getAddress(*this), lvalue.isVolatile(), |
| lvalue.getType(), Loc, lvalue.getBaseInfo(), |
| lvalue.getTBAAInfo(), lvalue.isNontemporal()); |
| } |
| |
| static bool hasBooleanRepresentation(QualType Ty) { |
| if (Ty->isBooleanType()) |
| return true; |
| |
| if (const EnumType *ET = Ty->getAs<EnumType>()) |
| return ET->getDecl()->getIntegerType()->isBooleanType(); |
| |
| if (const AtomicType *AT = Ty->getAs<AtomicType>()) |
| return hasBooleanRepresentation(AT->getValueType()); |
| |
| return false; |
| } |
| |
| static bool getRangeForType(CodeGenFunction &CGF, QualType Ty, |
| llvm::APInt &Min, llvm::APInt &End, |
| bool StrictEnums, bool IsBool) { |
| const EnumType *ET = Ty->getAs<EnumType>(); |
| bool IsRegularCPlusPlusEnum = CGF.getLangOpts().CPlusPlus && StrictEnums && |
| ET && !ET->getDecl()->isFixed(); |
| if (!IsBool && !IsRegularCPlusPlusEnum) |
| return false; |
| |
| if (IsBool) { |
| Min = llvm::APInt(CGF.getContext().getTypeSize(Ty), 0); |
| End = llvm::APInt(CGF.getContext().getTypeSize(Ty), 2); |
| } else { |
| const EnumDecl *ED = ET->getDecl(); |
| llvm::Type *LTy = CGF.ConvertTypeForMem(ED->getIntegerType()); |
| unsigned Bitwidth = LTy->getScalarSizeInBits(); |
| unsigned NumNegativeBits = ED->getNumNegativeBits(); |
| unsigned NumPositiveBits = ED->getNumPositiveBits(); |
| |
| if (NumNegativeBits) { |
| unsigned NumBits = std::max(NumNegativeBits, NumPositiveBits + 1); |
| assert(NumBits <= Bitwidth); |
| End = llvm::APInt(Bitwidth, 1) << (NumBits - 1); |
| Min = -End; |
| } else { |
| assert(NumPositiveBits <= Bitwidth); |
| End = llvm::APInt(Bitwidth, 1) << NumPositiveBits; |
| Min = llvm::APInt(Bitwidth, 0); |
| } |
| } |
| return true; |
| } |
| |
| llvm::MDNode *CodeGenFunction::getRangeForLoadFromType(QualType Ty) { |
| llvm::APInt Min, End; |
| if (!getRangeForType(*this, Ty, Min, End, CGM.getCodeGenOpts().StrictEnums, |
| hasBooleanRepresentation(Ty))) |
| return nullptr; |
| |
| llvm::MDBuilder MDHelper(getLLVMContext()); |
| return MDHelper.createRange(Min, End); |
| } |
| |
| bool CodeGenFunction::EmitScalarRangeCheck(llvm::Value *Value, QualType Ty, |
| SourceLocation Loc) { |
| bool HasBoolCheck = SanOpts.has(SanitizerKind::Bool); |
| bool HasEnumCheck = SanOpts.has(SanitizerKind::Enum); |
| if (!HasBoolCheck && !HasEnumCheck) |
| return false; |
| |
| bool IsBool = hasBooleanRepresentation(Ty) || |
| NSAPI(CGM.getContext()).isObjCBOOLType(Ty); |
| bool NeedsBoolCheck = HasBoolCheck && IsBool; |
| bool NeedsEnumCheck = HasEnumCheck && Ty->getAs<EnumType>(); |
| if (!NeedsBoolCheck && !NeedsEnumCheck) |
| return false; |
| |
| // Single-bit booleans don't need to be checked. Special-case this to avoid |
| // a bit width mismatch when handling bitfield values. This is handled by |
| // EmitFromMemory for the non-bitfield case. |
| if (IsBool && |
| cast<llvm::IntegerType>(Value->getType())->getBitWidth() == 1) |
| return false; |
| |
| llvm::APInt Min, End; |
| if (!getRangeForType(*this, Ty, Min, End, /*StrictEnums=*/true, IsBool)) |
| return true; |
| |
| auto &Ctx = getLLVMContext(); |
| SanitizerScope SanScope(this); |
| llvm::Value *Check; |
| --End; |
| if (!Min) { |
| Check = Builder.CreateICmpULE(Value, llvm::ConstantInt::get(Ctx, End)); |
| } else { |
| llvm::Value *Upper = |
| Builder.CreateICmpSLE(Value, llvm::ConstantInt::get(Ctx, End)); |
| llvm::Value *Lower = |
| Builder.CreateICmpSGE(Value, llvm::ConstantInt::get(Ctx, Min)); |
| Check = Builder.CreateAnd(Upper, Lower); |
| } |
| llvm::Constant *StaticArgs[] = {EmitCheckSourceLocation(Loc), |
| EmitCheckTypeDescriptor(Ty)}; |
| SanitizerMask Kind = |
| NeedsEnumCheck ? SanitizerKind::Enum : SanitizerKind::Bool; |
| EmitCheck(std::make_pair(Check, Kind), SanitizerHandler::LoadInvalidValue, |
| StaticArgs, EmitCheckValue(Value)); |
| return true; |
| } |
| |
| llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile, |
| QualType Ty, |
| SourceLocation Loc, |
| LValueBaseInfo BaseInfo, |
| TBAAAccessInfo TBAAInfo, |
| bool isNontemporal) { |
| if (!CGM.getCodeGenOpts().PreserveVec3Type) { |
| // For better performance, handle vector loads differently. |
| if (Ty->isVectorType()) { |
| const llvm::Type *EltTy = Addr.getElementType(); |
| |
| const auto *VTy = cast<llvm::FixedVectorType>(EltTy); |
| |
| // Handle vectors of size 3 like size 4 for better performance. |
| if (VTy->getNumElements() == 3) { |
| |
| // Bitcast to vec4 type. |
| auto *vec4Ty = llvm::FixedVectorType::get(VTy->getElementType(), 4); |
| Address Cast = Builder.CreateElementBitCast(Addr, vec4Ty, "castToVec4"); |
| // Now load value. |
| llvm::Value *V = Builder.CreateLoad(Cast, Volatile, "loadVec4"); |
| |
| // Shuffle vector to get vec3. |
| V = Builder.CreateShuffleVector(V, llvm::UndefValue::get(vec4Ty), |
| ArrayRef<int>{0, 1, 2}, "extractVec"); |
| return EmitFromMemory(V, Ty); |
| } |
| } |
| } |
| |
| // Atomic operations have to be done on integral types. |
| LValue AtomicLValue = |
| LValue::MakeAddr(Addr, Ty, getContext(), BaseInfo, TBAAInfo); |
| if (Ty->isAtomicType() || LValueIsSuitableForInlineAtomic(AtomicLValue)) { |
| return EmitAtomicLoad(AtomicLValue, Loc).getScalarVal(); |
| } |
| |
| llvm::LoadInst *Load = Builder.CreateLoad(Addr, Volatile); |
| if (isNontemporal) { |
| llvm::MDNode *Node = llvm::MDNode::get( |
| Load->getContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1))); |
| Load->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node); |
| } |
| |
| CGM.DecorateInstructionWithTBAA(Load, TBAAInfo); |
| |
| if (EmitScalarRangeCheck(Load, Ty, Loc)) { |
| // In order to prevent the optimizer from throwing away the check, don't |
| // attach range metadata to the load. |
| } else if (CGM.getCodeGenOpts().OptimizationLevel > 0) |
| if (llvm::MDNode *RangeInfo = getRangeForLoadFromType(Ty)) |
| Load->setMetadata(llvm::LLVMContext::MD_range, RangeInfo); |
| |
| return EmitFromMemory(Load, Ty); |
| } |
| |
| llvm::Value *CodeGenFunction::EmitToMemory(llvm::Value *Value, QualType Ty) { |
| // Bool has a different representation in memory than in registers. |
| if (hasBooleanRepresentation(Ty)) { |
| // This should really always be an i1, but sometimes it's already |
| // an i8, and it's awkward to track those cases down. |
| if (Value->getType()->isIntegerTy(1)) |
| return Builder.CreateZExt(Value, ConvertTypeForMem(Ty), "frombool"); |
| assert(Value->getType()->isIntegerTy(getContext().getTypeSize(Ty)) && |
| "wrong value rep of bool"); |
| } |
| |
| return Value; |
| } |
| |
| llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) { |
| // Bool has a different representation in memory than in registers. |
| if (hasBooleanRepresentation(Ty)) { |
| assert(Value->getType()->isIntegerTy(getContext().getTypeSize(Ty)) && |
| "wrong value rep of bool"); |
| return Builder.CreateTrunc(Value, Builder.getInt1Ty(), "tobool"); |
| } |
| |
| return Value; |
| } |
| |
| // Convert the pointer of \p Addr to a pointer to a vector (the value type of |
| // MatrixType), if it points to a array (the memory type of MatrixType). |
| static Address MaybeConvertMatrixAddress(Address Addr, CodeGenFunction &CGF, |
| bool IsVector = true) { |
| auto *ArrayTy = dyn_cast<llvm::ArrayType>( |
| cast<llvm::PointerType>(Addr.getPointer()->getType())->getElementType()); |
| if (ArrayTy && IsVector) { |
| auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(), |
| ArrayTy->getNumElements()); |
| |
| return Address(CGF.Builder.CreateElementBitCast(Addr, VectorTy)); |
| } |
| auto *VectorTy = dyn_cast<llvm::VectorType>( |
| cast<llvm::PointerType>(Addr.getPointer()->getType())->getElementType()); |
| if (VectorTy && !IsVector) { |
| auto *ArrayTy = llvm::ArrayType::get( |
| VectorTy->getElementType(), |
| cast<llvm::FixedVectorType>(VectorTy)->getNumElements()); |
| |
| return Address(CGF.Builder.CreateElementBitCast(Addr, ArrayTy)); |
| } |
| |
| return Addr; |
| } |
| |
| // Emit a store of a matrix LValue. This may require casting the original |
| // pointer to memory address (ArrayType) to a pointer to the value type |
| // (VectorType). |
| static void EmitStoreOfMatrixScalar(llvm::Value *value, LValue lvalue, |
| bool isInit, CodeGenFunction &CGF) { |
| Address Addr = MaybeConvertMatrixAddress(lvalue.getAddress(CGF), CGF, |
| value->getType()->isVectorTy()); |
| CGF.EmitStoreOfScalar(value, Addr, lvalue.isVolatile(), lvalue.getType(), |
| lvalue.getBaseInfo(), lvalue.getTBAAInfo(), isInit, |
| lvalue.isNontemporal()); |
| } |
| |
| void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, Address Addr, |
| bool Volatile, QualType Ty, |
| LValueBaseInfo BaseInfo, |
| TBAAAccessInfo TBAAInfo, |
| bool isInit, bool isNontemporal) { |
| if (!CGM.getCodeGenOpts().PreserveVec3Type) { |
| // Handle vectors differently to get better performance. |
| if (Ty->isVectorType()) { |
| llvm::Type *SrcTy = Value->getType(); |
| auto *VecTy = dyn_cast<llvm::VectorType>(SrcTy); |
| // Handle vec3 special. |
| if (VecTy && cast<llvm::FixedVectorType>(VecTy)->getNumElements() == 3) { |
| // Our source is a vec3, do a shuffle vector to make it a vec4. |
| Value = Builder.CreateShuffleVector(Value, llvm::UndefValue::get(VecTy), |
| ArrayRef<int>{0, 1, 2, -1}, |
| "extractVec"); |
| SrcTy = llvm::FixedVectorType::get(VecTy->getElementType(), 4); |
| } |
| if (Addr.getElementType() != SrcTy) { |
| Addr = Builder.CreateElementBitCast(Addr, SrcTy, "storetmp"); |
| } |
| } |
| } |
| |
| Value = EmitToMemory(Value, Ty); |
| |
| LValue AtomicLValue = |
| LValue::MakeAddr(Addr, Ty, getContext(), BaseInfo, TBAAInfo); |
| if (Ty->isAtomicType() || |
| (!isInit && LValueIsSuitableForInlineAtomic(AtomicLValue))) { |
| EmitAtomicStore(RValue::get(Value), AtomicLValue, isInit); |
| return; |
| } |
| |
| llvm::StoreInst *Store = Builder.CreateStore(Value, Addr, Volatile); |
| if (isNontemporal) { |
| llvm::MDNode *Node = |
| llvm::MDNode::get(Store->getContext(), |
| llvm::ConstantAsMetadata::get(Builder.getInt32(1))); |
| Store->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node); |
| } |
| |
| CGM.DecorateInstructionWithTBAA(Store, TBAAInfo); |
| } |
| |
| void CodeGenFunction::EmitStoreOfScalar(llvm::Value *value, LValue lvalue, |
| bool isInit) { |
| if (lvalue.getType()->isConstantMatrixType()) { |
| EmitStoreOfMatrixScalar(value, lvalue, isInit, *this); |
| return; |
| } |
| |
| EmitStoreOfScalar(value, lvalue.getAddress(*this), lvalue.isVolatile(), |
| lvalue.getType(), lvalue.getBaseInfo(), |
| lvalue.getTBAAInfo(), isInit, lvalue.isNontemporal()); |
| } |
| |
| // Emit a load of a LValue of matrix type. This may require casting the pointer |
| // to memory address (ArrayType) to a pointer to the value type (VectorType). |
| static RValue EmitLoadOfMatrixLValue(LValue LV, SourceLocation Loc, |
| CodeGenFunction &CGF) { |
| assert(LV.getType()->isConstantMatrixType()); |
| Address Addr = MaybeConvertMatrixAddress(LV.getAddress(CGF), CGF); |
| LV.setAddress(Addr); |
| return RValue::get(CGF.EmitLoadOfScalar(LV, Loc)); |
| } |
| |
| /// EmitLoadOfLValue - Given an expression that represents a value lvalue, this |
| /// method emits the address of the lvalue, then loads the result as an rvalue, |
| /// returning the rvalue. |
| RValue CodeGenFunction::EmitLoadOfLValue(LValue LV, SourceLocation Loc) { |
| if (LV.isObjCWeak()) { |
| // load of a __weak object. |
| Address AddrWeakObj = LV.getAddress(*this); |
| return RValue::get(CGM.getObjCRuntime().EmitObjCWeakRead(*this, |
| AddrWeakObj)); |
| } |
| if (LV.getQuals().getObjCLifetime() == Qualifiers::OCL_Weak) { |
| // In MRC mode, we do a load+autorelease. |
| if (!getLangOpts().ObjCAutoRefCount) { |
| return RValue::get(EmitARCLoadWeak(LV.getAddress(*this))); |
| } |
| |
| // In ARC mode, we load retained and then consume the value. |
| llvm::Value *Object = EmitARCLoadWeakRetained(LV.getAddress(*this)); |
| Object = EmitObjCConsumeObject(LV.getType(), Object); |
| return RValue::get(Object); |
| } |
| |
| if (LV.isSimple()) { |
| assert(!LV.getType()->isFunctionType()); |
| |
| if (LV.getType()->isConstantMatrixType()) |
| return EmitLoadOfMatrixLValue(LV, Loc, *this); |
| |
| // Everything needs a load. |
| return RValue::get(EmitLoadOfScalar(LV, Loc)); |
| } |
| |
| if (LV.isVectorElt()) { |
| llvm::LoadInst *Load = Builder.CreateLoad(LV.getVectorAddress(), |
| LV.isVolatileQualified()); |
| return RValue::get(Builder.CreateExtractElement(Load, LV.getVectorIdx(), |
| "vecext")); |
| } |
| |
| // If this is a reference to a subset of the elements of a vector, either |
| // shuffle the input or extract/insert them as appropriate. |
| if (LV.isExtVectorElt()) { |
| return EmitLoadOfExtVectorElementLValue(LV); |
| } |
| |
| // Global Register variables always invoke intrinsics |
| if (LV.isGlobalReg()) |
| return EmitLoadOfGlobalRegLValue(LV); |
| |
| if (LV.isMatrixElt()) { |
| llvm::LoadInst *Load = |
| Builder.CreateLoad(LV.getMatrixAddress(), LV.isVolatileQualified()); |
| return RValue::get( |
| Builder.CreateExtractElement(Load, LV.getMatrixIdx(), "matrixext")); |
| } |
| |
| assert(LV.isBitField() && "Unknown LValue type!"); |
| return EmitLoadOfBitfieldLValue(LV, Loc); |
| } |
| |
| RValue CodeGenFunction::EmitLoadOfBitfieldLValue(LValue LV, |
| SourceLocation Loc) { |
| const CGBitFieldInfo &Info = LV.getBitFieldInfo(); |
| |
| // Get the output type. |
| llvm::Type *ResLTy = ConvertType(LV.getType()); |
| |
| Address Ptr = LV.getBitFieldAddress(); |
| llvm::Value *Val = Builder.CreateLoad(Ptr, LV.isVolatileQualified(), "bf.load"); |
| |
| if (Info.IsSigned) { |
| assert(static_cast<unsigned>(Info.Offset + Info.Size) <= Info.StorageSize); |
| unsigned HighBits = Info.StorageSize - Info.Offset - Info.Size; |
| if (HighBits) |
| Val = Builder.CreateShl(Val, HighBits, "bf.shl"); |
| if (Info.Offset + HighBits) |
| Val = Builder.CreateAShr(Val, Info.Offset + HighBits, "bf.ashr"); |
| } else { |
| if (Info.Offset) |
| Val = Builder.CreateLShr(Val, Info.Offset, "bf.lshr"); |
| if (static_cast<unsigned>(Info.Offset) + Info.Size < Info.StorageSize) |
| Val = Builder.CreateAnd(Val, llvm::APInt::getLowBitsSet(Info.StorageSize, |
| Info.Size), |
| "bf.clear"); |
| } |
| Val = Builder.CreateIntCast(Val, ResLTy, Info.IsSigned, "bf.cast"); |
| EmitScalarRangeCheck(Val, LV.getType(), Loc); |
| return RValue::get(Val); |
| } |
| |
| // If this is a reference to a subset of the elements of a vector, create an |
| // appropriate shufflevector. |
| RValue CodeGenFunction::EmitLoadOfExtVectorElementLValue(LValue LV) { |
| llvm::Value *Vec = Builder.CreateLoad(LV.getExtVectorAddress(), |
| LV.isVolatileQualified()); |
| |
| const llvm::Constant *Elts = LV.getExtVectorElts(); |
| |
| // If the result of the expression is a non-vector type, we must be extracting |
| // a single element. Just codegen as an extractelement. |
| const VectorType *ExprVT = LV.getType()->getAs<VectorType>(); |
| if (!ExprVT) { |
| unsigned InIdx = getAccessedFieldNo(0, Elts); |
| llvm::Value *Elt = llvm::ConstantInt::get(SizeTy, InIdx); |
| return RValue::get(Builder.CreateExtractElement(Vec, Elt)); |
| } |
| |
| // Always use shuffle vector to try to retain the original program structure |
| unsigned NumResultElts = ExprVT->getNumElements(); |
| |
| SmallVector<int, 4> Mask; |
| for (unsigned i = 0; i != NumResultElts; ++i) |
| Mask.push_back(getAccessedFieldNo(i, Elts)); |
| |
| Vec = Builder.CreateShuffleVector(Vec, llvm::UndefValue::get(Vec->getType()), |
| Mask); |
| return RValue::get(Vec); |
| } |
| |
| /// Generates lvalue for partial ext_vector access. |
| Address CodeGenFunction::EmitExtVectorElementLValue(LValue LV) { |
| Address VectorAddress = LV.getExtVectorAddress(); |
| QualType EQT = LV.getType()->castAs<VectorType>()->getElementType(); |
| llvm::Type *VectorElementTy = CGM.getTypes().ConvertType(EQT); |
| |
| Address CastToPointerElement = |
| Builder.CreateElementBitCast(VectorAddress, VectorElementTy, |
| "conv.ptr.element"); |
| |
| const llvm::Constant *Elts = LV.getExtVectorElts(); |
| unsigned ix = getAccessedFieldNo(0, Elts); |
| |
| Address VectorBasePtrPlusIx = |
| Builder.CreateConstInBoundsGEP(CastToPointerElement, ix, |
| "vector.elt"); |
| |
| return VectorBasePtrPlusIx; |
| } |
| |
| /// Load of global gamed gegisters are always calls to intrinsics. |
| RValue CodeGenFunction::EmitLoadOfGlobalRegLValue(LValue LV) { |
| assert((LV.getType()->isIntegerType() || LV.getType()->isPointerType()) && |
| "Bad type for register variable"); |
| llvm::MDNode *RegName = cast<llvm::MDNode>( |
| cast<llvm::MetadataAsValue>(LV.getGlobalReg())->getMetadata()); |
| |
| // We accept integer and pointer types only |
| llvm::Type *OrigTy = CGM.getTypes().ConvertType(LV.getType()); |
| llvm::Type *Ty = OrigTy; |
| if (OrigTy->isPointerTy()) |
| Ty = CGM.getTypes().getDataLayout().getIntPtrType(OrigTy); |
| llvm::Type *Types[] = { Ty }; |
| |
| llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::read_register, Types); |
| llvm::Value *Call = Builder.CreateCall( |
| F, llvm::MetadataAsValue::get(Ty->getContext(), RegName)); |
| if (OrigTy->isPointerTy()) |
| Call = Builder.CreateIntToPtr(Call, OrigTy); |
| return RValue::get(Call); |
| } |
| |
| /// EmitStoreThroughLValue - Store the specified rvalue into the specified |
| /// lvalue, where both are guaranteed to the have the same type, and that type |
| /// is 'Ty'. |
| void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst, |
| bool isInit) { |
| if (!Dst.isSimple()) { |
| if (Dst.isVectorElt()) { |
| // Read/modify/write the vector, inserting the new element. |
| llvm::Value *Vec = Builder.CreateLoad(Dst.getVectorAddress(), |
| Dst.isVolatileQualified()); |
| Vec = Builder.CreateInsertElement(Vec, Src.getScalarVal(), |
| Dst.getVectorIdx(), "vecins"); |
| Builder.CreateStore(Vec, Dst.getVectorAddress(), |
| Dst.isVolatileQualified()); |
| return; |
| } |
| |
| // If this is an update of extended vector elements, insert them as |
| // appropriate. |
| if (Dst.isExtVectorElt()) |
| return EmitStoreThroughExtVectorComponentLValue(Src, Dst); |
| |
| if (Dst.isGlobalReg()) |
| return EmitStoreThroughGlobalRegLValue(Src, Dst); |
| |
| if (Dst.isMatrixElt()) { |
| llvm::Value *Vec = Builder.CreateLoad(Dst.getMatrixAddress()); |
| Vec = Builder.CreateInsertElement(Vec, Src.getScalarVal(), |
| Dst.getMatrixIdx(), "matins"); |
| Builder.CreateStore(Vec, Dst.getMatrixAddress(), |
| Dst.isVolatileQualified()); |
| return; |
| } |
| |
| assert(Dst.isBitField() && "Unknown LValue type"); |
| return EmitStoreThroughBitfieldLValue(Src, Dst); |
| } |
| |
| // There's special magic for assigning into an ARC-qualified l-value. |
| if (Qualifiers::ObjCLifetime Lifetime = Dst.getQuals().getObjCLifetime()) { |
| switch (Lifetime) { |
| case Qualifiers::OCL_None: |
| llvm_unreachable("present but none"); |
| |
| case Qualifiers::OCL_ExplicitNone: |
| // nothing special |
| break; |
| |
| case Qualifiers::OCL_Strong: |
| if (isInit) { |
| Src = RValue::get(EmitARCRetain(Dst.getType(), Src.getScalarVal())); |
| break; |
| } |
| EmitARCStoreStrong(Dst, Src.getScalarVal(), /*ignore*/ true); |
| return; |
| |
| case Qualifiers::OCL_Weak: |
| if (isInit) |
| // Initialize and then skip the primitive store. |
| EmitARCInitWeak(Dst.getAddress(*this), Src.getScalarVal()); |
| else |
| EmitARCStoreWeak(Dst.getAddress(*this), Src.getScalarVal(), |
| /*ignore*/ true); |
| return; |
| |
| case Qualifiers::OCL_Autoreleasing: |
| Src = RValue::get(EmitObjCExtendObjectLifetime(Dst.getType(), |
| Src.getScalarVal())); |
| // fall into the normal path |
| break; |
| } |
| } |
| |
| if (Dst.isObjCWeak() && !Dst.isNonGC()) { |
| // load of a __weak object. |
| Address LvalueDst = Dst.getAddress(*this); |
| llvm::Value *src = Src.getScalarVal(); |
| CGM.getObjCRuntime().EmitObjCWeakAssign(*this, src, LvalueDst); |
| return; |
| } |
| |
| if (Dst.isObjCStrong() && !Dst.isNonGC()) { |
| // load of a __strong object. |
| Address LvalueDst = Dst.getAddress(*this); |
| llvm::Value *src = Src.getScalarVal(); |
| if (Dst.isObjCIvar()) { |
| assert(Dst.getBaseIvarExp() && "BaseIvarExp is NULL"); |
| llvm::Type *ResultType = IntPtrTy; |
| Address dst = EmitPointerWithAlignment(Dst.getBaseIvarExp()); |
| llvm::Value *RHS = dst.getPointer(); |
| RHS = Builder.CreatePtrToInt(RHS, ResultType, "sub.ptr.rhs.cast"); |
| llvm::Value *LHS = |
| Builder.CreatePtrToInt(LvalueDst.getPointer(), ResultType, |
| "sub.ptr.lhs.cast"); |
| llvm::Value *BytesBetween = Builder.CreateSub(LHS, RHS, "ivar.offset"); |
| CGM.getObjCRuntime().EmitObjCIvarAssign(*this, src, dst, |
| BytesBetween); |
| } else if (Dst.isGlobalObjCRef()) { |
| CGM.getObjCRuntime().EmitObjCGlobalAssign(*this, src, LvalueDst, |
| Dst.isThreadLocalRef()); |
| } |
| else |
| CGM.getObjCRuntime().EmitObjCStrongCastAssign(*this, src, LvalueDst); |
| return; |
| } |
| |
| assert(Src.isScalar() && "Can't emit an agg store with this method"); |
| EmitStoreOfScalar(Src.getScalarVal(), Dst, isInit); |
| } |
| |
| void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst, |
| llvm::Value **Result) { |
| const CGBitFieldInfo &Info = Dst.getBitFieldInfo(); |
| llvm::Type *ResLTy = ConvertTypeForMem(Dst.getType()); |
| Address Ptr = Dst.getBitFieldAddress(); |
| |
| // Get the source value, truncated to the width of the bit-field. |
| llvm::Value *SrcVal = Src.getScalarVal(); |
| |
| // Cast the source to the storage type and shift it into place. |
| SrcVal = Builder.CreateIntCast(SrcVal, Ptr.getElementType(), |
| /*isSigned=*/false); |
| llvm::Value *MaskedVal = SrcVal; |
| |
| // See if there are other bits in the bitfield's storage we'll need to load |
| // and mask together with source before storing. |
| if (Info.StorageSize != Info.Size) { |
| assert(Info.StorageSize > Info.Size && "Invalid bitfield size."); |
| llvm::Value *Val = |
| Builder.CreateLoad(Ptr, Dst.isVolatileQualified(), "bf.load"); |
| |
| // Mask the source value as needed. |
| if (!hasBooleanRepresentation(Dst.getType())) |
| SrcVal = Builder.CreateAnd(SrcVal, |
| llvm::APInt::getLowBitsSet(Info.StorageSize, |
| Info.Size), |
| "bf.value"); |
| MaskedVal = SrcVal; |
| if (Info.Offset) |
| SrcVal = Builder.CreateShl(SrcVal, Info.Offset, "bf.shl"); |
| |
| // Mask out the original value. |
| Val = Builder.CreateAnd(Val, |
| ~llvm::APInt::getBitsSet(Info.StorageSize, |
| Info.Offset, |
| Info.Offset + Info.Size), |
| "bf.clear"); |
| |
| // Or together the unchanged values and the source value. |
| SrcVal = Builder.CreateOr(Val, SrcVal, "bf.set"); |
| } else { |
| assert(Info.Offset == 0); |
| // According to the AACPS: |
| // When a volatile bit-field is written, and its container does not overlap |
| // with any non-bit-field member, its container must be read exactly once and |
| // written exactly once using the access width appropriate to the type of the |
| // container. The two accesses are not atomic. |
| if (Dst.isVolatileQualified() && isAAPCS(CGM.getTarget()) && |
| CGM.getCodeGenOpts().ForceAAPCSBitfieldLoad) |
| Builder.CreateLoad(Ptr, true, "bf.load"); |
| } |
| |
| // Write the new value back out. |
| Builder.CreateStore(SrcVal, Ptr, Dst.isVolatileQualified()); |
| |
| // Return the new value of the bit-field, if requested. |
| if (Result) { |
| llvm::Value *ResultVal = MaskedVal; |
| |
| // Sign extend the value if needed. |
| if (Info.IsSigned) { |
| assert(Info.Size <= Info.StorageSize); |
| unsigned HighBits = Info.StorageSize - Info.Size; |
| if (HighBits) { |
| ResultVal = Builder.CreateShl(ResultVal, HighBits, "bf.result.shl"); |
| ResultVal = Builder.CreateAShr(ResultVal, HighBits, "bf.result.ashr"); |
| } |
| } |
| |
| ResultVal = Builder.CreateIntCast(ResultVal, ResLTy, Info.IsSigned, |
| "bf.result.cast"); |
| *Result = EmitFromMemory(ResultVal, Dst.getType()); |
| } |
| } |
| |
| void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src, |
| LValue Dst) { |
| // This access turns into a read/modify/write of the vector. Load the input |
| // value now. |
| llvm::Value *Vec = Builder.CreateLoad(Dst.getExtVectorAddress(), |
| Dst.isVolatileQualified()); |
| const llvm::Constant *Elts = Dst.getExtVectorElts(); |
| |
| llvm::Value *SrcVal = Src.getScalarVal(); |
| |
| if (const VectorType *VTy = Dst.getType()->getAs<VectorType>()) { |
| unsigned NumSrcElts = VTy->getNumElements(); |
| unsigned NumDstElts = |
| cast<llvm::FixedVectorType>(Vec->getType())->getNumElements(); |
| if (NumDstElts == NumSrcElts) { |
| // Use shuffle vector is the src and destination are the same number of |
| // elements and restore the vector mask since it is on the side it will be |
| // stored. |
| SmallVector<int, 4> Mask(NumDstElts); |
| for (unsigned i = 0; i != NumSrcElts; ++i) |
| Mask[getAccessedFieldNo(i, Elts)] = i; |
| |
| Vec = Builder.CreateShuffleVector( |
| SrcVal, llvm::UndefValue::get(Vec->getType()), Mask); |
| } else if (NumDstElts > NumSrcElts) { |
| // Extended the source vector to the same length and then shuffle it |
| // into the destination. |
| // FIXME: since we're shuffling with undef, can we just use the indices |
| // into that? This could be simpler. |
| SmallVector<int, 4> ExtMask; |
| for (unsigned i = 0; i != NumSrcElts; ++i) |
| ExtMask.push_back(i); |
| ExtMask.resize(NumDstElts, -1); |
| llvm::Value *ExtSrcVal = Builder.CreateShuffleVector( |
| SrcVal, llvm::UndefValue::get(SrcVal->getType()), ExtMask); |
| // build identity |
| SmallVector<int, 4> Mask; |
| for (unsigned i = 0; i != NumDstElts; ++i) |
| Mask.push_back(i); |
| |
| // When the vector size is odd and .odd or .hi is used, the last element |
| // of the Elts constant array will be one past the size of the vector. |
| // Ignore the last element here, if it is greater than the mask size. |
| if (getAccessedFieldNo(NumSrcElts - 1, Elts) == Mask.size()) |
| NumSrcElts--; |
| |
| // modify when what gets shuffled in |
| for (unsigned i = 0; i != NumSrcElts; ++i) |
| Mask[getAccessedFieldNo(i, Elts)] = i + NumDstElts; |
| Vec = Builder.CreateShuffleVector(Vec, ExtSrcVal, Mask); |
| } else { |
| // We should never shorten the vector |
| llvm_unreachable("unexpected shorten vector length"); |
| } |
| } else { |
| // If the Src is a scalar (not a vector) it must be updating one element. |
| unsigned InIdx = getAccessedFieldNo(0, Elts); |
| llvm::Value *Elt = llvm::ConstantInt::get(SizeTy, InIdx); |
| Vec = Builder.CreateInsertElement(Vec, SrcVal, Elt); |
| } |
| |
| Builder.CreateStore(Vec, Dst.getExtVectorAddress(), |
| Dst.isVolatileQualified()); |
| } |
| |
| /// Store of global named registers are always calls to intrinsics. |
| void CodeGenFunction::EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst) { |
| assert((Dst.getType()->isIntegerType() || Dst.getType()->isPointerType()) && |
| "Bad type for register variable"); |
| llvm::MDNode *RegName = cast<llvm::MDNode>( |
| cast<llvm::MetadataAsValue>(Dst.getGlobalReg())->getMetadata()); |
| assert(RegName && "Register LValue is not metadata"); |
| |
| // We accept integer and pointer types only |
| llvm::Type *OrigTy = CGM.getTypes().ConvertType(Dst.getType()); |
| llvm::Type *Ty = OrigTy; |
| if (OrigTy->isPointerTy()) |
| Ty = CGM.getTypes().getDataLayout().getIntPtrType(OrigTy); |
| llvm::Type *Types[] = { Ty }; |
| |
| llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::write_register, Types); |
| llvm::Value *Value = Src.getScalarVal(); |
| if (OrigTy->isPointerTy()) |
| Value = Builder.CreatePtrToInt(Value, Ty); |
| Builder.CreateCall( |
| F, {llvm::MetadataAsValue::get(Ty->getContext(), RegName), Value}); |
| } |
| |
| // setObjCGCLValueClass - sets class of the lvalue for the purpose of |
| // generating write-barries API. It is currently a global, ivar, |
| // or neither. |
| static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E, |
| LValue &LV, |
| bool IsMemberAccess=false) { |
| if (Ctx.getLangOpts().getGC() == LangOptions::NonGC) |
| return; |
| |
| if (isa<ObjCIvarRefExpr>(E)) { |
| QualType ExpTy = E->getType(); |
| if (IsMemberAccess && ExpTy->isPointerType()) { |
| // If ivar is a structure pointer, assigning to field of |
| // this struct follows gcc's behavior and makes it a non-ivar |
| // writer-barrier conservatively. |
| ExpTy = ExpTy->castAs<PointerType>()->getPointeeType(); |
| if (ExpTy->isRecordType()) { |
| LV.setObjCIvar(false); |
| return; |
| } |
| } |
| LV.setObjCIvar(true); |
| auto *Exp = cast<ObjCIvarRefExpr>(const_cast<Expr *>(E)); |
| LV.setBaseIvarExp(Exp->getBase()); |
| LV.setObjCArray(E->getType()->isArrayType()); |
| return; |
| } |
| |
| if (const auto *Exp = dyn_cast<DeclRefExpr>(E)) { |
| if (const auto *VD = dyn_cast<VarDecl>(Exp->getDecl())) { |
| if (VD->hasGlobalStorage()) { |
| LV.setGlobalObjCRef(true); |
| LV.setThreadLocalRef(VD->getTLSKind() != VarDecl::TLS_None); |
| } |
| } |
| LV.setObjCArray(E->getType()->isArrayType()); |
| return; |
| } |
| |
| if (const auto *Exp = dyn_cast<UnaryOperator>(E)) { |
| setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess); |
| return; |
| } |
| |
| if (const auto *Exp = dyn_cast<ParenExpr>(E)) { |
| setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess); |
| if (LV.isObjCIvar()) { |
| // If cast is to a structure pointer, follow gcc's behavior and make it |
| // a non-ivar write-barrier. |
| QualType ExpTy = E->getType(); |
| if (ExpTy->isPointerType()) |
| ExpTy = ExpTy->castAs<PointerType>()->getPointeeType(); |
| if (ExpTy->isRecordType()) |
| LV.setObjCIvar(false); |
| } |
| return; |
| } |
| |
| if (const auto *Exp = dyn_cast<GenericSelectionExpr>(E)) { |
| setObjCGCLValueClass(Ctx, Exp->getResultExpr(), LV); |
| return; |
| } |
| |
| if (const auto *Exp = dyn_cast<ImplicitCastExpr>(E)) { |
| setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess); |
| return; |
| } |
| |
| if (const auto *Exp = dyn_cast<CStyleCastExpr>(E)) { |
| setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess); |
| return; |
| } |
| |
| if (const auto *Exp = dyn_cast<ObjCBridgedCastExpr>(E)) { |
| setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess); |
| return; |
| } |
| |
| if (const auto *Exp = dyn_cast<ArraySubscriptExpr>(E)) { |
| setObjCGCLValueClass(Ctx, Exp->getBase(), LV); |
| if (LV.isObjCIvar() && !LV.isObjCArray()) |
| // Using array syntax to assigning to what an ivar points to is not |
| // same as assigning to the ivar itself. {id *Names;} Names[i] = 0; |
| LV.setObjCIvar(false); |
| else if (LV.isGlobalObjCRef() && !LV.isObjCArray()) |
| // Using array syntax to assigning to what global points to is not |
| // same as assigning to the global itself. {id *G;} G[i] = 0; |
| LV.setGlobalObjCRef(false); |
| return; |
| } |
| |
| if (const auto *Exp = dyn_cast<MemberExpr>(E)) { |
| setObjCGCLValueClass(Ctx, Exp->getBase(), LV, true); |
| // We don't know if member is an 'ivar', but this flag is looked at |
| // only in the context of LV.isObjCIvar(). |
| LV.setObjCArray(E->getType()->isArrayType()); |
| return; |
| } |
| } |
| |
| static llvm::Value * |
| EmitBitCastOfLValueToProperType(CodeGenFunction &CGF, |
| llvm::Value *V, llvm::Type *IRType, |
| StringRef Name = StringRef()) { |
| unsigned AS = cast<llvm::PointerType>(V->getType())->getAddressSpace(); |
| return CGF.Builder.CreateBitCast(V, IRType->getPointerTo(AS), Name); |
| } |
| |
| static LValue EmitThreadPrivateVarDeclLValue( |
| CodeGenFunction &CGF, const VarDecl *VD, QualType T, Address Addr, |
| llvm::Type *RealVarTy, SourceLocation Loc) { |
| if (CGF.CGM.getLangOpts().OpenMPIRBuilder) |
| Addr = CodeGenFunction::OMPBuilderCBHelpers::getAddrOfThreadPrivate( |
| CGF, VD, Addr, Loc); |
| else |
| Addr = |
| CGF.CGM.getOpenMPRuntime().getAddrOfThreadPrivate(CGF, VD, Addr, Loc); |
| |
| Addr = CGF.Builder.CreateElementBitCast(Addr, RealVarTy); |
| return CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl); |
| } |
| |
| static Address emitDeclTargetVarDeclLValue(CodeGenFunction &CGF, |
| const VarDecl *VD, QualType T) { |
| llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res = |
| OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD); |
| // Return an invalid address if variable is MT_To and unified |
| // memory is not enabled. For all other cases: MT_Link and |
| // MT_To with unified memory, return a valid address. |
| if (!Res || (*Res == OMPDeclareTargetDeclAttr::MT_To && |
| !CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) |
| return Address::invalid(); |
| assert(((*Res == OMPDeclareTargetDeclAttr::MT_Link) || |
| (*Res == OMPDeclareTargetDeclAttr::MT_To && |
| CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) && |
| "Expected link clause OR to clause with unified memory enabled."); |
| QualType PtrTy = CGF.getContext().getPointerType(VD->getType()); |
| Address Addr = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD); |
| return CGF.EmitLoadOfPointer(Addr, PtrTy->castAs<PointerType>()); |
| } |
| |
| Address |
| CodeGenFunction::EmitLoadOfReference(LValue RefLVal, |
| LValueBaseInfo *PointeeBaseInfo, |
| TBAAAccessInfo *PointeeTBAAInfo) { |
| llvm::LoadInst *Load = |
| Builder.CreateLoad(RefLVal.getAddress(*this), RefLVal.isVolatile()); |
| CGM.DecorateInstructionWithTBAA(Load, RefLVal.getTBAAInfo()); |
| |
| CharUnits Align = CGM.getNaturalTypeAlignment( |
| RefLVal.getType()->getPointeeType(), PointeeBaseInfo, PointeeTBAAInfo, |
| /* forPointeeType= */ true); |
| return Address(Load, Align); |
| } |
| |
| LValue CodeGenFunction::EmitLoadOfReferenceLValue(LValue RefLVal) { |
| LValueBaseInfo PointeeBaseInfo; |
| TBAAAccessInfo PointeeTBAAInfo; |
| Address PointeeAddr = EmitLoadOfReference(RefLVal, &PointeeBaseInfo, |
| &PointeeTBAAInfo); |
| return MakeAddrLValue(PointeeAddr, RefLVal.getType()->getPointeeType(), |
| PointeeBaseInfo, PointeeTBAAInfo); |
| } |
| |
| Address CodeGenFunction::EmitLoadOfPointer(Address Ptr, |
| const PointerType *PtrTy, |
| LValueBaseInfo *BaseInfo, |
| TBAAAccessInfo *TBAAInfo) { |
| llvm::Value *Addr = Builder.CreateLoad(Ptr); |
| return Address(Addr, CGM.getNaturalTypeAlignment(PtrTy->getPointeeType(), |
| BaseInfo, TBAAInfo, |
| /*forPointeeType=*/true)); |
| } |
| |
| LValue CodeGenFunction::EmitLoadOfPointerLValue(Address PtrAddr, |
| const PointerType *PtrTy) { |
| LValueBaseInfo BaseInfo; |
| TBAAAccessInfo TBAAInfo; |
| Address Addr = EmitLoadOfPointer(PtrAddr, PtrTy, &BaseInfo, &TBAAInfo); |
| return MakeAddrLValue(Addr, PtrTy->getPointeeType(), BaseInfo, TBAAInfo); |
| } |
| |
| static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF, |
| const Expr *E, const VarDecl *VD) { |
| QualType T = E->getType(); |
| |
| // If it's thread_local, emit a call to its wrapper function instead. |
| if (VD->getTLSKind() == VarDecl::TLS_Dynamic && |
| CGF.CGM.getCXXABI().usesThreadWrapperFunction(VD)) |
| return CGF.CGM.getCXXABI().EmitThreadLocalVarDeclLValue(CGF, VD, T); |
| // Check if the variable is marked as declare target with link clause in |
| // device codegen. |
| if (CGF.getLangOpts().OpenMPIsDevice) { |
| Address Addr = emitDeclTargetVarDeclLValue(CGF, VD, T); |
| if (Addr.isValid()) |
| return CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl); |
| } |
| |
| llvm::Value *V = CGF.CGM.GetAddrOfGlobalVar(VD); |
| llvm::Type *RealVarTy = CGF.getTypes().ConvertTypeForMem(VD->getType()); |
| V = EmitBitCastOfLValueToProperType(CGF, V, RealVarTy); |
| CharUnits Alignment = CGF.getContext().getDeclAlign(VD); |
| Address Addr(V, Alignment); |
| // Emit reference to the private copy of the variable if it is an OpenMP |
| // threadprivate variable. |
| if (CGF.getLangOpts().OpenMP && !CGF.getLangOpts().OpenMPSimd && |
| VD->hasAttr<OMPThreadPrivateDeclAttr>()) { |
| return EmitThreadPrivateVarDeclLValue(CGF, VD, T, Addr, RealVarTy, |
| E->getExprLoc()); |
| } |
| LValue LV = VD->getType()->isReferenceType() ? |
| CGF.EmitLoadOfReferenceLValue(Addr, VD->getType(), |
| AlignmentSource::Decl) : |
| CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl); |
| setObjCGCLValueClass(CGF.getContext(), E, LV); |
| return LV; |
| } |
| |
| static llvm::Constant *EmitFunctionDeclPointer(CodeGenModule &CGM, |
| GlobalDecl GD) { |
| const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl()); |
| if (FD->hasAttr<WeakRefAttr>()) { |
| ConstantAddress aliasee = CGM.GetWeakRefReference(FD); |
| return aliasee.getPointer(); |
| } |
| |
| llvm::Constant *V = CGM.GetAddrOfFunction(GD); |
| if (!FD->hasPrototype()) { |
| if (const FunctionProtoType *Proto = |
| FD->getType()->getAs<FunctionProtoType>()) { |
| // Ugly case: for a K&R-style definition, the type of the definition |
| // isn't the same as the type of a use. Correct for this with a |
| // bitcast. |
| QualType NoProtoType = |
| CGM.getContext().getFunctionNoProtoType(Proto->getReturnType()); |
| NoProtoType = CGM.getContext().getPointerType(NoProtoType); |
| V = llvm::ConstantExpr::getBitCast(V, |
| CGM.getTypes().ConvertType(NoProtoType)); |
| } |
| } |
| return V; |
| } |
| |
| static LValue EmitFunctionDeclLValue(CodeGenFunction &CGF, const Expr *E, |
| GlobalDecl GD) { |
| const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl()); |
| llvm::Value *V = EmitFunctionDeclPointer(CGF.CGM, GD); |
| CharUnits Alignment = CGF.getContext().getDeclAlign(FD); |
| return CGF.MakeAddrLValue(V, E->getType(), Alignment, |
| AlignmentSource::Decl); |
| } |
| |
| static LValue EmitCapturedFieldLValue(CodeGenFunction &CGF, const FieldDecl *FD, |
| llvm::Value *ThisValue) { |
| QualType TagType = CGF.getContext().getTagDeclType(FD->getParent()); |
| LValue LV = CGF.MakeNaturalAlignAddrLValue(ThisValue, TagType); |
| return CGF.EmitLValueForField(LV, FD |