Google Git
Sign in
fuchsia / third_party / swift / refs/tags/swift-DEVELOPMENT-SNAPSHOT-2019-04-04-a / . / lib / IRGen / GenOpaque.cpp
blob: cd033a06736560ceda4ebd10c60bfb6ee6d5073a [file] [log] [blame]
//===--- GenOpaque.cpp - Swift IR-generation for opaque values ------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
//
// This file implements IR generation for opaque values and value
// witness operations.
//
// In the comments throughout this file, three type names are used:
// 'B' is the type of a fixed-size buffer
// 'T' is the type which implements a protocol
// 'W' is the type of a witness to the protocol
//
//===----------------------------------------------------------------------===//
#include "llvm/ADT/SmallString.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/IR/DerivedTypes.h"
#include "swift/AST/IRGenOptions.h"
#include "swift/ABI/MetadataValues.h"
#include "swift/IRGen/ValueWitness.h"
#include "swift/SIL/TypeLowering.h"
#include "Callee.h"
#include "Explosion.h"
#include "FixedTypeInfo.h"
#include "IRGenFunction.h"
#include "IRGenModule.h"
#include "ProtocolInfo.h"
#include "GenOpaque.h"
using namespace swift;
using namespace irgen;
/// A fixed-size buffer is always three pointers in size and pointer-aligned.
/// If we align them more, we'll need to introduce padding to
/// make protocol types work.
Size irgen::getFixedBufferSize(IRGenModule &IGM) {
return NumWords_ValueBuffer * IGM.getPointerSize();
}
Alignment irgen::getFixedBufferAlignment(IRGenModule &IGM) {
return IGM.getPointerAlignment();
}
/// Lazily create the standard fixed-buffer type.
llvm::Type *IRGenModule::getFixedBufferTy() {
if (FixedBufferTy) return FixedBufferTy;
auto size = getFixedBufferSize(*this).getValue();
FixedBufferTy = llvm::ArrayType::get(Int8Ty, size);
return FixedBufferTy;
}
static llvm::Type *createWitnessType(IRGenModule &IGM, ValueWitness index) {
switch (index) {
// void (*destroy)(T *object, witness_t *self);
case ValueWitness::Destroy: {
llvm::Type *args[] = { IGM.OpaquePtrTy, IGM.TypeMetadataPtrTy };
return llvm::FunctionType::get(IGM.VoidTy, args, /*isVarArg*/ false);
}
// T *(*initializeBufferWithCopyOfBuffer)(B *dest, B *src, M *self);
case ValueWitness::InitializeBufferWithCopyOfBuffer: {
llvm::Type *bufPtrTy = IGM.getFixedBufferTy()->getPointerTo(0);
llvm::Type *args[] = { bufPtrTy, bufPtrTy, IGM.TypeMetadataPtrTy };
return llvm::FunctionType::get(IGM.OpaquePtrTy, args, /*isVarArg*/ false);
}
// T *(*assignWithCopy)(T *dest, T *src, M *self);
// T *(*assignWithTake)(T *dest, T *src, M *self);
// T *(*initializeWithCopy)(T *dest, T *src, M *self);
// T *(*initializeWithTake)(T *dest, T *src, M *self);
case ValueWitness::AssignWithCopy:
case ValueWitness::AssignWithTake:
case ValueWitness::InitializeWithCopy:
case ValueWitness::InitializeWithTake: {
llvm::Type *ptrTy = IGM.OpaquePtrTy;
llvm::Type *args[] = { ptrTy, ptrTy, IGM.TypeMetadataPtrTy };
return llvm::FunctionType::get(ptrTy, args, /*isVarArg*/ false);
}
/// unsigned (*getEnumTag)(T *obj, M *self);
case ValueWitness::GetEnumTag: {
llvm::Type *ptrTy = IGM.OpaquePtrTy;
llvm::Type *metaTy = IGM.TypeMetadataPtrTy;
llvm::Type *indexTy = IGM.Int32Ty;
llvm::Type *args[] = {ptrTy, metaTy};
return llvm::FunctionType::get(indexTy, args, /*isVarArg*/ false);
}
/// void (*destructiveProjectEnumData)(T *obj, M *self);
case ValueWitness::DestructiveProjectEnumData: {
llvm::Type *voidTy = IGM.VoidTy;
llvm::Type *ptrTy = IGM.OpaquePtrTy;
llvm::Type *metaTy = IGM.TypeMetadataPtrTy;
llvm::Type *args[] = {ptrTy, metaTy};
return llvm::FunctionType::get(voidTy, args, /*isVarArg*/ false);
}
/// void (*destructiveInjectEnumTag)(T *obj, unsigned tag, M *self);
case ValueWitness::DestructiveInjectEnumTag: {
llvm::Type *voidTy = IGM.VoidTy;
llvm::Type *ptrTy = IGM.OpaquePtrTy;
llvm::Type *indexTy = IGM.Int32Ty;
llvm::Type *metaTy = IGM.TypeMetadataPtrTy;
llvm::Type *args[] = {ptrTy, indexTy, metaTy};
return llvm::FunctionType::get(voidTy, args, /*isVarArg*/ false);
}
/// unsigned (*getEnumTagSinglePayload)(const T* enum, UINT_TYPE emptyCases,
/// M *self)
case ValueWitness::GetEnumTagSinglePayload: {
llvm::Type *ptrTy = IGM.OpaquePtrTy;
llvm::Type *indexTy = IGM.Int32Ty;
llvm::Type *metaTy = IGM.TypeMetadataPtrTy;
llvm::Type *args[] = { ptrTy, indexTy, metaTy };
return llvm::FunctionType::get(indexTy, args, false);
}
/// void (*storeEnumTagSinglePayload)(T* enum, UINT_TYPE whichCase,
/// UINT_TYPE emptyCases,
/// M *self)
case ValueWitness::StoreEnumTagSinglePayload: {
llvm::Type *voidTy = IGM.VoidTy;
llvm::Type *ptrTy = IGM.OpaquePtrTy;
llvm::Type *indexTy = IGM.Int32Ty;
llvm::Type *metaTy = IGM.TypeMetadataPtrTy;
llvm::Type *args[] = { ptrTy, indexTy, indexTy, metaTy };
return llvm::FunctionType::get(voidTy, args, false);
}
case ValueWitness::Size:
case ValueWitness::Stride:
return IGM.SizeTy;
case ValueWitness::Flags:
case ValueWitness::ExtraInhabitantCount:
return IGM.Int32Ty;
}
llvm_unreachable("bad value witness!");
}
static llvm::AttributeList getValueWitnessAttrs(IRGenModule &IGM,
ValueWitness index) {
assert(isValueWitnessFunction(index));
auto &ctx = IGM.getLLVMContext();
// All value witnesses are nounwind.
auto attrs = llvm::AttributeList::get(ctx, llvm::AttributeList::FunctionIndex,
llvm::Attribute::NoUnwind);
switch (index) {
// These have two arguments, but they can alias.
case ValueWitness::AssignWithCopy:
return attrs;
// These have one argument.
case ValueWitness::Destroy:
case ValueWitness::DestructiveInjectEnumTag:
case ValueWitness::DestructiveProjectEnumData:
case ValueWitness::GetEnumTag:
case ValueWitness::StoreEnumTagSinglePayload:
return attrs.addAttribute(ctx, 1, llvm::Attribute::NoAlias);
case ValueWitness::GetEnumTagSinglePayload:
return attrs
.addAttribute(ctx, llvm::AttributeList::FunctionIndex,
llvm::Attribute::ReadOnly)
.addAttribute(ctx, 1, llvm::Attribute::NoAlias);
// These have two arguments and they don't alias each other.
case ValueWitness::AssignWithTake:
case ValueWitness::InitializeBufferWithCopyOfBuffer:
case ValueWitness::InitializeWithCopy:
case ValueWitness::InitializeWithTake:
return attrs.addAttribute(ctx, 1, llvm::Attribute::NoAlias)
.addAttribute(ctx, 2, llvm::Attribute::NoAlias);
case ValueWitness::Size:
case ValueWitness::Flags:
case ValueWitness::ExtraInhabitantCount:
case ValueWitness::Stride:
llvm_unreachable("not a function value witness");
}
llvm_unreachable("bad witness");
}
/// Return the cached pointer-to-function type for the given value
/// witness index.
llvm::Type *IRGenModule::getValueWitnessTy(ValueWitness index) {
assert(unsigned(index) < MaxNumValueWitnesses);
auto &ty = ValueWitnessTys[unsigned(index)];
if (ty) return ty;
ty = createWitnessType(*this, index);
return ty;
}
Signature IRGenModule::getValueWitnessSignature(ValueWitness index) {
assert(isValueWitnessFunction(index));
auto fnTy = cast<llvm::FunctionType>(getValueWitnessTy(index));
auto attrs = getValueWitnessAttrs(*this, index);
return Signature(fnTy, attrs, DefaultCC);
}
static StringRef getValueWitnessLabel(ValueWitness index) {
switch (index) {
case ValueWitness::Destroy:
return "destroy";
case ValueWitness::InitializeBufferWithCopyOfBuffer:
return "initializeBufferWithCopyOfBuffer";
case ValueWitness::AssignWithCopy:
return "assignWithCopy";
case ValueWitness::AssignWithTake:
return "assignWithTake";
case ValueWitness::InitializeWithCopy:
return "initializeWithCopy";
case ValueWitness::InitializeWithTake:
return "initializeWithTake";
case ValueWitness::Size:
return "size";
case ValueWitness::Flags:
return "flags";
case ValueWitness::ExtraInhabitantCount:
return "extraInhabitantCount";
case ValueWitness::Stride:
return "stride";
case ValueWitness::GetEnumTag:
return "getEnumTag";
case ValueWitness::DestructiveProjectEnumData:
return "destructiveProjectEnumData";
case ValueWitness::DestructiveInjectEnumTag:
return "destructiveInjectEnumTag";
case ValueWitness::GetEnumTagSinglePayload:
return "getEnumTagSinglePayload";
case ValueWitness::StoreEnumTagSinglePayload:
return "storeEnumTagSinglePayload";
}
llvm_unreachable("bad value witness index");
}
static llvm::PointerType *
getOrCreateValueWitnessTablePtrTy(IRGenModule &IGM, llvm::PointerType *&cache,
StringRef name, bool includeEnumWitnesses) {
if (cache) return cache;
SmallVector<llvm::Type*, 16> types;
#define FUNC(lowerId, upperId, retTy, paramTys) \
types.push_back(IGM.Int8PtrTy);
#define DATA(lowerId, upperId, ty) \
types.push_back(IGM.getValueWitnessTy(ValueWitness::upperId));
// Add the base value witnesses.
#define WANT_ONLY_REQUIRED_VALUE_WITNESSES
#define FUNCTION_VALUE_WITNESS FUNC
#define DATA_VALUE_WITNESS DATA
#include "swift/ABI/ValueWitness.def"
// Add the enum value witnesses.
if (includeEnumWitnesses) {
#define WANT_ONLY_ENUM_VALUE_WITNESSES
#define FUNCTION_VALUE_WITNESS FUNC
#define DATA_VALUE_WITNESS DATA
#include "swift/ABI/ValueWitness.def"
}
#undef DATA
#undef FUNC
auto structTy = llvm::StructType::create(types, name);
auto ptrTy = structTy->getPointerTo();
cache = ptrTy;
return ptrTy;
}
llvm::StructType *IRGenModule::getValueWitnessTableTy() {
return cast<llvm::StructType>(getValueWitnessTablePtrTy()->getElementType());
}
llvm::PointerType *IRGenModule::getValueWitnessTablePtrTy() {
return getOrCreateValueWitnessTablePtrTy(*this, ValueWitnessTablePtrTy,
"swift.vwtable", false);
}
llvm::StructType *IRGenModule::getEnumValueWitnessTableTy() {
return cast<llvm::StructType>(getEnumValueWitnessTablePtrTy()
->getElementType());
}
llvm::PointerType *IRGenModule::getEnumValueWitnessTablePtrTy() {
return getOrCreateValueWitnessTablePtrTy(*this, EnumValueWitnessTablePtrTy,
"swift.enum_vwtable", true);
}
/// Load a specific witness from a known table. The result is
/// always an i8*.
llvm::Value *irgen::emitInvariantLoadOfOpaqueWitness(IRGenFunction &IGF,
llvm::Value *table,
WitnessIndex index) {
assert(table->getType() == IGF.IGM.WitnessTablePtrTy);
// GEP to the appropriate index, avoiding spurious IR in the trivial case.
llvm::Value *slot = table;
if (index.getValue() != 0)
slot = IGF.Builder.CreateConstInBoundsGEP1_32(
/*Ty=*/nullptr, table, index.getValue());
auto witness =
IGF.Builder.CreateLoad(Address(slot, IGF.IGM.getPointerAlignment()));
IGF.setInvariantLoad(witness);
return witness;
}
/// Load a specific witness from a known table. The result is
/// always an i8*.
llvm::Value *irgen::emitInvariantLoadOfOpaqueWitness(IRGenFunction &IGF,
llvm::Value *table,
llvm::Value *index) {
assert(table->getType() == IGF.IGM.WitnessTablePtrTy);
// GEP to the appropriate index.
llvm::Value *slot = IGF.Builder.CreateInBoundsGEP(table, index);
auto witness =
IGF.Builder.CreateLoad(Address(slot, IGF.IGM.getPointerAlignment()));
IGF.setInvariantLoad(witness);
return witness;
}
/// Given a value witness table, load one of the value witnesses.
/// The result has the appropriate type for the witness.
static llvm::Value *emitLoadOfValueWitnessValue(IRGenFunction &IGF,
llvm::Value *table,
ValueWitness witness) {
assert(!isValueWitnessFunction(witness));
assert(unsigned(witness) <= unsigned(ValueWitness::ExtraInhabitantCount) &&
"extraInhabitantCount not the last non-function value witness");
auto pointerSize = IGF.IGM.getPointerSize();
// Most of the witnesses are at an offset that's just a multiple of the
// pointer size, but the extra-inhabitant count is packed in after the
// 32-bit flags.
// This computation is correct for all pointer sizes, including 16.
// It would be wrong if size_t is ever a different size from the pointer
// size, though.
Size offset =
(witness == ValueWitness::ExtraInhabitantCount
? unsigned(ValueWitness::Flags) * pointerSize + Size(4)
: unsigned(witness) * pointerSize);
Address addr = Address(table, IGF.IGM.getPointerAlignment());
addr = IGF.Builder.CreateBitCast(addr, IGF.IGM.getValueWitnessTablePtrTy());
addr = IGF.Builder.CreateStructGEP(addr, unsigned(witness), offset);
auto load = IGF.Builder.CreateLoad(addr, getValueWitnessLabel(witness));
IGF.setInvariantLoad(load);
return load;
}
/// Given a type metadata pointer, load one of the value witnesses from its
/// value witness table.
static llvm::Value *
emitLoadOfValueWitnessValueFromMetadata(IRGenFunction &IGF,
llvm::Value *metadata,
ValueWitness index) {
llvm::Value *vwtable = IGF.emitValueWitnessTableRefForMetadata(metadata);
return emitLoadOfValueWitnessValue(IGF, vwtable, index);
}
/// Given a value witness table, load one of the value witnesses.
/// The result has the appropriate type for the witness.
static FunctionPointer emitLoadOfValueWitnessFunction(IRGenFunction &IGF,
llvm::Value *table,
ValueWitness index) {
assert(isValueWitnessFunction(index));
WitnessIndex windex = [&] {
unsigned i = unsigned(index);
if (i > unsigned(ValueWitness::Flags)) {
if (IGF.IGM.getPointerSize() == Size(8)) {
i--; // one pointer width skips both flags and xiCount
} else if (IGF.IGM.getPointerSize() == Size(4)) {
// no adjustment required
} else {
assert(IGF.IGM.getPointerSize() == Size(2));
i += 2; // flags and xiCount take up two pointers apiece
}
}
return WitnessIndex(i, false);
}();
llvm::Value *witness = emitInvariantLoadOfOpaqueWitness(IGF, table, windex);
auto label = getValueWitnessLabel(index);
auto signature = IGF.IGM.getValueWitnessSignature(index);
auto type = signature.getType()->getPointerTo();
witness = IGF.Builder.CreateBitCast(witness, type, label);
return FunctionPointer(witness, signature);
}
/// Given a type metadata pointer, load one of the function
/// value witnesses from its value witness table.
static FunctionPointer
emitLoadOfValueWitnessFunctionFromMetadata(IRGenFunction &IGF,
llvm::Value *metadata,
ValueWitness index) {
llvm::Value *vwtable = IGF.emitValueWitnessTableRefForMetadata(metadata);
return emitLoadOfValueWitnessFunction(IGF, vwtable, index);
}
llvm::Value *IRGenFunction::emitValueWitnessValue(SILType type,
ValueWitness index) {
assert(!isValueWitnessFunction(index));
auto key = LocalTypeDataKind::forValueWitness(index);
if (auto witness = tryGetLocalTypeDataForLayout(type, key)) {
return witness;
}
auto vwtable = emitValueWitnessTableRef(type);
auto witness = emitLoadOfValueWitnessValue(*this, vwtable, index);
setScopedLocalTypeDataForLayout(type, key, witness);
return witness;
}
FunctionPointer
IRGenFunction::emitValueWitnessFunctionRef(SILType type,
llvm::Value *&metadataSlot,
ValueWitness index) {
assert(isValueWitnessFunction(index));
auto key = LocalTypeDataKind::forValueWitness(index);
if (auto witness = tryGetLocalTypeDataForLayout(type, key)) {
metadataSlot = emitTypeMetadataRefForLayout(type);
auto signature = IGM.getValueWitnessSignature(index);
return FunctionPointer(witness, signature);
}
auto vwtable = emitValueWitnessTableRef(type, &metadataSlot);
auto witness = emitLoadOfValueWitnessFunction(*this, vwtable, index);
setScopedLocalTypeDataForLayout(type, key, witness.getPointer());
return witness;
}
static llvm::Value *emitCastToOpaquePtr(IRGenFunction &IGF,
Address object) {
return IGF.Builder.CreateBitCast(object.getAddress(), IGF.IGM.OpaquePtrTy);
}
llvm::Value *irgen::emitInitializeBufferWithCopyOfBufferCall(IRGenFunction &IGF,
SILType T,
Address destBuffer,
Address srcBuffer) {
auto metadata = IGF.emitTypeMetadataRefForLayout(T);
return emitInitializeBufferWithCopyOfBufferCall(IGF, metadata,
destBuffer, srcBuffer);
}
/// Emit a call to do an 'initializeBufferWithCopyOfBuffer' operation.
llvm::Value *
irgen::emitInitializeBufferWithCopyOfBufferCall(IRGenFunction &IGF,
llvm::Value *metadata,
Address destBuffer,
Address srcBuffer) {
auto copyFn = emitLoadOfValueWitnessFunctionFromMetadata(IGF, metadata,
ValueWitness::InitializeBufferWithCopyOfBuffer);
llvm::CallInst *call =
IGF.Builder.CreateCall(copyFn,
{destBuffer.getAddress(), srcBuffer.getAddress(), metadata});
return call;
}
/// Emit a dynamic alloca call to allocate enough memory to hold an object of
/// type 'T' and an optional llvm.stackrestore point if 'isInEntryBlock' is
/// false.
StackAddress IRGenFunction::emitDynamicAlloca(SILType T,
const llvm::Twine &name) {
llvm::Value *size = emitLoadOfSize(*this, T);
return emitDynamicAlloca(IGM.Int8Ty, size, Alignment(16), name);
}
StackAddress IRGenFunction::emitDynamicAlloca(llvm::Type *eltTy,
llvm::Value *arraySize,
Alignment align,
const llvm::Twine &name) {
// In coroutines, call llvm.coro.alloca.alloc.
if (isCoroutine()) {
// Compute the number of bytes to allocate.
llvm::Value *byteCount;
auto eltSize = IGM.DataLayout.getTypeAllocSize(eltTy);
if (eltSize == 1) {
byteCount = arraySize;
} else {
byteCount = Builder.CreateMul(arraySize, IGM.getSize(Size(eltSize)));
}
auto alignment = llvm::ConstantInt::get(IGM.Int32Ty, align.getValue());
// Allocate memory. This produces an abstract token.
auto allocFn = llvm::Intrinsic::getDeclaration(
&IGM.Module, llvm::Intrinsic::ID::coro_alloca_alloc, { IGM.SizeTy });
auto allocToken = Builder.CreateCall(allocFn, { byteCount, alignment });
// Get the allocation result.
auto getFn = llvm::Intrinsic::getDeclaration(
&IGM.Module, llvm::Intrinsic::ID::coro_alloca_get);
auto ptr = Builder.CreateCall(getFn, { allocToken });
return {Address(ptr, align), allocToken};
}
// Otherwise, use a dynamic alloca.
llvm::Value *stackRestorePoint = nullptr;
// Save the stack pointer if we are not in the entry block (we could be
// executed more than once).
bool isInEntryBlock = (Builder.GetInsertBlock() == &*CurFn->begin());
if (!isInEntryBlock) {
auto *stackSaveFn = llvm::Intrinsic::getDeclaration(
&IGM.Module, llvm::Intrinsic::ID::stacksave);
stackRestorePoint = Builder.CreateCall(stackSaveFn, {}, "spsave");
}
// Emit the dynamic alloca.
auto *alloca = Builder.IRBuilderBase::CreateAlloca(eltTy, arraySize, name);
alloca->setAlignment(align.getValue());
assert(!isInEntryBlock ||
getActiveDominancePoint().isUniversal() &&
"Must be in entry block if we insert dynamic alloca's without "
"stackrestores");
return {Address(alloca, align), stackRestorePoint};
}
/// Deallocate dynamic alloca's memory if requested by restoring the stack
/// location before the dynamic alloca's call.
void IRGenFunction::emitDeallocateDynamicAlloca(StackAddress address) {
// In coroutines, unconditionally call llvm.coro.alloca.free.
// Except if the address is invalid, this happens when this is a StackAddress
// for a partial_apply [stack] that did not need a context object on the
// stack.
if (isCoroutine() && address.getAddress().isValid()) {
auto allocToken = address.getExtraInfo();
assert(allocToken && "dynamic alloca in coroutine without alloc token?");
auto freeFn = llvm::Intrinsic::getDeclaration(
&IGM.Module, llvm::Intrinsic::ID::coro_alloca_free);
Builder.CreateCall(freeFn, allocToken);
return;
}
// Otherwise, call llvm.stackrestore if an address was saved.
auto savedSP = address.getExtraInfo();
if (savedSP == nullptr)
return;
auto *stackRestoreFn = llvm::Intrinsic::getDeclaration(
&IGM.Module, llvm::Intrinsic::ID::stackrestore);
Builder.CreateCall(stackRestoreFn, savedSP);
}
/// Emit a call to do an 'initializeArrayWithCopy' operation.
void irgen::emitInitializeArrayWithCopyCall(IRGenFunction &IGF,
SILType T,
Address destObject,
Address srcObject,
llvm::Value *count) {
auto metadata = IGF.emitTypeMetadataRefForLayout(T);
auto dest = emitCastToOpaquePtr(IGF, destObject);
auto src = emitCastToOpaquePtr(IGF, srcObject);
IGF.Builder.CreateCall(IGF.IGM.getArrayInitWithCopyFn(),
{dest, src, count, metadata});
}
/// Emit a call to do an 'initializeArrayWithTakeNoAlias' operation.
void irgen::emitInitializeArrayWithTakeNoAliasCall(IRGenFunction &IGF,
SILType T,
Address destObject,
Address srcObject,
llvm::Value *count) {
auto metadata = IGF.emitTypeMetadataRefForLayout(T);
auto dest = emitCastToOpaquePtr(IGF, destObject);
auto src = emitCastToOpaquePtr(IGF, srcObject);
IGF.Builder.CreateCall(IGF.IGM.getArrayInitWithTakeNoAliasFn(),
{dest, src, count, metadata});
}
/// Emit a call to do an 'initializeArrayWithTakeFrontToBack' operation.
void irgen::emitInitializeArrayWithTakeFrontToBackCall(IRGenFunction &IGF,
SILType T,
Address destObject,
Address srcObject,
llvm::Value *count) {
auto metadata = IGF.emitTypeMetadataRefForLayout(T);
auto dest = emitCastToOpaquePtr(IGF, destObject);
auto src = emitCastToOpaquePtr(IGF, srcObject);
IGF.Builder.CreateCall(IGF.IGM.getArrayInitWithTakeFrontToBackFn(),
{dest, src, count, metadata});
}
/// Emit a call to do an 'initializeArrayWithTakeBackToFront' operation.
void irgen::emitInitializeArrayWithTakeBackToFrontCall(IRGenFunction &IGF,
SILType T,
Address destObject,
Address srcObject,
llvm::Value *count) {
auto metadata = IGF.emitTypeMetadataRefForLayout(T);
auto dest = emitCastToOpaquePtr(IGF, destObject);
auto src = emitCastToOpaquePtr(IGF, srcObject);
IGF.Builder.CreateCall(IGF.IGM.getArrayInitWithTakeBackToFrontFn(),
{dest, src, count, metadata});
}
/// Emit a call to do an 'assignWithCopy' operation.
void irgen::emitAssignWithCopyCall(IRGenFunction &IGF,
SILType T,
Address destObject,
Address srcObject) {
llvm::Value *metadata;
auto copyFn = IGF.emitValueWitnessFunctionRef(T, metadata,
ValueWitness::AssignWithCopy);
auto dest = emitCastToOpaquePtr(IGF, destObject);
auto src = emitCastToOpaquePtr(IGF, srcObject);
IGF.Builder.CreateCall(copyFn, {dest, src, metadata});
}
/// Emit a call to do an 'assignWithCopy' operation.
void irgen::emitAssignWithCopyCall(IRGenFunction &IGF,
llvm::Value *metadata,
Address destObject,
Address srcObject) {
auto copyFn = emitLoadOfValueWitnessFunctionFromMetadata(IGF, metadata,
ValueWitness::AssignWithCopy);
auto dest = emitCastToOpaquePtr(IGF, destObject);
auto src = emitCastToOpaquePtr(IGF, srcObject);
IGF.Builder.CreateCall(copyFn, {dest, src, metadata});
}
/// Emit a call to do an 'arrayAssignWithCopyNoAlias' operation.
void irgen::emitAssignArrayWithCopyNoAliasCall(IRGenFunction &IGF, SILType T,
Address destObject, Address srcObject,
llvm::Value *count) {
auto metadata = IGF.emitTypeMetadataRefForLayout(T);
auto dest = emitCastToOpaquePtr(IGF, destObject);
auto src = emitCastToOpaquePtr(IGF, srcObject);
IGF.Builder.CreateCall(IGF.IGM.getArrayAssignWithCopyNoAliasFn(),
{dest, src, count, metadata});
}
/// Emit a call to do an 'arrayAssignWithCopyFrontToBack' operation.
void irgen::emitAssignArrayWithCopyFrontToBackCall(IRGenFunction &IGF,
SILType T,
Address destObject,
Address srcObject,
llvm::Value *count) {
auto metadata = IGF.emitTypeMetadataRefForLayout(T);
auto dest = emitCastToOpaquePtr(IGF, destObject);
auto src = emitCastToOpaquePtr(IGF, srcObject);
IGF.Builder.CreateCall(IGF.IGM.getArrayAssignWithCopyFrontToBackFn(),
{dest, src, count, metadata});
}
/// Emit a call to do an 'arrayAssignWithCopyBackToFront' operation.
void irgen::emitAssignArrayWithCopyBackToFrontCall(IRGenFunction &IGF,
SILType T,
Address destObject,
Address srcObject,
llvm::Value *count) {
auto metadata = IGF.emitTypeMetadataRefForLayout(T);
auto dest = emitCastToOpaquePtr(IGF, destObject);
auto src = emitCastToOpaquePtr(IGF, srcObject);
IGF.Builder.CreateCall(IGF.IGM.getArrayAssignWithCopyBackToFrontFn(),
{dest, src, count, metadata});
}
/// Emit a call to do an 'assignWithTake' operation.
void irgen::emitAssignWithTakeCall(IRGenFunction &IGF,
SILType T,
Address destObject,
Address srcObject) {
llvm::Value *metadata;
auto copyFn = IGF.emitValueWitnessFunctionRef(T, metadata,
ValueWitness::AssignWithTake);
auto dest = emitCastToOpaquePtr(IGF, destObject);
auto src = emitCastToOpaquePtr(IGF, srcObject);
IGF.Builder.CreateCall(copyFn, {dest, src, metadata});
}
/// Emit a call to do an 'arrayAssignWithTake' operation.
void irgen::emitAssignArrayWithTakeCall(IRGenFunction &IGF, SILType T,
Address destObject, Address srcObject,
llvm::Value *count) {
auto metadata = IGF.emitTypeMetadataRefForLayout(T);
auto dest = emitCastToOpaquePtr(IGF, destObject);
auto src = emitCastToOpaquePtr(IGF, srcObject);
IGF.Builder.CreateCall(IGF.IGM.getArrayAssignWithTakeFn(),
{dest, src, count, metadata});
}
/// Emit a call to do a 'destroyArray' operation.
void irgen::emitDestroyArrayCall(IRGenFunction &IGF,
SILType T,
Address object,
llvm::Value *count) {
// If T is a trivial/POD type, nothing needs to be done.
if (IGF.IGM.getTypeLowering(T).isTrivial())
return;
auto metadata = IGF.emitTypeMetadataRefForLayout(T);
auto obj = emitCastToOpaquePtr(IGF, object);
IGF.Builder.CreateCall(IGF.IGM.getArrayDestroyFn(), {obj, count, metadata});
}
/// Emit a trampoline to call the getEnumTagSinglePayload witness. API:
/// UINT_TYPE (const T* enum, UINT_TYPE emptyCases, M *self)
static llvm::Constant *
getGetEnumTagSinglePayloadTrampolineFn(IRGenModule &IGM) {
llvm::Type *argTys[] = {IGM.OpaquePtrTy, IGM.Int32Ty, IGM.TypeMetadataPtrTy};
llvm::SmallString<40> fnName("__swift_getEnumTagSinglePayload");
auto func = IGM.getOrCreateHelperFunction(
fnName, IGM.Int32Ty, argTys,
[&](IRGenFunction &IGF) {
auto it = IGF.CurFn->arg_begin();
auto *enumAddr = &*(it++);
auto *numEmptyCases = &*(it++);
auto *metadata = &*(it++);
auto &Builder = IGF.Builder;
auto witnessFunc = emitLoadOfValueWitnessFunctionFromMetadata(
IGF, metadata, ValueWitness::GetEnumTagSinglePayload);
auto *result = Builder.CreateCall(witnessFunc,
{enumAddr, numEmptyCases, metadata});
Builder.CreateRet(result);
},
true /*noinline*/);
// This function is readonly.
cast<llvm::Function>(func)->addFnAttr(llvm::Attribute::ReadOnly);
return func;
}
/// Emit a trampoline to call the storeEnumTagSinglePayload witness. API:
/// VOID_TYPE (const T* enum, UINT_TYPE whichCase, UINT_TYPE emptyCases,
/// M *self)
static llvm::Constant *
getStoreEnumTagSinglePayloadTrampolineFn(IRGenModule &IGM) {
llvm::Type *argTys[] = {IGM.OpaquePtrTy, IGM.Int32Ty, IGM.Int32Ty,
IGM.TypeMetadataPtrTy};
llvm::SmallString<40> fnName("__swift_storeEnumTagSinglePayload");
return IGM.getOrCreateHelperFunction(
fnName, IGM.VoidTy, argTys,
[&](IRGenFunction &IGF) {
auto it = IGF.CurFn->arg_begin();
auto *enumAddr = &*(it++);
auto *whichCase = &*(it++);
auto *numEmptyCases = &*(it++);
auto *metadata = &*(it++);
auto &Builder = IGF.Builder;
auto witnessFunc = emitLoadOfValueWitnessFunctionFromMetadata(
IGF, metadata, ValueWitness::StoreEnumTagSinglePayload);
Builder.CreateCall(witnessFunc,
{enumAddr, whichCase, numEmptyCases, metadata});
Builder.CreateRetVoid();
},
true /*noinline*/);
}
llvm::Value *irgen::emitGetEnumTagSinglePayloadCall(IRGenFunction &IGF,
SILType T,
llvm::Value *numEmptyCases,
Address destObject) {
if (!IGF.optimizeForSize()) {
llvm::Value *metadata;
auto fn = IGF.emitValueWitnessFunctionRef(
T, metadata, ValueWitness::GetEnumTagSinglePayload);
auto dest = emitCastToOpaquePtr(IGF, destObject);
llvm::CallInst *call = IGF.Builder.CreateCall(
fn, {dest, numEmptyCases, metadata});
return call;
}
auto *metadata = IGF.emitTypeMetadataRefForLayout(T);
auto *func = getGetEnumTagSinglePayloadTrampolineFn(IGF.IGM);
auto dest = emitCastToOpaquePtr(IGF, destObject);
auto *result = IGF.Builder.CreateCall(func, {dest, numEmptyCases, metadata});
return result;
}
void irgen::emitStoreEnumTagSinglePayloadCall(
IRGenFunction &IGF, SILType T, llvm::Value *whichCase,
llvm::Value *numEmptyCases, Address destObject) {
if (!IGF.optimizeForSize()) {
llvm::Value *metadata;
auto fn = IGF.emitValueWitnessFunctionRef(
T, metadata, ValueWitness::StoreEnumTagSinglePayload);
auto dest = emitCastToOpaquePtr(IGF, destObject);
IGF.Builder.CreateCall(fn, {dest, whichCase, numEmptyCases, metadata});
return;
}
auto *metadata = IGF.emitTypeMetadataRefForLayout(T);
auto *func = getStoreEnumTagSinglePayloadTrampolineFn(IGF.IGM);
auto dest = emitCastToOpaquePtr(IGF, destObject);
IGF.Builder.CreateCall(func, {dest, whichCase, numEmptyCases, metadata});
}
/// Emit a call to the 'getEnumTag' operation.
llvm::Value *irgen::emitGetEnumTagCall(IRGenFunction &IGF,
SILType T,
Address srcObject) {
llvm::Value *metadata;
auto fn = IGF.emitValueWitnessFunctionRef(T, metadata,
ValueWitness::GetEnumTag);
auto src = emitCastToOpaquePtr(IGF, srcObject);
llvm::CallInst *call =
IGF.Builder.CreateCall(fn, {src, metadata});
return call;
}
/// Emit a call to the 'destructiveProjectEnumData' operation.
/// The type must be dynamically known to have enum witnesses.
void irgen::emitDestructiveProjectEnumDataCall(IRGenFunction &IGF,
SILType T,
Address srcObject) {
llvm::Value *metadata;
auto fn = IGF.emitValueWitnessFunctionRef(T, metadata,
ValueWitness::DestructiveProjectEnumData);
auto src = emitCastToOpaquePtr(IGF, srcObject);
IGF.Builder.CreateCall(fn, {src, metadata});
}
/// Emit a call to the 'destructiveInjectEnumTag' operation.
/// The type must be dynamically known to have enum witnesses.
void irgen::emitDestructiveInjectEnumTagCall(IRGenFunction &IGF,
SILType T,
llvm::Value *tagValue,
Address srcObject) {
llvm::Value *metadata;
auto fn = IGF.emitValueWitnessFunctionRef(T, metadata,
ValueWitness::DestructiveInjectEnumTag);
auto src = emitCastToOpaquePtr(IGF, srcObject);
IGF.Builder.CreateCall(fn, {src, tagValue, metadata});
}
/// Load the 'size' value witness from the given table as a size_t.
llvm::Value *irgen::emitLoadOfSize(IRGenFunction &IGF, SILType T) {
return IGF.emitValueWitnessValue(T, ValueWitness::Size);
}
/// Load the 'alignmentMask' value witness from the given table as a size_t.
llvm::Value *irgen::emitLoadOfAlignmentMask(IRGenFunction &IGF, SILType T) {
auto flags = IGF.emitValueWitnessValue(T, ValueWitness::Flags);
return emitAlignMaskFromFlags(IGF, flags);
}
/// Load the 'isPOD' valueWitness from the given table as an i1.
llvm::Value *irgen::emitLoadOfIsPOD(IRGenFunction &IGF, SILType T) {
auto flags = IGF.emitValueWitnessValue(T, ValueWitness::Flags);
auto mask = IGF.IGM.getInt32(ValueWitnessFlags::IsNonPOD);
auto masked = IGF.Builder.CreateAnd(flags, mask);
return IGF.Builder.CreateICmpEQ(masked, IGF.IGM.getInt32(0),
flags->getName() + ".isPOD");
}
/// Load the 'isBitwiseTakable' valueWitness from the given table as an i1.
llvm::Value *irgen::emitLoadOfIsBitwiseTakable(IRGenFunction &IGF, SILType T) {
auto flags = IGF.emitValueWitnessValue(T, ValueWitness::Flags);
auto mask = IGF.IGM.getInt32(ValueWitnessFlags::IsNonBitwiseTakable);
auto masked = IGF.Builder.CreateAnd(flags, mask);
return IGF.Builder.CreateICmpEQ(masked, IGF.IGM.getInt32(0),
flags->getName() + ".isBitwiseTakable");
}
/// Load the 'isInline' valueWitness from the given table as an i1.
llvm::Value *irgen::emitLoadOfIsInline(IRGenFunction &IGF, SILType T) {
auto flags = IGF.emitValueWitnessValue(T, ValueWitness::Flags);
auto mask = IGF.IGM.getInt32(ValueWitnessFlags::IsNonInline);
auto masked = IGF.Builder.CreateAnd(flags, mask);
return IGF.Builder.CreateICmpEQ(masked, IGF.IGM.getInt32(0),
flags->getName() + ".isInline");
}
/// Load the 'stride' value witness from the given table as a size_t.
llvm::Value *irgen::emitLoadOfStride(IRGenFunction &IGF, SILType T) {
return IGF.emitValueWitnessValue(T, ValueWitness::Stride);
}
llvm::Value *irgen::emitLoadOfExtraInhabitantCount(IRGenFunction &IGF,
SILType T) {
return IGF.emitValueWitnessValue(T, ValueWitness::ExtraInhabitantCount);
}
std::pair<llvm::Value *, llvm::Value *>
irgen::emitLoadOfIsInline(IRGenFunction &IGF, llvm::Value *metadata) {
auto *flags = emitLoadOfValueWitnessValueFromMetadata(IGF, metadata,
ValueWitness::Flags);
auto mask = IGF.IGM.getInt32(ValueWitnessFlags::IsNonInline);
auto masked = IGF.Builder.CreateAnd(flags, mask);
return std::make_pair(
IGF.Builder.CreateICmpEQ(masked, IGF.IGM.getInt32(0),
flags->getName() + ".isInline"),
flags);
}
llvm::Value *irgen::emitLoadOfSize(IRGenFunction &IGF, llvm::Value *metadata) {
auto *size = emitLoadOfValueWitnessValueFromMetadata(IGF, metadata,
ValueWitness::Size);
return size;
}
llvm::Value *irgen::emitAlignMaskFromFlags(IRGenFunction &IGF,
llvm::Value *flags) {
auto flagsAsSize = IGF.Builder.CreateZExtOrTrunc(flags, IGF.IGM.SizeTy);
auto *alignMask = IGF.IGM.getSize(Size(ValueWitnessFlags::AlignmentMask));
return IGF.Builder.CreateAnd(flagsAsSize, alignMask,
flags->getName() + ".alignmentMask");
}
/// Emit a call to do an 'initializeWithCopy' operation.
void irgen::emitInitializeWithCopyCall(IRGenFunction &IGF,
SILType T,
Address dest,
Address src) {
llvm::Value *metadata;
auto fn = IGF.emitValueWitnessFunctionRef(T, metadata,
ValueWitness::InitializeWithCopy);
auto destPtr = emitCastToOpaquePtr(IGF, dest);
auto srcPtr = emitCastToOpaquePtr(IGF, src);
IGF.Builder.CreateCall(fn, {destPtr, srcPtr, metadata});
}
llvm::Value *irgen::emitInitializeWithCopyCall(IRGenFunction &IGF,
llvm::Value *metadata,
Address dest, Address src) {
auto copyFn = emitLoadOfValueWitnessFunctionFromMetadata(
IGF, metadata, ValueWitness::InitializeWithCopy);
auto destPtr = emitCastToOpaquePtr(IGF, dest);
auto srcPtr = emitCastToOpaquePtr(IGF, src);
llvm::CallInst *call = IGF.Builder.CreateCall(
copyFn, {destPtr, srcPtr, metadata});
return call;
}
/// Emit a call to do an 'initializeWithTake' operation.
void irgen::emitInitializeWithTakeCall(IRGenFunction &IGF,
SILType T,
Address dest,
Address src) {
llvm::Value *metadata;
auto fn = IGF.emitValueWitnessFunctionRef(T, metadata,
ValueWitness::InitializeWithTake);
auto destPtr = emitCastToOpaquePtr(IGF, dest);
auto srcPtr = emitCastToOpaquePtr(IGF, src);
IGF.Builder.CreateCall(fn, {destPtr, srcPtr, metadata});
}
llvm::Value *irgen::emitInitializeWithTakeCall(IRGenFunction &IGF,
llvm::Value *metadata,
Address dest, Address src) {
auto copyFn = emitLoadOfValueWitnessFunctionFromMetadata(
IGF, metadata, ValueWitness::InitializeWithTake);
auto destPtr = emitCastToOpaquePtr(IGF, dest);
auto srcPtr = emitCastToOpaquePtr(IGF, src);
llvm::CallInst *call =
IGF.Builder.CreateCall(copyFn, {destPtr, srcPtr, metadata});
return call;
}
/// Emit a call to do a 'destroy' operation.
void irgen::emitDestroyCall(IRGenFunction &IGF,
SILType T,
Address object) {
// If T is a trivial/POD type, nothing needs to be done.
if (IGF.IGM.getTypeLowering(T).isTrivial())
return;
llvm::Value *metadata;
auto fn = IGF.emitValueWitnessFunctionRef(T, metadata,
ValueWitness::Destroy);
auto objectPtr = emitCastToOpaquePtr(IGF, object);
IGF.Builder.CreateCall(fn, {objectPtr, metadata});
}
void irgen::emitDestroyCall(IRGenFunction &IGF, llvm::Value *metadata,
Address object) {
auto fn = emitLoadOfValueWitnessFunctionFromMetadata(IGF, metadata,
ValueWitness::Destroy);
auto objectPtr = emitCastToOpaquePtr(IGF, object);
IGF.Builder.CreateCall(fn, {objectPtr, metadata});
}
static llvm::Constant *getAllocateValueBufferFunction(IRGenModule &IGM) {
llvm::Type *argTys[] = {IGM.TypeMetadataPtrTy, IGM.OpaquePtrTy};
llvm::SmallString<40> fnName("__swift_allocate_value_buffer");
return IGM.getOrCreateHelperFunction(
fnName, IGM.OpaquePtrTy, argTys,
[&](IRGenFunction &IGF) {
auto it = IGF.CurFn->arg_begin();
auto *metadata = &*(it++);
auto buffer = Address(&*(it++), Alignment(1));
// Dynamically check whether this type is inline or needs an allocation.
llvm::Value *isInline, *flags;
std::tie(isInline, flags) = emitLoadOfIsInline(IGF, metadata);
auto *outlineBB = IGF.createBasicBlock("outline.allocateValueInBuffer");
auto *doneBB = IGF.createBasicBlock("done");
llvm::Value *addressInline, *addressOutline;
addressInline = buffer.getAddress();
auto *origBB = IGF.Builder.GetInsertBlock();
IGF.Builder.CreateCondBr(isInline, doneBB, outlineBB);
IGF.Builder.emitBlock(outlineBB);
{
auto *size = emitLoadOfSize(IGF, metadata);
auto *alignMask = emitAlignMaskFromFlags(IGF, flags);
auto valueAddr =
IGF.emitAllocRawCall(size, alignMask, "outline.ValueBuffer");
IGF.Builder.CreateStore(
valueAddr, Address(IGF.Builder.CreateBitCast(
buffer.getAddress(),
valueAddr->getType()->getPointerTo()),
Alignment(1)));
addressOutline =
IGF.Builder.CreateBitCast(valueAddr, IGM.OpaquePtrTy);
IGF.Builder.CreateBr(doneBB);
}
IGF.Builder.emitBlock(doneBB);
auto *addressOfValue = IGF.Builder.CreatePHI(IGM.OpaquePtrTy, 2);
addressOfValue->addIncoming(addressInline, origBB);
addressOfValue->addIncoming(addressOutline, outlineBB);
IGF.Builder.CreateRet(addressOfValue);
},
true /*noinline*/);
}
Address irgen::emitAllocateValueInBuffer(IRGenFunction &IGF, SILType type,
Address buffer) {
// Handle FixedSize types.
auto &IGM = IGF.IGM;
auto storagePtrTy = IGM.getStoragePointerType(type);
auto &Builder = IGF.Builder;
if (auto *fixedTI = dyn_cast<FixedTypeInfo>(&IGF.getTypeInfo(type))) {
auto packing = fixedTI->getFixedPacking(IGM);
// Inline representation.
if (packing == FixedPacking::OffsetZero) {
return Address(Builder.CreateBitCast(buffer.getAddress(), storagePtrTy),
buffer.getAlignment());
}
// Outline representation.
assert(packing == FixedPacking::Allocate && "Expect non dynamic packing");
auto size = fixedTI->getStaticSize(IGM);
auto alignMask = fixedTI->getStaticAlignmentMask(IGM);
auto valueAddr =
IGF.emitAllocRawCall(size, alignMask, "outline.ValueBuffer");
Builder.CreateStore(
valueAddr,
Address(Builder.CreateBitCast(buffer.getAddress(),
valueAddr->getType()->getPointerTo()),
buffer.getAlignment()));
return Address(Builder.CreateBitCast(valueAddr, storagePtrTy),
buffer.getAlignment());
}
// Dynamic packing.
/// Call a function to handle the non-fixed case.
auto *allocateFun = getAllocateValueBufferFunction(IGF.IGM);
auto *metadata = IGF.emitTypeMetadataRefForLayout(type);
auto *call = Builder.CreateCall(
allocateFun,
{metadata, Builder.CreateBitCast(buffer.getAddress(), IGM.OpaquePtrTy)});
call->setCallingConv(IGF.IGM.DefaultCC);
call->setDoesNotThrow();
auto addressOfValue = Builder.CreateBitCast(call, storagePtrTy);
return Address(addressOfValue, Alignment(1));
}
static llvm::Constant *getProjectValueInBufferFunction(IRGenModule &IGM) {
llvm::Type *argTys[] = {IGM.TypeMetadataPtrTy, IGM.OpaquePtrTy};
llvm::SmallString<40> fnName("__swift_project_value_buffer");
return IGM.getOrCreateHelperFunction(
fnName, IGM.OpaquePtrTy, argTys,
[&](IRGenFunction &IGF) {
auto it = IGF.CurFn->arg_begin();
auto *metadata = &*(it++);
auto buffer = Address(&*(it++), Alignment(1));
auto &Builder = IGF.Builder;
// Dynamically check whether this type is inline or needs an allocation.
llvm::Value *isInline, *flags;
std::tie(isInline, flags) = emitLoadOfIsInline(IGF, metadata);
auto *outlineBB = IGF.createBasicBlock("outline.projectValueInBuffer");
auto *doneBB = IGF.createBasicBlock("done");
llvm::Value *addressInline, *addressOutline;
auto *origBB = Builder.GetInsertBlock();
addressInline = buffer.getAddress();
Builder.CreateCondBr(isInline, doneBB, outlineBB);
Builder.emitBlock(outlineBB);
{
addressOutline = Builder.CreateLoad(
Address(Builder.CreateBitCast(buffer.getAddress(),
IGM.OpaquePtrTy->getPointerTo()),
Alignment(1)));
Builder.CreateBr(doneBB);
}
Builder.emitBlock(doneBB);
auto *addressOfValue = Builder.CreatePHI(IGM.OpaquePtrTy, 2);
addressOfValue->addIncoming(addressInline, origBB);
addressOfValue->addIncoming(addressOutline, outlineBB);
Builder.CreateRet(addressOfValue);
},
true /*noinline*/);
}
Address irgen::emitProjectValueInBuffer(IRGenFunction &IGF, SILType type,
Address buffer) {
// Handle FixedSize types.
auto &IGM = IGF.IGM;
auto storagePtrTy = IGM.getStoragePointerType(type);
auto &Builder = IGF.Builder;
if (auto *fixedTI = dyn_cast<FixedTypeInfo>(&IGF.getTypeInfo(type))) {
auto packing = fixedTI->getFixedPacking(IGM);
// Inline representation.
if (packing == FixedPacking::OffsetZero) {
return Address(Builder.CreateBitCast(buffer.getAddress(), storagePtrTy),
buffer.getAlignment());
}
// Outline representation.
assert(packing == FixedPacking::Allocate && "Expect non dynamic packing");
auto valueAddr = Builder.CreateLoad(
Address(Builder.CreateBitCast(buffer.getAddress(),
storagePtrTy->getPointerTo()),
buffer.getAlignment()));
return Address(Builder.CreateBitCast(valueAddr, storagePtrTy),
buffer.getAlignment());
}
// Dynamic packing.
auto *projectFun = getProjectValueInBufferFunction(IGF.IGM);
auto *metadata = IGF.emitTypeMetadataRefForLayout(type);
auto *call = Builder.CreateCall(
projectFun,
{metadata, Builder.CreateBitCast(buffer.getAddress(), IGM.OpaquePtrTy)});
call->setCallingConv(IGF.IGM.DefaultCC);
call->setDoesNotThrow();
auto addressOfValue = Builder.CreateBitCast(call, storagePtrTy);
return Address(addressOfValue, Alignment(1));
}
static llvm::Constant *getDeallocateValueInBufferFunction(IRGenModule &IGM) {
llvm::Type *argTys[] = {IGM.TypeMetadataPtrTy, IGM.OpaquePtrTy};
llvm::SmallString<40> fnName("__swift_deallocate_value_buffer");
return IGM.getOrCreateHelperFunction(
fnName, IGM.VoidTy, argTys,
[&](IRGenFunction &IGF) {
auto it = IGF.CurFn->arg_begin();
auto *metadata = &*(it++);
auto buffer = Address(&*(it++), Alignment(1));
auto &Builder = IGF.Builder;
// Dynamically check whether this type is inline or needs an allocation.
llvm::Value *isInline, *flags;
std::tie(isInline, flags) = emitLoadOfIsInline(IGF, metadata);
auto *outlineBB = IGF.createBasicBlock("outline.deallocateValueInBuffer");
auto *doneBB = IGF.createBasicBlock("done");
Builder.CreateCondBr(isInline, doneBB, outlineBB);
Builder.emitBlock(outlineBB);
{
auto *size = emitLoadOfSize(IGF, metadata);
auto *alignMask = emitAlignMaskFromFlags(IGF, flags);
auto *ptr = Builder.CreateLoad(Address(
Builder.CreateBitCast(buffer.getAddress(), IGM.Int8PtrPtrTy),
buffer.getAlignment()));
IGF.emitDeallocRawCall(ptr, size, alignMask);
Builder.CreateBr(doneBB);
}
Builder.emitBlock(doneBB);
Builder.CreateRetVoid();
},
true /*noinline*/);
}
void irgen::emitDeallocateValueInBuffer(IRGenFunction &IGF,
SILType type,
Address buffer) {
// Handle FixedSize types.
auto &IGM = IGF.IGM;
auto &Builder = IGF.Builder;
if (auto *fixedTI = dyn_cast<FixedTypeInfo>(&IGF.getTypeInfo(type))) {
auto packing = fixedTI->getFixedPacking(IGM);
// Inline representation.
if (packing == FixedPacking::OffsetZero)
return;
// Outline representation.
assert(packing == FixedPacking::Allocate && "Expect non dynamic packing");
auto size = fixedTI->getStaticSize(IGM);
auto alignMask = fixedTI->getStaticAlignmentMask(IGM);
auto *ptr = Builder.CreateLoad(Address(
Builder.CreateBitCast(buffer.getAddress(), IGM.Int8PtrPtrTy),
buffer.getAlignment()));
IGF.emitDeallocRawCall(ptr, size, alignMask);
return;
}
// Dynamic packing.
auto *projectFun = getDeallocateValueInBufferFunction(IGF.IGM);
auto *metadata = IGF.emitTypeMetadataRefForLayout(type);
auto *call = Builder.CreateCall(
projectFun,
{metadata, Builder.CreateBitCast(buffer.getAddress(), IGM.OpaquePtrTy)});
call->setCallingConv(IGF.IGM.DefaultCC);
call->setDoesNotThrow();
}
llvm::Value *
irgen::emitGetEnumTagSinglePayloadGenericCall(IRGenFunction &IGF,
SILType payloadType,
const TypeInfo &payloadTI,
llvm::Value *numExtraCases,
Address address,
GetExtraInhabitantTagEmitter emitter) {
auto getExtraInhabitantTagFn =
getOrCreateGetExtraInhabitantTagFunction(IGF.IGM, payloadType,
payloadTI, emitter);
// We assume this is never a reabstracted type.
auto type = payloadType.getASTType();
assert(type->isLegalFormalType());
auto metadata = IGF.emitTypeMetadataRef(type);
auto ptr = IGF.Builder.CreateBitCast(address.getAddress(),
IGF.IGM.OpaquePtrTy);
auto getEnumTagGenericFn =
IGF.IGM.getGetEnumTagSinglePayloadGenericFn();
auto call = IGF.Builder.CreateCall(getEnumTagGenericFn,
{ptr,
numExtraCases,
metadata,
getExtraInhabitantTagFn});
call->setCallingConv(IGF.IGM.SwiftCC);
return call;
}
llvm::Constant *
irgen::getOrCreateGetExtraInhabitantTagFunction(IRGenModule &IGM,
SILType objectType,
const TypeInfo &objectTI,
GetExtraInhabitantTagEmitter emitter) {
// We assume this is never a reabstracted type.
CanType type = objectType.getASTType();
assert(type->isLegalFormalType());
auto fnTy = llvm::FunctionType::get(IGM.Int32Ty,
{IGM.OpaquePtrTy,
IGM.Int32Ty,
IGM.TypeMetadataPtrTy},
false);
// TODO: use a meaningful mangled name and internal/shared linkage.
auto fn = llvm::Function::Create(fnTy, llvm::Function::PrivateLinkage,
"__swift_get_extra_inhabitant_index",
&IGM.Module);
fn->setCallingConv(IGM.SwiftCC);
IRGenFunction IGF(IGM, fn);
auto parameters = IGF.collectParameters();
auto ptr = parameters.claimNext();
auto xiCount = parameters.claimNext();
auto metadata = parameters.claimNext();
// Bind the metadata to make any archetypes available.
IGF.bindLocalTypeDataFromTypeMetadata(type, IsExact, metadata,
MetadataState::Complete);
// Form a well-typed address from the opaque pointer.
ptr = IGF.Builder.CreateBitCast(ptr,
objectTI.getStorageType()->getPointerTo());
Address addr = objectTI.getAddressForPointer(ptr);
auto tag = emitter(IGF, addr, xiCount);
IGF.Builder.CreateRet(tag);
return fn;
}
void
irgen::emitStoreEnumTagSinglePayloadGenericCall(IRGenFunction &IGF,
SILType payloadType,
const TypeInfo &payloadTI,
llvm::Value *whichCase,
llvm::Value *numExtraCases,
Address address,
StoreExtraInhabitantTagEmitter emitter) {
auto storeExtraInhabitantTagFn =
getOrCreateStoreExtraInhabitantTagFunction(IGF.IGM, payloadType,
payloadTI, emitter);
// We assume this is never a reabstracted type.
auto type = payloadType.getASTType();
assert(type->isLegalFormalType());
auto metadata = IGF.emitTypeMetadataRef(type);
auto ptr = IGF.Builder.CreateBitCast(address.getAddress(),
IGF.IGM.OpaquePtrTy);
auto storeEnumTagGenericFn =
IGF.IGM.getStoreEnumTagSinglePayloadGenericFn();
auto call = IGF.Builder.CreateCall(storeEnumTagGenericFn,
{ptr,
whichCase,
numExtraCases,
metadata,
storeExtraInhabitantTagFn});
call->setCallingConv(IGF.IGM.SwiftCC);
}
llvm::Constant *
irgen::getOrCreateStoreExtraInhabitantTagFunction(IRGenModule &IGM,
SILType objectType,
const TypeInfo &objectTI,
StoreExtraInhabitantTagEmitter emitter) {
// We assume this is never a reabstracted type.
CanType type = objectType.getASTType();
assert(type->isLegalFormalType());
auto fnTy = llvm::FunctionType::get(IGM.VoidTy,
{IGM.OpaquePtrTy,
IGM.Int32Ty,
IGM.Int32Ty,
IGM.TypeMetadataPtrTy},
false);
// TODO: use a meaningful mangled name and internal/shared linkage.
auto fn = llvm::Function::Create(fnTy, llvm::Function::PrivateLinkage,
"__swift_get_extra_inhabitant_index",
&IGM.Module);
fn->setCallingConv(IGM.SwiftCC);
IRGenFunction IGF(IGM, fn);
auto parameters = IGF.collectParameters();
auto ptr = parameters.claimNext();
auto tag = parameters.claimNext();
auto xiCount = parameters.claimNext();
auto metadata = parameters.claimNext();
// Bind the metadata to make any archetypes available.
IGF.bindLocalTypeDataFromTypeMetadata(type, IsExact, metadata,
MetadataState::Complete);
// Form a well-typed address from the opaque pointer.
ptr = IGF.Builder.CreateBitCast(ptr,
objectTI.getStorageType()->getPointerTo());
Address addr = objectTI.getAddressForPointer(ptr);
emitter(IGF, addr, tag, xiCount);
IGF.Builder.CreateRetVoid();
return fn;
}
llvm::Value *TypeInfo::getExtraInhabitantTagDynamic(IRGenFunction &IGF,
Address address,
SILType T,
llvm::Value *knownXICount,
bool isOutlined) const {
if (auto fixedTI = dyn_cast<FixedTypeInfo>(this)) {
auto index = fixedTI->getExtraInhabitantIndex(IGF, address, T, isOutlined);
// The runtime APIs expect that 0 means the payload case and 1+
// means the extra cases, but in IRGen, getExtraInhabitantIndex
// returns -1 for the payload case and 0+ for extra inhabitants.
// This is an easy adjustment to make.
auto one = llvm::ConstantInt::get(IGF.IGM.Int32Ty, 1);
auto tag = IGF.Builder.CreateAdd(index, one);
return tag;
} else {
if (!knownXICount)
knownXICount = emitLoadOfExtraInhabitantCount(IGF, T);
auto tag = getEnumTagSinglePayload(IGF, /*num extra cases*/ knownXICount,
address, T, isOutlined);
return tag;
}
}
void TypeInfo::storeExtraInhabitantTagDynamic(IRGenFunction &IGF,
llvm::Value *tag,
Address address,
SILType T,
bool isOutlined) const {
if (auto fixedTI = dyn_cast<FixedTypeInfo>(this)) {
// The runtime APIs expect that 0 means the payload case and 1+
// means the extra cases, but in IRGen, storeExtraInhabitant
// expects extra inhabitants to be indexed as 0+.
// This is an easy adjustment to make.
auto one = llvm::ConstantInt::get(IGF.IGM.Int32Ty, 1);
auto index = IGF.Builder.CreateSub(tag, one);
fixedTI->storeExtraInhabitant(IGF, index, address, T, isOutlined);
} else {
storeEnumTagSinglePayload(IGF, tag, tag, address, T, isOutlined);
}
}