| //===--- EnumPayload.cpp - Payload management for 'enum' Types ------------===// |
| // |
| // This source file is part of the Swift.org open source project |
| // |
| // Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors |
| // Licensed under Apache License v2.0 with Runtime Library Exception |
| // |
| // See https://swift.org/LICENSE.txt for license information |
| // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "EnumPayload.h" |
| #include "Explosion.h" |
| #include "GenEnum.h" |
| #include "IRGenModule.h" |
| #include <map> |
| |
| using namespace swift; |
| using namespace irgen; |
| |
| // FIXME: Everything here brazenly assumes little-endian-ness. |
| |
| static llvm::Value *forcePayloadValue(EnumPayload::LazyValue &value) { |
| if (auto v = value.dyn_cast<llvm::Value *>()) |
| return v; |
| |
| auto null = llvm::Constant::getNullValue(value.dyn_cast<llvm::Type*>()); |
| value = null; |
| return null; |
| } |
| |
| static llvm::Type *getPayloadType(EnumPayload::LazyValue value) { |
| if (auto t = value.dyn_cast<llvm::Type *>()) |
| return t; |
| |
| return value.dyn_cast<llvm::Value *>()->getType(); |
| } |
| |
| EnumPayload EnumPayload::zero(IRGenModule &IGM, EnumPayloadSchema schema) { |
| // We don't need to create any values yet; they can be filled in when |
| // real values are inserted. |
| EnumPayload result; |
| schema.forEachType(IGM, [&](llvm::Type *type) { |
| result.PayloadValues.push_back(type); |
| }); |
| return result; |
| } |
| |
| EnumPayload EnumPayload::fromBitPattern(IRGenModule &IGM, |
| APInt bitPattern, |
| EnumPayloadSchema schema) { |
| EnumPayload result; |
| |
| schema.forEachType(IGM, [&](llvm::Type *type) { |
| unsigned bitSize = IGM.DataLayout.getTypeSizeInBits(type); |
| |
| llvm::IntegerType *intTy |
| = llvm::IntegerType::get(IGM.getLLVMContext(), bitSize); |
| |
| // Take some bits off of the bottom of the pattern. |
| auto bits = bitPattern.zextOrTrunc(bitSize); |
| auto val = llvm::ConstantInt::get(intTy, bits); |
| if (val->getType() != type) { |
| if (type->isPointerTy()) |
| val = llvm::ConstantExpr::getIntToPtr(val, type); |
| else |
| val = llvm::ConstantExpr::getBitCast(val, type); |
| } |
| |
| result.PayloadValues.push_back(val); |
| |
| // Shift the remaining bits down. |
| bitPattern = bitPattern.lshr(bitSize); |
| }); |
| |
| return result; |
| } |
| |
| // Fn: void(LazyValue &payloadValue, unsigned payloadBitWidth, |
| // unsigned payloadValueOffset, unsigned valueBitWidth, |
| // unsigned valueOffset) |
| template<typename Fn> |
| static void withValueInPayload(IRGenFunction &IGF, |
| const EnumPayload &payload, |
| llvm::Type *valueType, |
| int numBitsUsedInValue, |
| unsigned payloadOffset, |
| Fn &&f) { |
| auto &DataLayout = IGF.IGM.DataLayout; |
| int valueTypeBitWidth = DataLayout.getTypeSizeInBits(valueType); |
| int valueBitWidth = |
| numBitsUsedInValue < 0 ? valueTypeBitWidth : numBitsUsedInValue; |
| assert(numBitsUsedInValue <= valueTypeBitWidth); |
| |
| // Find the elements we need to touch. |
| // TODO: Linear search through the payload elements is lame. |
| MutableArrayRef<EnumPayload::LazyValue> payloads = payload.PayloadValues; |
| llvm::Type *payloadType; |
| int payloadBitWidth; |
| int valueOffset = 0, payloadValueOffset = payloadOffset; |
| for (;;) { |
| payloadType = getPayloadType(payloads.front()); |
| payloadBitWidth = IGF.IGM.DataLayout.getTypeSizeInBits(payloadType); |
| |
| // Does this element overlap the area we need to touch? |
| if (payloadValueOffset < payloadBitWidth) { |
| // See how much of the value we can fit here. |
| int valueChunkWidth = payloadBitWidth - payloadValueOffset; |
| valueChunkWidth = std::min(valueChunkWidth, valueBitWidth - valueOffset); |
| |
| f(payloads.front(), |
| payloadBitWidth, payloadValueOffset, |
| valueTypeBitWidth, valueOffset); |
| |
| // If we used the entire value, we're done. |
| valueOffset += valueChunkWidth; |
| if (valueOffset >= valueBitWidth) |
| return; |
| } |
| |
| payloadValueOffset = std::max(payloadValueOffset - payloadBitWidth, 0); |
| payloads = payloads.slice(1); |
| } |
| } |
| |
| void EnumPayload::insertValue(IRGenFunction &IGF, llvm::Value *value, |
| unsigned payloadOffset, |
| int numBitsUsedInValue) { |
| withValueInPayload(IGF, *this, value->getType(), numBitsUsedInValue, payloadOffset, |
| [&](LazyValue &payloadValue, |
| unsigned payloadBitWidth, |
| unsigned payloadValueOffset, |
| unsigned valueBitWidth, |
| unsigned valueOffset) { |
| auto payloadType = getPayloadType(payloadValue); |
| // See if the value matches the payload type exactly. In this case we |
| // don't need to do any work to use the value. |
| if (payloadValueOffset == 0 && valueOffset == 0) { |
| if (value->getType() == payloadType) { |
| payloadValue = value; |
| return; |
| } |
| // If only the width matches exactly, we can still do a bitcast. |
| if (payloadBitWidth == valueBitWidth) { |
| auto bitcast = IGF.Builder.CreateBitOrPointerCast(value, payloadType); |
| payloadValue = bitcast; |
| return; |
| } |
| } |
| |
| // Select out the chunk of the value to merge with the existing payload. |
| llvm::Value *subvalue = value; |
| |
| auto valueIntTy = |
| llvm::IntegerType::get(IGF.IGM.getLLVMContext(), valueBitWidth); |
| auto payloadIntTy = |
| llvm::IntegerType::get(IGF.IGM.getLLVMContext(), payloadBitWidth); |
| auto payloadTy = getPayloadType(payloadValue); |
| subvalue = IGF.Builder.CreateBitOrPointerCast(subvalue, valueIntTy); |
| if (valueOffset > 0) |
| subvalue = IGF.Builder.CreateLShr(subvalue, |
| llvm::ConstantInt::get(valueIntTy, valueOffset)); |
| subvalue = IGF.Builder.CreateZExtOrTrunc(subvalue, payloadIntTy); |
| #if defined(__BIG_ENDIAN__) |
| if ((valueBitWidth == 32 || valueBitWidth == 16 || valueBitWidth == 8 || valueBitWidth == 1) && |
| payloadBitWidth > (payloadValueOffset + valueBitWidth)) { |
| unsigned shiftBitWidth = valueBitWidth; |
| if (valueBitWidth == 1) { |
| shiftBitWidth = 8; |
| } |
| subvalue = IGF.Builder.CreateShl(subvalue, |
| llvm::ConstantInt::get(payloadIntTy, (payloadBitWidth - shiftBitWidth) - payloadValueOffset)); |
| } |
| #else |
| if (payloadValueOffset > 0) |
| subvalue = IGF.Builder.CreateShl(subvalue, |
| llvm::ConstantInt::get(payloadIntTy, payloadValueOffset)); |
| #endif |
| |
| // If there hasn't yet been a value stored here, we can use the adjusted |
| // value directly. |
| if (payloadValue.is<llvm::Type *>()) { |
| payloadValue = IGF.Builder.CreateBitOrPointerCast(subvalue, payloadTy); |
| } |
| // Otherwise, bitwise-or it in, brazenly assuming there are zeroes |
| // underneath. |
| else { |
| // TODO: This creates a bunch of bitcasting noise for non-integer |
| // payload fields. |
| auto lastValue = payloadValue.get<llvm::Value *>(); |
| lastValue = IGF.Builder.CreateBitOrPointerCast(lastValue, payloadIntTy); |
| lastValue = IGF.Builder.CreateOr(lastValue, subvalue); |
| payloadValue = IGF.Builder.CreateBitOrPointerCast(lastValue, payloadTy); |
| } |
| }); |
| } |
| |
| llvm::Value *EnumPayload::extractValue(IRGenFunction &IGF, llvm::Type *type, |
| unsigned payloadOffset) const { |
| llvm::Value *result = nullptr; |
| withValueInPayload(IGF, *this, type, -1, payloadOffset, |
| [&](LazyValue &payloadValue, |
| unsigned payloadBitWidth, |
| unsigned payloadValueOffset, |
| unsigned valueBitWidth, |
| unsigned valueOffset) { |
| auto payloadType = getPayloadType(payloadValue); |
| // If the desired type matches the payload slot exactly, we don't need |
| // to do anything. |
| if (payloadValueOffset == 0 && valueOffset == 0) { |
| if (type == payloadType) { |
| result = forcePayloadValue(payloadValue); |
| return; |
| } |
| // If only the width matches exactly, do a bitcast. |
| if (payloadBitWidth == valueBitWidth) { |
| result = |
| IGF.Builder.CreateBitOrPointerCast(forcePayloadValue(payloadValue), |
| type); |
| return; |
| } |
| } |
| |
| // Integrate the chunk of payload into the result value. |
| auto value = forcePayloadValue(payloadValue); |
| auto valueIntTy = |
| llvm::IntegerType::get(IGF.IGM.getLLVMContext(), valueBitWidth); |
| auto payloadIntTy = |
| llvm::IntegerType::get(IGF.IGM.getLLVMContext(), payloadBitWidth); |
| |
| value = IGF.Builder.CreateBitOrPointerCast(value, payloadIntTy); |
| #if defined(__BIG_ENDIAN__) |
| if ((valueBitWidth == 32 || valueBitWidth == 16 || valueBitWidth == 8 || valueBitWidth == 1) && |
| payloadBitWidth > (payloadValueOffset + valueBitWidth)) { |
| unsigned shiftBitWidth = valueBitWidth; |
| if (valueBitWidth == 1) { |
| shiftBitWidth = 8; |
| } |
| value = IGF.Builder.CreateLShr(value, |
| llvm::ConstantInt::get(value->getType(), (payloadBitWidth - shiftBitWidth) - payloadValueOffset)); |
| } |
| #else |
| if (payloadValueOffset > 0) |
| value = IGF.Builder.CreateLShr(value, |
| llvm::ConstantInt::get(value->getType(), payloadValueOffset)); |
| #endif |
| if (valueBitWidth > payloadBitWidth) |
| value = IGF.Builder.CreateZExt(value, valueIntTy); |
| if (valueOffset > 0) |
| value = IGF.Builder.CreateShl(value, |
| llvm::ConstantInt::get(value->getType(), valueOffset)); |
| if (valueBitWidth < payloadBitWidth) |
| value = IGF.Builder.CreateTrunc(value, valueIntTy); |
| |
| if (!result) |
| result = value; |
| else |
| result = IGF.Builder.CreateOr(result, value); |
| }); |
| return IGF.Builder.CreateBitOrPointerCast(result, type); |
| } |
| |
| EnumPayload EnumPayload::fromExplosion(IRGenModule &IGM, |
| Explosion &in, EnumPayloadSchema schema){ |
| EnumPayload result; |
| |
| schema.forEachType(IGM, [&](llvm::Type *type) { |
| auto next = in.claimNext(); |
| assert(next->getType() == type && "explosion doesn't match payload schema"); |
| result.PayloadValues.push_back(next); |
| }); |
| |
| return result; |
| } |
| |
| void EnumPayload::explode(IRGenModule &IGM, Explosion &out) const { |
| for (LazyValue &value : PayloadValues) { |
| out.add(forcePayloadValue(value)); |
| } |
| } |
| |
| void EnumPayload::packIntoEnumPayload(IRGenFunction &IGF, |
| EnumPayload &outerPayload, |
| unsigned bitOffset) const { |
| auto &DL = IGF.IGM.DataLayout; |
| for (auto &value : PayloadValues) { |
| auto v = forcePayloadValue(value); |
| outerPayload.insertValue(IGF, v, bitOffset); |
| bitOffset += DL.getTypeSizeInBits(v->getType()); |
| } |
| } |
| |
| EnumPayload EnumPayload::unpackFromEnumPayload(IRGenFunction &IGF, |
| const EnumPayload &outerPayload, |
| unsigned bitOffset, |
| EnumPayloadSchema schema) { |
| EnumPayload result; |
| auto &DL = IGF.IGM.DataLayout; |
| schema.forEachType(IGF.IGM, [&](llvm::Type *type) { |
| auto v = outerPayload.extractValue(IGF, type, bitOffset); |
| result.PayloadValues.push_back(v); |
| bitOffset += DL.getTypeSizeInBits(type); |
| }); |
| return result; |
| } |
| |
| static llvm::Type *getPayloadStorageType(IRGenModule &IGM, |
| const EnumPayload &payload) { |
| if (payload.StorageType) |
| return payload.StorageType; |
| |
| if (payload.PayloadValues.size() == 1) { |
| payload.StorageType = getPayloadType(payload.PayloadValues.front()); |
| return payload.StorageType; |
| } |
| |
| SmallVector<llvm::Type *, 2> elementTypes; |
| for (auto value : payload.PayloadValues) { |
| elementTypes.push_back(getPayloadType(value)); |
| } |
| |
| payload.StorageType = llvm::StructType::get(IGM.getLLVMContext(), |
| elementTypes); |
| return payload.StorageType; |
| } |
| |
| EnumPayload EnumPayload::load(IRGenFunction &IGF, Address address, |
| EnumPayloadSchema schema) { |
| EnumPayload result = EnumPayload::zero(IGF.IGM, schema); |
| if (result.PayloadValues.empty()) |
| return result; |
| |
| auto storageTy = getPayloadStorageType(IGF.IGM, result); |
| address = IGF.Builder.CreateBitCast(address, storageTy->getPointerTo()); |
| |
| if (result.PayloadValues.size() == 1) { |
| result.PayloadValues.front() = IGF.Builder.CreateLoad(address); |
| } else { |
| Size offset(0); |
| for (unsigned i : indices(result.PayloadValues)) { |
| auto &value = result.PayloadValues[i]; |
| auto member = IGF.Builder.CreateStructGEP(address, i, offset); |
| auto loadedValue = IGF.Builder.CreateLoad(member); |
| value = loadedValue; |
| offset += Size(IGF.IGM.DataLayout.getTypeAllocSize(loadedValue->getType())); |
| } |
| } |
| |
| return result; |
| } |
| |
| void EnumPayload::store(IRGenFunction &IGF, Address address) const { |
| if (PayloadValues.empty()) |
| return; |
| |
| auto storageTy = getPayloadStorageType(IGF.IGM, *this); |
| address = IGF.Builder.CreateBitCast(address, storageTy->getPointerTo()); |
| |
| if (PayloadValues.size() == 1) { |
| IGF.Builder.CreateStore(forcePayloadValue(PayloadValues.front()), address); |
| return; |
| } else { |
| Size offset(0); |
| for (unsigned i : indices(PayloadValues)) { |
| auto &value = PayloadValues[i]; |
| auto member = IGF.Builder.CreateStructGEP(address, i, offset); |
| auto valueToStore = forcePayloadValue(value); |
| IGF.Builder.CreateStore(valueToStore, member); |
| offset += Size(IGF.IGM.DataLayout |
| .getTypeAllocSize(valueToStore->getType())); |
| } |
| } |
| } |
| |
| namespace { |
| struct ult { |
| bool operator()(const APInt &a, const APInt &b) const { |
| return a.ult(b); |
| } |
| }; |
| } // end anonymous namespace |
| |
| static void emitSubSwitch(IRGenFunction &IGF, |
| MutableArrayRef<EnumPayload::LazyValue> values, |
| APInt mask, |
| MutableArrayRef<std::pair<APInt, llvm::BasicBlock *>> cases, |
| SwitchDefaultDest dflt) { |
| recur: |
| assert(!values.empty() && "didn't exit out when exhausting all values?!"); |
| |
| assert(!cases.empty() && "switching with no cases?!"); |
| |
| auto &DL = IGF.IGM.DataLayout; |
| auto &pv = values.front(); |
| values = values.slice(1); |
| auto payloadTy = getPayloadType(pv); |
| unsigned size = DL.getTypeSizeInBits(payloadTy); |
| |
| // Grab a chunk of the mask. |
| auto maskPiece = mask.zextOrTrunc(size); |
| mask = mask.lshr(size); |
| |
| // If the piece is zero, this doesn't affect the switch. We can just move |
| // forward and recur. |
| if (maskPiece == 0) { |
| for (auto &casePair : cases) |
| casePair.first = casePair.first.lshr(size); |
| goto recur; |
| } |
| |
| // Force the value we will test. |
| auto v = forcePayloadValue(pv); |
| auto payloadIntTy = llvm::IntegerType::get(IGF.IGM.getLLVMContext(), size); |
| |
| // Need to coerce to integer for 'icmp eq' if it's not already an integer |
| // or pointer. (Switching or masking will also require a cast to integer.) |
| if (!isa<llvm::IntegerType>(v->getType()) |
| && !isa<llvm::PointerType>(v->getType())) |
| v = IGF.Builder.CreateBitOrPointerCast(v, payloadIntTy); |
| |
| // Apply the mask if it's interesting. |
| if (!maskPiece.isAllOnesValue()) { |
| v = IGF.Builder.CreateBitOrPointerCast(v, payloadIntTy); |
| auto maskConstant = llvm::ConstantInt::get(payloadIntTy, maskPiece); |
| v = IGF.Builder.CreateAnd(v, maskConstant); |
| } |
| |
| // Gather the values we will switch over for this payload chunk. |
| // FIXME: std::map is lame. Should hash APInts. |
| std::map<APInt, SmallVector<std::pair<APInt, llvm::BasicBlock*>, 2>, ult> |
| subCases; |
| |
| for (auto casePair : cases) { |
| // Grab a chunk of the value. |
| auto valuePiece = casePair.first.zextOrTrunc(size); |
| // Index the case according to this chunk. |
| subCases[valuePiece].push_back({std::move(casePair.first).lshr(size), |
| casePair.second}); |
| } |
| |
| bool needsAdditionalCases = !values.empty() && mask != 0; |
| SmallVector<std::pair<llvm::BasicBlock *, decltype(cases)>, 2> recursiveCases; |
| |
| auto blockForCases |
| = [&](MutableArrayRef<std::pair<APInt, llvm::BasicBlock*>> cases) |
| -> llvm::BasicBlock * |
| { |
| // If we need to recur, emit a new block. |
| if (needsAdditionalCases) { |
| auto newBB = IGF.createBasicBlock(""); |
| recursiveCases.push_back({newBB, cases}); |
| return newBB; |
| } |
| // Otherwise, we can jump directly to the ultimate destination. |
| assert(cases.size() == 1 && "more than one case for final destination?!"); |
| return cases.front().second; |
| }; |
| |
| // If there's only one case, do a cond_br. |
| if (subCases.size() == 1) { |
| auto &subCase = *subCases.begin(); |
| llvm::BasicBlock *block = blockForCases(subCase.second); |
| // If the default case is unreachable, we don't need to conditionally |
| // branch. |
| if (dflt.getInt()) { |
| IGF.Builder.CreateBr(block); |
| goto next; |
| } |
| |
| auto &valuePiece = subCase.first; |
| llvm::Value *valueConstant = llvm::ConstantInt::get(payloadIntTy, |
| valuePiece); |
| valueConstant = IGF.Builder.CreateBitOrPointerCast(valueConstant, |
| v->getType()); |
| auto cmp = IGF.Builder.CreateICmpEQ(v, valueConstant); |
| IGF.Builder.CreateCondBr(cmp, block, dflt.getPointer()); |
| goto next; |
| } |
| |
| // Otherwise, do a switch. |
| { |
| v = IGF.Builder.CreateBitOrPointerCast(v, payloadIntTy); |
| auto swi = IGF.Builder.CreateSwitch(v, dflt.getPointer(), subCases.size()); |
| |
| for (auto &subCase : subCases) { |
| auto &valuePiece = subCase.first; |
| auto valueConstant = llvm::ConstantInt::get(IGF.IGM.getLLVMContext(), |
| valuePiece); |
| |
| swi->addCase(valueConstant, blockForCases(subCase.second)); |
| } |
| } |
| |
| next: |
| // Emit the recursive cases. |
| for (auto &recursive : recursiveCases) { |
| IGF.Builder.emitBlock(recursive.first); |
| emitSubSwitch(IGF, values, mask, recursive.second, dflt); |
| } |
| } |
| |
| void EnumPayload::emitSwitch(IRGenFunction &IGF, |
| APInt mask, |
| ArrayRef<std::pair<APInt, llvm::BasicBlock *>> cases, |
| SwitchDefaultDest dflt) const { |
| // If there's only one case to test, do a simple compare and branch. |
| if (cases.size() == 1) { |
| // If the default case is unreachable, don't bother branching at all. |
| if (dflt.getInt()) { |
| IGF.Builder.CreateBr(cases[0].second); |
| return; |
| } |
| |
| auto *cmp = emitCompare(IGF, mask, cases[0].first); |
| IGF.Builder.CreateCondBr(cmp, cases[0].second, dflt.getPointer()); |
| return; |
| } |
| |
| // Otherwise, break down the decision tree. |
| SmallVector<std::pair<APInt, llvm::BasicBlock*>, 4> mutableCases |
| (cases.begin(), cases.end()); |
| |
| emitSubSwitch(IGF, PayloadValues, mask, mutableCases, dflt); |
| |
| assert(IGF.Builder.hasPostTerminatorIP()); |
| } |
| |
| llvm::Value * |
| EnumPayload::emitCompare(IRGenFunction &IGF, APInt mask, APInt value) const { |
| // Succeed trivially for an empty payload, or if the payload is masked |
| // out completely. |
| if (PayloadValues.empty() || mask == 0) |
| return llvm::ConstantInt::get(IGF.IGM.Int1Ty, 1U); |
| |
| assert((~mask & value) == 0 |
| && "value has masked out bits set?!"); |
| |
| auto &DL = IGF.IGM.DataLayout; |
| llvm::Value *condition = nullptr; |
| for (auto &pv : PayloadValues) { |
| auto v = forcePayloadValue(pv); |
| unsigned size = DL.getTypeSizeInBits(v->getType()); |
| |
| // Break off a piece of the mask and value. |
| auto maskPiece = mask.zextOrTrunc(size); |
| auto valuePiece = value.zextOrTrunc(size); |
| |
| mask = mask.lshr(size); |
| value = value.lshr(size); |
| |
| // If this piece is zero, it doesn't affect the comparison. |
| if (maskPiece == 0) |
| continue; |
| |
| // Apply the mask and test. |
| bool isMasked = !maskPiece.isAllOnesValue(); |
| auto intTy = llvm::IntegerType::get(IGF.IGM.getLLVMContext(), size); |
| // Need to bitcast to an integer in order to use 'icmp eq' if the piece |
| // isn't already an int or pointer, or in order to apply a mask. |
| if (isMasked |
| || (!isa<llvm::IntegerType>(v->getType()) |
| && !isa<llvm::PointerType>(v->getType()))) |
| v = IGF.Builder.CreateBitOrPointerCast(v, intTy); |
| |
| if (isMasked) { |
| auto maskConstant = llvm::ConstantInt::get(intTy, maskPiece); |
| v = IGF.Builder.CreateAnd(v, maskConstant); |
| } |
| |
| llvm::Value *valueConstant = llvm::ConstantInt::get(intTy, valuePiece); |
| valueConstant = IGF.Builder.CreateBitOrPointerCast(valueConstant, |
| v->getType()); |
| auto cmp = IGF.Builder.CreateICmpEQ(v, valueConstant); |
| if (!condition) |
| condition = cmp; |
| else |
| condition = IGF.Builder.CreateAnd(condition, cmp); |
| } |
| |
| // We should have handled the cases where there are no significant conditions |
| // in the early exit. |
| assert(condition && "no significant condition?!"); |
| return condition; |
| } |
| |
| void |
| EnumPayload::emitApplyAndMask(IRGenFunction &IGF, APInt mask) { |
| // Early exit if the mask has no effect. |
| if (mask.isAllOnesValue()) |
| return; |
| |
| auto &DL = IGF.IGM.DataLayout; |
| for (auto &pv : PayloadValues) { |
| auto payloadTy = getPayloadType(pv); |
| unsigned size = DL.getTypeSizeInBits(payloadTy); |
| |
| // Break off a chunk of the mask. |
| auto maskPiece = mask.zextOrTrunc(size); |
| mask = mask.lshr(size); |
| |
| // If this piece is all ones, it has no effect. |
| if (maskPiece.isAllOnesValue()) |
| continue; |
| |
| // If the payload value is vacant, the mask can't change it. |
| if (pv.is<llvm::Type *>()) |
| continue; |
| |
| // If this piece is zero, it wipes out the chunk entirely, and we can |
| // drop it. |
| if (maskPiece == 0) { |
| pv = payloadTy; |
| continue; |
| } |
| |
| // Otherwise, apply the mask to the existing value. |
| auto v = pv.get<llvm::Value*>(); |
| auto payloadIntTy = llvm::IntegerType::get(IGF.IGM.getLLVMContext(), size); |
| auto maskConstant = llvm::ConstantInt::get(payloadIntTy, maskPiece); |
| v = IGF.Builder.CreateBitOrPointerCast(v, payloadIntTy); |
| v = IGF.Builder.CreateAnd(v, maskConstant); |
| v = IGF.Builder.CreateBitOrPointerCast(v, payloadTy); |
| pv = v; |
| } |
| } |
| |
| void |
| EnumPayload::emitApplyOrMask(IRGenFunction &IGF, APInt mask) { |
| // Early exit if the mask has no effect. |
| if (mask == 0) |
| return; |
| |
| auto &DL = IGF.IGM.DataLayout; |
| for (auto &pv : PayloadValues) { |
| auto payloadTy = getPayloadType(pv); |
| unsigned size = DL.getTypeSizeInBits(payloadTy); |
| |
| // Break off a chunk of the mask. |
| auto maskPiece = mask.zextOrTrunc(size); |
| mask = mask.lshr(size); |
| |
| // If this piece is zero, it has no effect. |
| if (maskPiece == 0) |
| continue; |
| |
| auto payloadIntTy = llvm::IntegerType::get(IGF.IGM.getLLVMContext(), size); |
| auto maskConstant = llvm::ConstantInt::get(payloadIntTy, maskPiece); |
| |
| // If the payload value is vacant, or the mask is all ones, |
| // we can adopt the mask value directly. |
| if (pv.is<llvm::Type *>() || maskPiece.isAllOnesValue()) { |
| pv = IGF.Builder.CreateBitOrPointerCast(maskConstant, payloadTy); |
| continue; |
| } |
| |
| // Otherwise, apply the mask to the existing value. |
| auto v = pv.get<llvm::Value*>(); |
| v = IGF.Builder.CreateBitOrPointerCast(v, payloadIntTy); |
| v = IGF.Builder.CreateOr(v, maskConstant); |
| v = IGF.Builder.CreateBitOrPointerCast(v, payloadTy); |
| pv = v; |
| } |
| } |
| |
| void |
| EnumPayload::emitApplyOrMask(IRGenFunction &IGF, |
| EnumPayload mask) { |
| unsigned count = PayloadValues.size(); |
| assert(count == mask.PayloadValues.size()); |
| |
| auto &DL = IGF.IGM.DataLayout; |
| for (unsigned i = 0; i < count; i++ ) { |
| auto payloadTy = getPayloadType(PayloadValues[i]); |
| unsigned size = DL.getTypeSizeInBits(payloadTy); |
| |
| auto payloadIntTy = llvm::IntegerType::get(IGF.IGM.getLLVMContext(), size); |
| |
| if (mask.PayloadValues[i].is<llvm::Type *>()) { |
| // We're ORing with zero, do nothing |
| } else if (PayloadValues[i].is<llvm::Type *>()) { |
| PayloadValues[i] = mask.PayloadValues[i]; |
| } else { |
| auto v1 = IGF.Builder.CreateBitOrPointerCast( |
| PayloadValues[i].get<llvm::Value *>(), |
| payloadIntTy); |
| |
| auto v2 = IGF.Builder.CreateBitOrPointerCast( |
| mask.PayloadValues[i].get<llvm::Value *>(), |
| payloadIntTy); |
| |
| PayloadValues[i] = IGF.Builder.CreateBitOrPointerCast( |
| IGF.Builder.CreateOr(v1, v2), |
| payloadTy); |
| } |
| } |
| } |
| |
| /// Gather spare bits into the low bits of a smaller integer value. |
| llvm::Value *irgen::emitGatherSpareBits(IRGenFunction &IGF, |
| const SpareBitVector &spareBitMask, |
| llvm::Value *spareBits, |
| unsigned resultLowBit, |
| unsigned resultBitWidth) { |
| auto destTy |
| = llvm::IntegerType::get(IGF.IGM.getLLVMContext(), resultBitWidth); |
| unsigned usedBits = resultLowBit; |
| llvm::Value *result = nullptr; |
| |
| auto spareBitEnumeration = spareBitMask.enumerateSetBits(); |
| for (auto optSpareBit = spareBitEnumeration.findNext(); |
| optSpareBit.hasValue() && usedBits < resultBitWidth; |
| optSpareBit = spareBitEnumeration.findNext()) { |
| unsigned u = optSpareBit.getValue(); |
| assert(u >= (usedBits - resultLowBit) && |
| "used more bits than we've processed?!"); |
| |
| // Shift the bits into place. |
| llvm::Value *newBits; |
| if (u > usedBits) |
| newBits = IGF.Builder.CreateLShr(spareBits, u - usedBits); |
| else if (u < usedBits) |
| newBits = IGF.Builder.CreateShl(spareBits, usedBits - u); |
| else |
| newBits = spareBits; |
| newBits = IGF.Builder.CreateZExtOrTrunc(newBits, destTy); |
| |
| // See how many consecutive bits we have. |
| unsigned numBits = 1; |
| ++u; |
| // We don't need more bits than the size of the result. |
| unsigned maxBits = resultBitWidth - usedBits; |
| for (unsigned e = spareBitMask.size(); |
| u < e && numBits < maxBits && spareBitMask[u]; |
| ++u) { |
| ++numBits; |
| (void) spareBitEnumeration.findNext(); |
| } |
| |
| // Mask out the selected bits. |
| auto val = APInt::getAllOnesValue(numBits); |
| if (numBits < resultBitWidth) |
| val = val.zext(resultBitWidth); |
| val = val.shl(usedBits); |
| auto *mask = llvm::ConstantInt::get(IGF.IGM.getLLVMContext(), val); |
| newBits = IGF.Builder.CreateAnd(newBits, mask); |
| |
| // Accumulate the result. |
| if (result) |
| result = IGF.Builder.CreateOr(result, newBits); |
| else |
| result = newBits; |
| |
| usedBits += numBits; |
| } |
| |
| return result; |
| } |
| |
| |
| |
| llvm::Value * |
| EnumPayload::emitGatherSpareBits(IRGenFunction &IGF, |
| const SpareBitVector &spareBits, |
| unsigned firstBitOffset, |
| unsigned bitWidth) const { |
| auto &DL = IGF.IGM.DataLayout; |
| unsigned payloadOffset = 0; |
| llvm::Value *spareBitValue = nullptr; |
| auto destTy = llvm::IntegerType::get(IGF.IGM.getLLVMContext(), bitWidth); |
| for (auto &pv : PayloadValues) { |
| // If this value is zero, it has nothing to add to the spare bits. |
| auto v = pv.dyn_cast<llvm::Value*>(); |
| if (!v) { |
| payloadOffset += DL.getTypeSizeInBits(pv.get<llvm::Type*>()); |
| continue; |
| } |
| |
| unsigned size = DL.getTypeSizeInBits(v->getType()); |
| // Slice the spare bit vector. |
| // FIXME: this is inefficient. |
| auto spareBitsPart = SpareBitVector::getConstant(size, false); |
| unsigned numBitsInPart = 0; |
| for (unsigned i = 0; i < size; ++i) |
| if (spareBits[payloadOffset + i]) { |
| spareBitsPart.setBit(i); |
| ++numBitsInPart; |
| } |
| |
| payloadOffset += size; |
| |
| // If there were no spare bits in this part, it has nothing to add. |
| if (numBitsInPart == 0) |
| continue; |
| |
| if (firstBitOffset >= bitWidth) |
| break; |
| |
| // Get the spare bits from this part. |
| auto bits = irgen::emitGatherSpareBits(IGF, spareBitsPart, |
| v, firstBitOffset, bitWidth); |
| firstBitOffset += numBitsInPart; |
| |
| // Accumulate it into the full set. |
| if (!spareBitValue) |
| spareBitValue = bits; |
| else |
| spareBitValue = IGF.Builder.CreateOr(spareBitValue, bits); |
| } |
| if (!spareBitValue) |
| return llvm::ConstantInt::get(destTy, 0); |
| return spareBitValue; |
| } |
