| use super::{FunctionCx, LocalRef}; |
| use super::place::PlaceRef; |
| |
| use crate::MemFlags; |
| use crate::base; |
| use crate::glue; |
| use crate::traits::*; |
| |
| use rustc::mir::interpret::{ConstValue, ErrorHandled, Pointer, Scalar}; |
| use rustc::mir; |
| use rustc::ty; |
| use rustc::ty::layout::{self, Align, LayoutOf, TyLayout, Size}; |
| |
| use std::fmt; |
| |
| /// The representation of a Rust value. The enum variant is in fact |
| /// uniquely determined by the value's type, but is kept as a |
| /// safety check. |
| #[derive(Copy, Clone, Debug)] |
| pub enum OperandValue<V> { |
| /// A reference to the actual operand. The data is guaranteed |
| /// to be valid for the operand's lifetime. |
| /// The second value, if any, is the extra data (vtable or length) |
| /// which indicates that it refers to an unsized rvalue. |
| Ref(V, Option<V>, Align), |
| /// A single LLVM value. |
| Immediate(V), |
| /// A pair of immediate LLVM values. Used by fat pointers too. |
| Pair(V, V) |
| } |
| |
| /// An `OperandRef` is an "SSA" reference to a Rust value, along with |
| /// its type. |
| /// |
| /// NOTE: unless you know a value's type exactly, you should not |
| /// generate LLVM opcodes acting on it and instead act via methods, |
| /// to avoid nasty edge cases. In particular, using `Builder::store` |
| /// directly is sure to cause problems -- use `OperandRef::store` |
| /// instead. |
| #[derive(Copy, Clone)] |
| pub struct OperandRef<'tcx, V> { |
| // The value. |
| pub val: OperandValue<V>, |
| |
| // The layout of value, based on its Rust type. |
| pub layout: TyLayout<'tcx>, |
| } |
| |
| impl<V: CodegenObject> fmt::Debug for OperandRef<'tcx, V> { |
| fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { |
| write!(f, "OperandRef({:?} @ {:?})", self.val, self.layout) |
| } |
| } |
| |
| impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { |
| pub fn new_zst<Bx: BuilderMethods<'a, 'tcx, Value = V>>( |
| bx: &mut Bx, |
| layout: TyLayout<'tcx> |
| ) -> OperandRef<'tcx, V> { |
| assert!(layout.is_zst()); |
| OperandRef { |
| val: OperandValue::Immediate(bx.const_undef(bx.immediate_backend_type(layout))), |
| layout |
| } |
| } |
| |
| pub fn from_const<Bx: BuilderMethods<'a, 'tcx, Value = V>>( |
| bx: &mut Bx, |
| val: &'tcx ty::Const<'tcx> |
| ) -> Self { |
| let layout = bx.layout_of(val.ty); |
| |
| if layout.is_zst() { |
| return OperandRef::new_zst(bx, layout); |
| } |
| |
| let val_val = match val.val { |
| ty::ConstKind::Value(val_val) => val_val, |
| _ => bug!("encountered bad ConstKind in codegen"), |
| }; |
| |
| let val = match val_val { |
| ConstValue::Scalar(x) => { |
| let scalar = match layout.abi { |
| layout::Abi::Scalar(ref x) => x, |
| _ => bug!("from_const: invalid ByVal layout: {:#?}", layout) |
| }; |
| let llval = bx.scalar_to_backend( |
| x, |
| scalar, |
| bx.immediate_backend_type(layout), |
| ); |
| OperandValue::Immediate(llval) |
| }, |
| ConstValue::Slice { data, start, end } => { |
| let a_scalar = match layout.abi { |
| layout::Abi::ScalarPair(ref a, _) => a, |
| _ => bug!("from_const: invalid ScalarPair layout: {:#?}", layout) |
| }; |
| let a = Scalar::from(Pointer::new( |
| bx.tcx().alloc_map.lock().create_memory_alloc(data), |
| Size::from_bytes(start as u64), |
| )).into(); |
| let a_llval = bx.scalar_to_backend( |
| a, |
| a_scalar, |
| bx.scalar_pair_element_backend_type(layout, 0, true), |
| ); |
| let b_llval = bx.const_usize((end - start) as u64); |
| OperandValue::Pair(a_llval, b_llval) |
| }, |
| ConstValue::ByRef { alloc, offset } => { |
| return bx.load_operand(bx.from_const_alloc(layout, alloc, offset)); |
| }, |
| }; |
| |
| OperandRef { |
| val, |
| layout |
| } |
| } |
| |
| /// Asserts that this operand refers to a scalar and returns |
| /// a reference to its value. |
| pub fn immediate(self) -> V { |
| match self.val { |
| OperandValue::Immediate(s) => s, |
| _ => bug!("not immediate: {:?}", self) |
| } |
| } |
| |
| pub fn deref<Cx: LayoutTypeMethods<'tcx>>( |
| self, |
| cx: &Cx |
| ) -> PlaceRef<'tcx, V> { |
| let projected_ty = self.layout.ty.builtin_deref(true) |
| .unwrap_or_else(|| bug!("deref of non-pointer {:?}", self)).ty; |
| let (llptr, llextra) = match self.val { |
| OperandValue::Immediate(llptr) => (llptr, None), |
| OperandValue::Pair(llptr, llextra) => (llptr, Some(llextra)), |
| OperandValue::Ref(..) => bug!("Deref of by-Ref operand {:?}", self) |
| }; |
| let layout = cx.layout_of(projected_ty); |
| PlaceRef { |
| llval: llptr, |
| llextra, |
| layout, |
| align: layout.align.abi, |
| } |
| } |
| |
| /// If this operand is a `Pair`, we return an aggregate with the two values. |
| /// For other cases, see `immediate`. |
| pub fn immediate_or_packed_pair<Bx: BuilderMethods<'a, 'tcx, Value = V>>( |
| self, |
| bx: &mut Bx |
| ) -> V { |
| if let OperandValue::Pair(a, b) = self.val { |
| let llty = bx.cx().backend_type(self.layout); |
| debug!("Operand::immediate_or_packed_pair: packing {:?} into {:?}", |
| self, llty); |
| // Reconstruct the immediate aggregate. |
| let mut llpair = bx.cx().const_undef(llty); |
| let imm_a = base::from_immediate(bx, a); |
| let imm_b = base::from_immediate(bx, b); |
| llpair = bx.insert_value(llpair, imm_a, 0); |
| llpair = bx.insert_value(llpair, imm_b, 1); |
| llpair |
| } else { |
| self.immediate() |
| } |
| } |
| |
| /// If the type is a pair, we return a `Pair`, otherwise, an `Immediate`. |
| pub fn from_immediate_or_packed_pair<Bx: BuilderMethods<'a, 'tcx, Value = V>>( |
| bx: &mut Bx, |
| llval: V, |
| layout: TyLayout<'tcx> |
| ) -> Self { |
| let val = if let layout::Abi::ScalarPair(ref a, ref b) = layout.abi { |
| debug!("Operand::from_immediate_or_packed_pair: unpacking {:?} @ {:?}", |
| llval, layout); |
| |
| // Deconstruct the immediate aggregate. |
| let a_llval = bx.extract_value(llval, 0); |
| let a_llval = base::to_immediate_scalar(bx, a_llval, a); |
| let b_llval = bx.extract_value(llval, 1); |
| let b_llval = base::to_immediate_scalar(bx, b_llval, b); |
| OperandValue::Pair(a_llval, b_llval) |
| } else { |
| OperandValue::Immediate(llval) |
| }; |
| OperandRef { val, layout } |
| } |
| |
| pub fn extract_field<Bx: BuilderMethods<'a, 'tcx, Value = V>>( |
| &self, |
| bx: &mut Bx, |
| i: usize |
| ) -> Self { |
| let field = self.layout.field(bx.cx(), i); |
| let offset = self.layout.fields.offset(i); |
| |
| let mut val = match (self.val, &self.layout.abi) { |
| // If the field is ZST, it has no data. |
| _ if field.is_zst() => { |
| return OperandRef::new_zst(bx, field); |
| } |
| |
| // Newtype of a scalar, scalar pair or vector. |
| (OperandValue::Immediate(_), _) | |
| (OperandValue::Pair(..), _) if field.size == self.layout.size => { |
| assert_eq!(offset.bytes(), 0); |
| self.val |
| } |
| |
| // Extract a scalar component from a pair. |
| (OperandValue::Pair(a_llval, b_llval), &layout::Abi::ScalarPair(ref a, ref b)) => { |
| if offset.bytes() == 0 { |
| assert_eq!(field.size, a.value.size(bx.cx())); |
| OperandValue::Immediate(a_llval) |
| } else { |
| assert_eq!(offset, a.value.size(bx.cx()) |
| .align_to(b.value.align(bx.cx()).abi)); |
| assert_eq!(field.size, b.value.size(bx.cx())); |
| OperandValue::Immediate(b_llval) |
| } |
| } |
| |
| // `#[repr(simd)]` types are also immediate. |
| (OperandValue::Immediate(llval), &layout::Abi::Vector { .. }) => { |
| OperandValue::Immediate( |
| bx.extract_element(llval, bx.cx().const_usize(i as u64))) |
| } |
| |
| _ => bug!("OperandRef::extract_field({:?}): not applicable", self) |
| }; |
| |
| // HACK(eddyb) have to bitcast pointers until LLVM removes pointee types. |
| // Bools in union fields needs to be truncated. |
| let to_immediate_or_cast = |bx: &mut Bx, val, ty| { |
| if ty == bx.cx().type_i1() { |
| bx.trunc(val, ty) |
| } else { |
| bx.bitcast(val, ty) |
| } |
| }; |
| |
| match val { |
| OperandValue::Immediate(ref mut llval) => { |
| *llval = to_immediate_or_cast(bx, *llval, bx.cx().immediate_backend_type(field)); |
| } |
| OperandValue::Pair(ref mut a, ref mut b) => { |
| *a = to_immediate_or_cast(bx, *a, bx.cx() |
| .scalar_pair_element_backend_type(field, 0, true)); |
| *b = to_immediate_or_cast(bx, *b, bx.cx() |
| .scalar_pair_element_backend_type(field, 1, true)); |
| } |
| OperandValue::Ref(..) => bug!() |
| } |
| |
| OperandRef { |
| val, |
| layout: field |
| } |
| } |
| } |
| |
| impl<'a, 'tcx, V: CodegenObject> OperandValue<V> { |
| pub fn store<Bx: BuilderMethods<'a, 'tcx, Value = V>>( |
| self, |
| bx: &mut Bx, |
| dest: PlaceRef<'tcx, V> |
| ) { |
| self.store_with_flags(bx, dest, MemFlags::empty()); |
| } |
| |
| pub fn volatile_store<Bx: BuilderMethods<'a, 'tcx, Value = V>>( |
| self, |
| bx: &mut Bx, |
| dest: PlaceRef<'tcx, V> |
| ) { |
| self.store_with_flags(bx, dest, MemFlags::VOLATILE); |
| } |
| |
| pub fn unaligned_volatile_store<Bx: BuilderMethods<'a, 'tcx, Value = V>>( |
| self, |
| bx: &mut Bx, |
| dest: PlaceRef<'tcx, V>, |
| ) { |
| self.store_with_flags(bx, dest, MemFlags::VOLATILE | MemFlags::UNALIGNED); |
| } |
| |
| pub fn nontemporal_store<Bx: BuilderMethods<'a, 'tcx, Value = V>>( |
| self, |
| bx: &mut Bx, |
| dest: PlaceRef<'tcx, V> |
| ) { |
| self.store_with_flags(bx, dest, MemFlags::NONTEMPORAL); |
| } |
| |
| fn store_with_flags<Bx: BuilderMethods<'a, 'tcx, Value = V>>( |
| self, |
| bx: &mut Bx, |
| dest: PlaceRef<'tcx, V>, |
| flags: MemFlags, |
| ) { |
| debug!("OperandRef::store: operand={:?}, dest={:?}", self, dest); |
| // Avoid generating stores of zero-sized values, because the only way to have a zero-sized |
| // value is through `undef`, and store itself is useless. |
| if dest.layout.is_zst() { |
| return; |
| } |
| match self { |
| OperandValue::Ref(r, None, source_align) => { |
| base::memcpy_ty(bx, dest.llval, dest.align, r, source_align, |
| dest.layout, flags) |
| } |
| OperandValue::Ref(_, Some(_), _) => { |
| bug!("cannot directly store unsized values"); |
| } |
| OperandValue::Immediate(s) => { |
| let val = base::from_immediate(bx, s); |
| bx.store_with_flags(val, dest.llval, dest.align, flags); |
| } |
| OperandValue::Pair(a, b) => { |
| let (a_scalar, b_scalar) = match dest.layout.abi { |
| layout::Abi::ScalarPair(ref a, ref b) => (a, b), |
| _ => bug!("store_with_flags: invalid ScalarPair layout: {:#?}", dest.layout) |
| }; |
| let b_offset = a_scalar.value.size(bx).align_to(b_scalar.value.align(bx).abi); |
| |
| let llptr = bx.struct_gep(dest.llval, 0); |
| let val = base::from_immediate(bx, a); |
| let align = dest.align; |
| bx.store_with_flags(val, llptr, align, flags); |
| |
| let llptr = bx.struct_gep(dest.llval, 1); |
| let val = base::from_immediate(bx, b); |
| let align = dest.align.restrict_for_offset(b_offset); |
| bx.store_with_flags(val, llptr, align, flags); |
| } |
| } |
| } |
| |
| pub fn store_unsized<Bx: BuilderMethods<'a, 'tcx, Value = V>>( |
| self, |
| bx: &mut Bx, |
| indirect_dest: PlaceRef<'tcx, V> |
| ) { |
| debug!("OperandRef::store_unsized: operand={:?}, indirect_dest={:?}", self, indirect_dest); |
| let flags = MemFlags::empty(); |
| |
| // `indirect_dest` must have `*mut T` type. We extract `T` out of it. |
| let unsized_ty = indirect_dest.layout.ty.builtin_deref(true) |
| .unwrap_or_else(|| bug!("indirect_dest has non-pointer type: {:?}", indirect_dest)).ty; |
| |
| let (llptr, llextra) = |
| if let OperandValue::Ref(llptr, Some(llextra), _) = self { |
| (llptr, llextra) |
| } else { |
| bug!("store_unsized called with a sized value") |
| }; |
| |
| // FIXME: choose an appropriate alignment, or use dynamic align somehow |
| let max_align = Align::from_bits(128).unwrap(); |
| let min_align = Align::from_bits(8).unwrap(); |
| |
| // Allocate an appropriate region on the stack, and copy the value into it |
| let (llsize, _) = glue::size_and_align_of_dst(bx, unsized_ty, Some(llextra)); |
| let lldst = bx.array_alloca(bx.cx().type_i8(), llsize, max_align); |
| bx.memcpy(lldst, max_align, llptr, min_align, llsize, flags); |
| |
| // Store the allocated region and the extra to the indirect place. |
| let indirect_operand = OperandValue::Pair(lldst, llextra); |
| indirect_operand.store(bx, indirect_dest); |
| } |
| } |
| |
| impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { |
| fn maybe_codegen_consume_direct( |
| &mut self, |
| bx: &mut Bx, |
| place_ref: &mir::PlaceRef<'_, 'tcx> |
| ) -> Option<OperandRef<'tcx, Bx::Value>> { |
| debug!("maybe_codegen_consume_direct(place_ref={:?})", place_ref); |
| |
| if let mir::PlaceBase::Local(index) = place_ref.base { |
| match self.locals[*index] { |
| LocalRef::Operand(Some(mut o)) => { |
| // Moves out of scalar and scalar pair fields are trivial. |
| for elem in place_ref.projection.iter() { |
| match elem { |
| mir::ProjectionElem::Field(ref f, _) => { |
| o = o.extract_field(bx, f.index()); |
| } |
| mir::ProjectionElem::Index(_) | |
| mir::ProjectionElem::ConstantIndex { .. } => { |
| // ZSTs don't require any actual memory access. |
| // FIXME(eddyb) deduplicate this with the identical |
| // checks in `codegen_consume` and `extract_field`. |
| let elem = o.layout.field(bx.cx(), 0); |
| if elem.is_zst() { |
| o = OperandRef::new_zst(bx, elem); |
| } else { |
| return None; |
| } |
| } |
| _ => return None, |
| } |
| } |
| |
| Some(o) |
| } |
| LocalRef::Operand(None) => { |
| bug!("use of {:?} before def", place_ref); |
| } |
| LocalRef::Place(..) | LocalRef::UnsizedPlace(..) => { |
| // watch out for locals that do not have an |
| // alloca; they are handled somewhat differently |
| None |
| } |
| } |
| } else { |
| None |
| } |
| } |
| |
| pub fn codegen_consume( |
| &mut self, |
| bx: &mut Bx, |
| place_ref: &mir::PlaceRef<'_, 'tcx> |
| ) -> OperandRef<'tcx, Bx::Value> { |
| debug!("codegen_consume(place_ref={:?})", place_ref); |
| |
| let ty = self.monomorphized_place_ty(place_ref); |
| let layout = bx.cx().layout_of(ty); |
| |
| // ZSTs don't require any actual memory access. |
| if layout.is_zst() { |
| return OperandRef::new_zst(bx, layout); |
| } |
| |
| if let Some(o) = self.maybe_codegen_consume_direct(bx, place_ref) { |
| return o; |
| } |
| |
| // for most places, to consume them we just load them |
| // out from their home |
| let place = self.codegen_place(bx, place_ref); |
| bx.load_operand(place) |
| } |
| |
| pub fn codegen_operand( |
| &mut self, |
| bx: &mut Bx, |
| operand: &mir::Operand<'tcx> |
| ) -> OperandRef<'tcx, Bx::Value> { |
| debug!("codegen_operand(operand={:?})", operand); |
| |
| match *operand { |
| mir::Operand::Copy(ref place) | |
| mir::Operand::Move(ref place) => { |
| self.codegen_consume(bx, &place.as_ref()) |
| } |
| |
| mir::Operand::Constant(ref constant) => { |
| self.eval_mir_constant_to_operand(bx, constant) |
| .unwrap_or_else(|err| { |
| match err { |
| // errored or at least linted |
| ErrorHandled::Reported => {}, |
| ErrorHandled::TooGeneric => { |
| bug!("codgen encountered polymorphic constant") |
| }, |
| } |
| // Allow RalfJ to sleep soundly knowing that even refactorings that remove |
| // the above error (or silence it under some conditions) will not cause UB. |
| bx.abort(); |
| // We still have to return an operand but it doesn't matter, |
| // this code is unreachable. |
| let ty = self.monomorphize(&constant.literal.ty); |
| let layout = bx.cx().layout_of(ty); |
| bx.load_operand(PlaceRef::new_sized( |
| bx.cx().const_undef(bx.cx().type_ptr_to(bx.cx().backend_type(layout))), |
| layout, |
| )) |
| }) |
| } |
| } |
| } |
| } |
