| use std::{collections::hash_map::Entry, io::Write, iter, path::Path}; |
| |
| use rustc_apfloat::Float; |
| use rustc_ast::expand::allocator::alloc_error_handler_name; |
| use rustc_hir::{def::DefKind, def_id::CrateNum}; |
| use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; |
| use rustc_middle::mir; |
| use rustc_middle::ty; |
| use rustc_span::Symbol; |
| use rustc_target::{ |
| abi::{Align, Size}, |
| spec::abi::Abi, |
| }; |
| |
| use super::alloc::{check_alloc_request, EvalContextExt as _}; |
| use super::backtrace::EvalContextExt as _; |
| use crate::*; |
| use helpers::{ToHost, ToSoft}; |
| |
| /// Type of dynamic symbols (for `dlsym` et al) |
| #[derive(Debug, Copy, Clone)] |
| pub struct DynSym(Symbol); |
| |
| #[allow(clippy::should_implement_trait)] |
| impl DynSym { |
| pub fn from_str(name: &str) -> Self { |
| DynSym(Symbol::intern(name)) |
| } |
| } |
| |
| impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriInterpCx<'mir, 'tcx> {} |
| pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { |
| /// Emulates calling a foreign item, failing if the item is not supported. |
| /// This function will handle `goto_block` if needed. |
| /// Returns Ok(None) if the foreign item was completely handled |
| /// by this function. |
| /// Returns Ok(Some(body)) if processing the foreign item |
| /// is delegated to another function. |
| fn emulate_foreign_item( |
| &mut self, |
| link_name: Symbol, |
| abi: Abi, |
| args: &[OpTy<'tcx, Provenance>], |
| dest: &MPlaceTy<'tcx, Provenance>, |
| ret: Option<mir::BasicBlock>, |
| unwind: mir::UnwindAction, |
| ) -> InterpResult<'tcx, Option<(&'mir mir::Body<'tcx>, ty::Instance<'tcx>)>> { |
| let this = self.eval_context_mut(); |
| let tcx = this.tcx.tcx; |
| |
| // Some shims forward to other MIR bodies. |
| match link_name.as_str() { |
| // This matches calls to the foreign item `panic_impl`. |
| // The implementation is provided by the function with the `#[panic_handler]` attribute. |
| "panic_impl" => { |
| // We don't use `check_shim` here because we are just forwarding to the lang |
| // item. Argument count checking will be performed when the returned `Body` is |
| // called. |
| this.check_abi_and_shim_symbol_clash(abi, Abi::Rust, link_name)?; |
| let panic_impl_id = tcx.lang_items().panic_impl().unwrap(); |
| let panic_impl_instance = ty::Instance::mono(tcx, panic_impl_id); |
| return Ok(Some(( |
| this.load_mir(panic_impl_instance.def, None)?, |
| panic_impl_instance, |
| ))); |
| } |
| "__rust_alloc_error_handler" => { |
| // Forward to the right symbol that implements this function. |
| let Some(handler_kind) = this.tcx.alloc_error_handler_kind(()) else { |
| // in real code, this symbol does not exist without an allocator |
| throw_unsup_format!( |
| "`__rust_alloc_error_handler` cannot be called when no alloc error handler is set" |
| ); |
| }; |
| let name = alloc_error_handler_name(handler_kind); |
| let handler = this |
| .lookup_exported_symbol(Symbol::intern(name))? |
| .expect("missing alloc error handler symbol"); |
| return Ok(Some(handler)); |
| } |
| _ => {} |
| } |
| |
| // The rest either implements the logic, or falls back to `lookup_exported_symbol`. |
| match this.emulate_foreign_item_inner(link_name, abi, args, dest, unwind)? { |
| EmulateItemResult::NeedsJumping => { |
| trace!("{:?}", this.dump_place(&dest.clone().into())); |
| this.return_to_block(ret)?; |
| } |
| EmulateItemResult::AlreadyJumped => (), |
| EmulateItemResult::NotSupported => { |
| if let Some(body) = this.lookup_exported_symbol(link_name)? { |
| return Ok(Some(body)); |
| } |
| |
| this.handle_unsupported_foreign_item(format!( |
| "can't call foreign function `{link_name}` on OS `{os}`", |
| os = this.tcx.sess.target.os, |
| ))?; |
| return Ok(None); |
| } |
| } |
| |
| Ok(None) |
| } |
| |
| fn is_dyn_sym(&self, name: &str) -> bool { |
| let this = self.eval_context_ref(); |
| match this.tcx.sess.target.os.as_ref() { |
| os if this.target_os_is_unix() => shims::unix::foreign_items::is_dyn_sym(name, os), |
| "windows" => shims::windows::foreign_items::is_dyn_sym(name), |
| _ => false, |
| } |
| } |
| |
| /// Emulates a call to a `DynSym`. |
| fn emulate_dyn_sym( |
| &mut self, |
| sym: DynSym, |
| abi: Abi, |
| args: &[OpTy<'tcx, Provenance>], |
| dest: &MPlaceTy<'tcx, Provenance>, |
| ret: Option<mir::BasicBlock>, |
| unwind: mir::UnwindAction, |
| ) -> InterpResult<'tcx> { |
| let res = self.emulate_foreign_item(sym.0, abi, args, dest, ret, unwind)?; |
| assert!(res.is_none(), "DynSyms that delegate are not supported"); |
| Ok(()) |
| } |
| |
| /// Lookup the body of a function that has `link_name` as the symbol name. |
| fn lookup_exported_symbol( |
| &mut self, |
| link_name: Symbol, |
| ) -> InterpResult<'tcx, Option<(&'mir mir::Body<'tcx>, ty::Instance<'tcx>)>> { |
| let this = self.eval_context_mut(); |
| let tcx = this.tcx.tcx; |
| |
| // If the result was cached, just return it. |
| // (Cannot use `or_insert` since the code below might have to throw an error.) |
| let entry = this.machine.exported_symbols_cache.entry(link_name); |
| let instance = *match entry { |
| Entry::Occupied(e) => e.into_mut(), |
| Entry::Vacant(e) => { |
| // Find it if it was not cached. |
| let mut instance_and_crate: Option<(ty::Instance<'_>, CrateNum)> = None; |
| helpers::iter_exported_symbols(tcx, |cnum, def_id| { |
| let attrs = tcx.codegen_fn_attrs(def_id); |
| let symbol_name = if let Some(export_name) = attrs.export_name { |
| export_name |
| } else if attrs.flags.contains(CodegenFnAttrFlags::NO_MANGLE) { |
| tcx.item_name(def_id) |
| } else { |
| // Skip over items without an explicitly defined symbol name. |
| return Ok(()); |
| }; |
| if symbol_name == link_name { |
| if let Some((original_instance, original_cnum)) = instance_and_crate { |
| // Make sure we are consistent wrt what is 'first' and 'second'. |
| let original_span = tcx.def_span(original_instance.def_id()).data(); |
| let span = tcx.def_span(def_id).data(); |
| if original_span < span { |
| throw_machine_stop!(TerminationInfo::MultipleSymbolDefinitions { |
| link_name, |
| first: original_span, |
| first_crate: tcx.crate_name(original_cnum), |
| second: span, |
| second_crate: tcx.crate_name(cnum), |
| }); |
| } else { |
| throw_machine_stop!(TerminationInfo::MultipleSymbolDefinitions { |
| link_name, |
| first: span, |
| first_crate: tcx.crate_name(cnum), |
| second: original_span, |
| second_crate: tcx.crate_name(original_cnum), |
| }); |
| } |
| } |
| if !matches!(tcx.def_kind(def_id), DefKind::Fn | DefKind::AssocFn) { |
| throw_ub_format!( |
| "attempt to call an exported symbol that is not defined as a function" |
| ); |
| } |
| instance_and_crate = Some((ty::Instance::mono(tcx, def_id), cnum)); |
| } |
| Ok(()) |
| })?; |
| |
| e.insert(instance_and_crate.map(|ic| ic.0)) |
| } |
| }; |
| match instance { |
| None => Ok(None), // no symbol with this name |
| Some(instance) => Ok(Some((this.load_mir(instance.def, None)?, instance))), |
| } |
| } |
| } |
| |
| impl<'mir, 'tcx: 'mir> EvalContextExtPriv<'mir, 'tcx> for crate::MiriInterpCx<'mir, 'tcx> {} |
| trait EvalContextExtPriv<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { |
| fn emulate_foreign_item_inner( |
| &mut self, |
| link_name: Symbol, |
| abi: Abi, |
| args: &[OpTy<'tcx, Provenance>], |
| dest: &MPlaceTy<'tcx, Provenance>, |
| unwind: mir::UnwindAction, |
| ) -> InterpResult<'tcx, EmulateItemResult> { |
| let this = self.eval_context_mut(); |
| |
| // First deal with any external C functions in linked .so file. |
| #[cfg(target_os = "linux")] |
| if this.machine.native_lib.as_ref().is_some() { |
| use crate::shims::native_lib::EvalContextExt as _; |
| // An Ok(false) here means that the function being called was not exported |
| // by the specified `.so` file; we should continue and check if it corresponds to |
| // a provided shim. |
| if this.call_native_fn(link_name, dest, args)? { |
| return Ok(EmulateItemResult::NeedsJumping); |
| } |
| } |
| |
| // When adding a new shim, you should follow the following pattern: |
| // ``` |
| // "shim_name" => { |
| // let [arg1, arg2, arg3] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; |
| // let result = this.shim_name(arg1, arg2, arg3)?; |
| // this.write_scalar(result, dest)?; |
| // } |
| // ``` |
| // and then define `shim_name` as a helper function in an extension trait in a suitable file |
| // (see e.g. `unix/fs.rs`): |
| // ``` |
| // fn shim_name( |
| // &mut self, |
| // arg1: &OpTy<'tcx, Provenance>, |
| // arg2: &OpTy<'tcx, Provenance>, |
| // arg3: &OpTy<'tcx, Provenance>, |
| // arg4: &OpTy<'tcx, Provenance>) |
| // -> InterpResult<'tcx, Scalar<Provenance>> { |
| // let this = self.eval_context_mut(); |
| // |
| // // First thing: load all the arguments. Details depend on the shim. |
| // let arg1 = this.read_scalar(arg1)?.to_u32()?; |
| // let arg2 = this.read_pointer(arg2)?; // when you need to work with the pointer directly |
| // let arg3 = this.deref_pointer_as(arg3, this.libc_ty_layout("some_libc_struct"))?; // when you want to load/store |
| // // through the pointer and supply the type information yourself |
| // let arg4 = this.deref_pointer(arg4)?; // when you want to load/store through the pointer and trust |
| // // the user-given type (which you shouldn't usually do) |
| // |
| // // ... |
| // |
| // Ok(Scalar::from_u32(42)) |
| // } |
| // ``` |
| // You might find existing shims not following this pattern, most |
| // likely because they predate it or because for some reason they cannot be made to fit. |
| |
| // Here we dispatch all the shims for foreign functions. If you have a platform specific |
| // shim, add it to the corresponding submodule. |
| match link_name.as_str() { |
| // Miri-specific extern functions |
| "miri_start_unwind" => { |
| // `check_shim` happens inside `handle_miri_start_unwind`. |
| this.handle_miri_start_unwind(abi, link_name, args, unwind)?; |
| return Ok(EmulateItemResult::AlreadyJumped); |
| } |
| "miri_run_provenance_gc" => { |
| let [] = this.check_shim(abi, Abi::Rust, link_name, args)?; |
| this.run_provenance_gc(); |
| } |
| "miri_get_alloc_id" => { |
| let [ptr] = this.check_shim(abi, Abi::Rust, link_name, args)?; |
| let ptr = this.read_pointer(ptr)?; |
| let (alloc_id, _, _) = this.ptr_get_alloc_id(ptr).map_err(|_e| { |
| err_machine_stop!(TerminationInfo::Abort(format!( |
| "pointer passed to `miri_get_alloc_id` must not be dangling, got {ptr:?}" |
| ))) |
| })?; |
| this.write_scalar(Scalar::from_u64(alloc_id.0.get()), dest)?; |
| } |
| "miri_print_borrow_state" => { |
| let [id, show_unnamed] = this.check_shim(abi, Abi::Rust, link_name, args)?; |
| let id = this.read_scalar(id)?.to_u64()?; |
| let show_unnamed = this.read_scalar(show_unnamed)?.to_bool()?; |
| if let Some(id) = std::num::NonZero::new(id) { |
| this.print_borrow_state(AllocId(id), show_unnamed)?; |
| } |
| } |
| "miri_pointer_name" => { |
| // This associates a name to a tag. Very useful for debugging, and also makes |
| // tests more strict. |
| let [ptr, nth_parent, name] = this.check_shim(abi, Abi::Rust, link_name, args)?; |
| let ptr = this.read_pointer(ptr)?; |
| let nth_parent = this.read_scalar(nth_parent)?.to_u8()?; |
| let name = this.read_immediate(name)?; |
| |
| let name = this.read_byte_slice(&name)?; |
| // We must make `name` owned because we need to |
| // end the shared borrow from `read_byte_slice` before we can |
| // start the mutable borrow for `give_pointer_debug_name`. |
| let name = String::from_utf8_lossy(name).into_owned(); |
| this.give_pointer_debug_name(ptr, nth_parent, &name)?; |
| } |
| "miri_static_root" => { |
| let [ptr] = this.check_shim(abi, Abi::Rust, link_name, args)?; |
| let ptr = this.read_pointer(ptr)?; |
| let (alloc_id, offset, _) = this.ptr_get_alloc_id(ptr)?; |
| if offset != Size::ZERO { |
| throw_unsup_format!( |
| "pointer passed to `miri_static_root` must point to beginning of an allocated block" |
| ); |
| } |
| this.machine.static_roots.push(alloc_id); |
| } |
| "miri_host_to_target_path" => { |
| let [ptr, out, out_size] = this.check_shim(abi, Abi::Rust, link_name, args)?; |
| let ptr = this.read_pointer(ptr)?; |
| let out = this.read_pointer(out)?; |
| let out_size = this.read_scalar(out_size)?.to_target_usize(this)?; |
| |
| // The host affects program behavior here, so this requires isolation to be disabled. |
| this.check_no_isolation("`miri_host_to_target_path`")?; |
| |
| // We read this as a plain OsStr and write it as a path, which will convert it to the target. |
| let path = this.read_os_str_from_c_str(ptr)?.to_owned(); |
| let (success, needed_size) = |
| this.write_path_to_c_str(Path::new(&path), out, out_size)?; |
| // Return value: 0 on success, otherwise the size it would have needed. |
| this.write_int(if success { 0 } else { needed_size }, dest)?; |
| } |
| // Obtains the size of a Miri backtrace. See the README for details. |
| "miri_backtrace_size" => { |
| this.handle_miri_backtrace_size(abi, link_name, args, dest)?; |
| } |
| // Obtains a Miri backtrace. See the README for details. |
| "miri_get_backtrace" => { |
| // `check_shim` happens inside `handle_miri_get_backtrace`. |
| this.handle_miri_get_backtrace(abi, link_name, args, dest)?; |
| } |
| // Resolves a Miri backtrace frame. See the README for details. |
| "miri_resolve_frame" => { |
| // `check_shim` happens inside `handle_miri_resolve_frame`. |
| this.handle_miri_resolve_frame(abi, link_name, args, dest)?; |
| } |
| // Writes the function and file names of a Miri backtrace frame into a user provided buffer. See the README for details. |
| "miri_resolve_frame_names" => { |
| this.handle_miri_resolve_frame_names(abi, link_name, args)?; |
| } |
| // Writes some bytes to the interpreter's stdout/stderr. See the |
| // README for details. |
| "miri_write_to_stdout" | "miri_write_to_stderr" => { |
| let [msg] = this.check_shim(abi, Abi::Rust, link_name, args)?; |
| let msg = this.read_immediate(msg)?; |
| let msg = this.read_byte_slice(&msg)?; |
| // Note: we're ignoring errors writing to host stdout/stderr. |
| let _ignore = match link_name.as_str() { |
| "miri_write_to_stdout" => std::io::stdout().write_all(msg), |
| "miri_write_to_stderr" => std::io::stderr().write_all(msg), |
| _ => unreachable!(), |
| }; |
| } |
| // Promises that a pointer has a given symbolic alignment. |
| "miri_promise_symbolic_alignment" => { |
| use rustc_target::abi::AlignFromBytesError; |
| |
| let [ptr, align] = this.check_shim(abi, Abi::Rust, link_name, args)?; |
| let ptr = this.read_pointer(ptr)?; |
| let align = this.read_target_usize(align)?; |
| if !align.is_power_of_two() { |
| throw_unsup_format!( |
| "`miri_promise_symbolic_alignment`: alignment must be a power of 2, got {align}" |
| ); |
| } |
| let align = Align::from_bytes(align).unwrap_or_else(|err| { |
| match err { |
| AlignFromBytesError::NotPowerOfTwo(_) => unreachable!(), |
| // When the alignment is a power of 2 but too big, clamp it to MAX. |
| AlignFromBytesError::TooLarge(_) => Align::MAX, |
| } |
| }); |
| let (_, addr) = ptr.into_parts(); // we know the offset is absolute |
| // Cannot panic since `align` is a power of 2 and hence non-zero. |
| if addr.bytes().checked_rem(align.bytes()).unwrap() != 0 { |
| throw_unsup_format!( |
| "`miri_promise_symbolic_alignment`: pointer is not actually aligned" |
| ); |
| } |
| if let Ok((alloc_id, offset, ..)) = this.ptr_try_get_alloc_id(ptr) { |
| let (_size, alloc_align, _kind) = this.get_alloc_info(alloc_id); |
| // If the newly promised alignment is bigger than the native alignment of this |
| // allocation, and bigger than the previously promised alignment, then set it. |
| if align > alloc_align |
| && !this |
| .machine |
| .symbolic_alignment |
| .get_mut() |
| .get(&alloc_id) |
| .is_some_and(|&(_, old_align)| align <= old_align) |
| { |
| this.machine.symbolic_alignment.get_mut().insert(alloc_id, (offset, align)); |
| } |
| } |
| } |
| |
| // Aborting the process. |
| "exit" => { |
| let [code] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; |
| let code = this.read_scalar(code)?.to_i32()?; |
| throw_machine_stop!(TerminationInfo::Exit { code: code.into(), leak_check: false }); |
| } |
| "abort" => { |
| let [] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; |
| throw_machine_stop!(TerminationInfo::Abort( |
| "the program aborted execution".to_owned() |
| )) |
| } |
| |
| // Standard C allocation |
| "malloc" => { |
| let [size] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; |
| let size = this.read_target_usize(size)?; |
| let res = this.malloc(size, /*zero_init:*/ false)?; |
| this.write_pointer(res, dest)?; |
| } |
| "calloc" => { |
| let [items, len] = |
| this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; |
| let items = this.read_target_usize(items)?; |
| let len = this.read_target_usize(len)?; |
| let size = items |
| .checked_mul(len) |
| .ok_or_else(|| err_ub_format!("overflow during calloc size computation"))?; |
| let res = this.malloc(size, /*zero_init:*/ true)?; |
| this.write_pointer(res, dest)?; |
| } |
| "free" => { |
| let [ptr] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; |
| let ptr = this.read_pointer(ptr)?; |
| this.free(ptr)?; |
| } |
| "realloc" => { |
| let [old_ptr, new_size] = |
| this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; |
| let old_ptr = this.read_pointer(old_ptr)?; |
| let new_size = this.read_target_usize(new_size)?; |
| let res = this.realloc(old_ptr, new_size)?; |
| this.write_pointer(res, dest)?; |
| } |
| |
| // Rust allocation |
| "__rust_alloc" | "miri_alloc" => { |
| let default = |this: &mut MiriInterpCx<'mir, 'tcx>| { |
| // Only call `check_shim` when `#[global_allocator]` isn't used. When that |
| // macro is used, we act like no shim exists, so that the exported function can run. |
| let [size, align] = this.check_shim(abi, Abi::Rust, link_name, args)?; |
| let size = this.read_target_usize(size)?; |
| let align = this.read_target_usize(align)?; |
| |
| check_alloc_request(size, align)?; |
| |
| let memory_kind = match link_name.as_str() { |
| "__rust_alloc" => MiriMemoryKind::Rust, |
| "miri_alloc" => MiriMemoryKind::Miri, |
| _ => unreachable!(), |
| }; |
| |
| let ptr = this.allocate_ptr( |
| Size::from_bytes(size), |
| Align::from_bytes(align).unwrap(), |
| memory_kind.into(), |
| )?; |
| |
| this.write_pointer(ptr, dest) |
| }; |
| |
| match link_name.as_str() { |
| "__rust_alloc" => return this.emulate_allocator(default), |
| "miri_alloc" => { |
| default(this)?; |
| return Ok(EmulateItemResult::NeedsJumping); |
| } |
| _ => unreachable!(), |
| } |
| } |
| "__rust_alloc_zeroed" => { |
| return this.emulate_allocator(|this| { |
| // See the comment for `__rust_alloc` why `check_shim` is only called in the |
| // default case. |
| let [size, align] = this.check_shim(abi, Abi::Rust, link_name, args)?; |
| let size = this.read_target_usize(size)?; |
| let align = this.read_target_usize(align)?; |
| |
| check_alloc_request(size, align)?; |
| |
| let ptr = this.allocate_ptr( |
| Size::from_bytes(size), |
| Align::from_bytes(align).unwrap(), |
| MiriMemoryKind::Rust.into(), |
| )?; |
| |
| // We just allocated this, the access is definitely in-bounds. |
| this.write_bytes_ptr( |
| ptr.into(), |
| iter::repeat(0u8).take(usize::try_from(size).unwrap()), |
| ) |
| .unwrap(); |
| this.write_pointer(ptr, dest) |
| }); |
| } |
| "__rust_dealloc" | "miri_dealloc" => { |
| let default = |this: &mut MiriInterpCx<'mir, 'tcx>| { |
| // See the comment for `__rust_alloc` why `check_shim` is only called in the |
| // default case. |
| let [ptr, old_size, align] = |
| this.check_shim(abi, Abi::Rust, link_name, args)?; |
| let ptr = this.read_pointer(ptr)?; |
| let old_size = this.read_target_usize(old_size)?; |
| let align = this.read_target_usize(align)?; |
| |
| let memory_kind = match link_name.as_str() { |
| "__rust_dealloc" => MiriMemoryKind::Rust, |
| "miri_dealloc" => MiriMemoryKind::Miri, |
| _ => unreachable!(), |
| }; |
| |
| // No need to check old_size/align; we anyway check that they match the allocation. |
| this.deallocate_ptr( |
| ptr, |
| Some((Size::from_bytes(old_size), Align::from_bytes(align).unwrap())), |
| memory_kind.into(), |
| ) |
| }; |
| |
| match link_name.as_str() { |
| "__rust_dealloc" => { |
| return this.emulate_allocator(default); |
| } |
| "miri_dealloc" => { |
| default(this)?; |
| return Ok(EmulateItemResult::NeedsJumping); |
| } |
| _ => unreachable!(), |
| } |
| } |
| "__rust_realloc" => { |
| return this.emulate_allocator(|this| { |
| // See the comment for `__rust_alloc` why `check_shim` is only called in the |
| // default case. |
| let [ptr, old_size, align, new_size] = |
| this.check_shim(abi, Abi::Rust, link_name, args)?; |
| let ptr = this.read_pointer(ptr)?; |
| let old_size = this.read_target_usize(old_size)?; |
| let align = this.read_target_usize(align)?; |
| let new_size = this.read_target_usize(new_size)?; |
| // No need to check old_size; we anyway check that they match the allocation. |
| |
| check_alloc_request(new_size, align)?; |
| |
| let align = Align::from_bytes(align).unwrap(); |
| let new_ptr = this.reallocate_ptr( |
| ptr, |
| Some((Size::from_bytes(old_size), align)), |
| Size::from_bytes(new_size), |
| align, |
| MiriMemoryKind::Rust.into(), |
| )?; |
| this.write_pointer(new_ptr, dest) |
| }); |
| } |
| |
| // C memory handling functions |
| "memcmp" => { |
| let [left, right, n] = |
| this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; |
| let left = this.read_pointer(left)?; |
| let right = this.read_pointer(right)?; |
| let n = Size::from_bytes(this.read_target_usize(n)?); |
| |
| // C requires that this must always be a valid pointer (C18 §7.1.4). |
| this.ptr_get_alloc_id(left)?; |
| this.ptr_get_alloc_id(right)?; |
| |
| let result = { |
| let left_bytes = this.read_bytes_ptr_strip_provenance(left, n)?; |
| let right_bytes = this.read_bytes_ptr_strip_provenance(right, n)?; |
| |
| use std::cmp::Ordering::*; |
| match left_bytes.cmp(right_bytes) { |
| Less => -1i32, |
| Equal => 0, |
| Greater => 1, |
| } |
| }; |
| |
| this.write_scalar(Scalar::from_i32(result), dest)?; |
| } |
| "memrchr" => { |
| let [ptr, val, num] = |
| this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; |
| let ptr = this.read_pointer(ptr)?; |
| let val = this.read_scalar(val)?.to_i32()?; |
| let num = this.read_target_usize(num)?; |
| // The docs say val is "interpreted as unsigned char". |
| #[allow(clippy::cast_sign_loss, clippy::cast_possible_truncation)] |
| let val = val as u8; |
| |
| // C requires that this must always be a valid pointer (C18 §7.1.4). |
| this.ptr_get_alloc_id(ptr)?; |
| |
| if let Some(idx) = this |
| .read_bytes_ptr_strip_provenance(ptr, Size::from_bytes(num))? |
| .iter() |
| .rev() |
| .position(|&c| c == val) |
| { |
| let idx = u64::try_from(idx).unwrap(); |
| #[allow(clippy::arithmetic_side_effects)] // idx < num, so this never wraps |
| let new_ptr = ptr.offset(Size::from_bytes(num - idx - 1), this)?; |
| this.write_pointer(new_ptr, dest)?; |
| } else { |
| this.write_null(dest)?; |
| } |
| } |
| "memchr" => { |
| let [ptr, val, num] = |
| this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; |
| let ptr = this.read_pointer(ptr)?; |
| let val = this.read_scalar(val)?.to_i32()?; |
| let num = this.read_target_usize(num)?; |
| // The docs say val is "interpreted as unsigned char". |
| #[allow(clippy::cast_sign_loss, clippy::cast_possible_truncation)] |
| let val = val as u8; |
| |
| // C requires that this must always be a valid pointer (C18 §7.1.4). |
| this.ptr_get_alloc_id(ptr)?; |
| |
| let idx = this |
| .read_bytes_ptr_strip_provenance(ptr, Size::from_bytes(num))? |
| .iter() |
| .position(|&c| c == val); |
| if let Some(idx) = idx { |
| let new_ptr = ptr.offset(Size::from_bytes(idx as u64), this)?; |
| this.write_pointer(new_ptr, dest)?; |
| } else { |
| this.write_null(dest)?; |
| } |
| } |
| "strlen" => { |
| let [ptr] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; |
| let ptr = this.read_pointer(ptr)?; |
| // This reads at least 1 byte, so we are already enforcing that this is a valid pointer. |
| let n = this.read_c_str(ptr)?.len(); |
| this.write_scalar( |
| Scalar::from_target_usize(u64::try_from(n).unwrap(), this), |
| dest, |
| )?; |
| } |
| "wcslen" => { |
| let [ptr] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; |
| let ptr = this.read_pointer(ptr)?; |
| // This reads at least 1 byte, so we are already enforcing that this is a valid pointer. |
| let n = this.read_wchar_t_str(ptr)?.len(); |
| this.write_scalar( |
| Scalar::from_target_usize(u64::try_from(n).unwrap(), this), |
| dest, |
| )?; |
| } |
| "memcpy" => { |
| let [ptr_dest, ptr_src, n] = |
| this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; |
| let ptr_dest = this.read_pointer(ptr_dest)?; |
| let ptr_src = this.read_pointer(ptr_src)?; |
| let n = this.read_target_usize(n)?; |
| |
| // C requires that this must always be a valid pointer, even if `n` is zero, so we better check that. |
| // (This is more than Rust requires, so `mem_copy` is not sufficient.) |
| this.ptr_get_alloc_id(ptr_dest)?; |
| this.ptr_get_alloc_id(ptr_src)?; |
| |
| this.mem_copy(ptr_src, ptr_dest, Size::from_bytes(n), true)?; |
| this.write_pointer(ptr_dest, dest)?; |
| } |
| "strcpy" => { |
| let [ptr_dest, ptr_src] = |
| this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; |
| let ptr_dest = this.read_pointer(ptr_dest)?; |
| let ptr_src = this.read_pointer(ptr_src)?; |
| |
| // We use `read_c_str` to determine the amount of data to copy, |
| // and then use `mem_copy` for the actual copy. This means |
| // pointer provenance is preserved by this implementation of `strcpy`. |
| // That is probably overly cautious, but there also is no fundamental |
| // reason to have `strcpy` destroy pointer provenance. |
| // This reads at least 1 byte, so we are already enforcing that this is a valid pointer. |
| let n = this.read_c_str(ptr_src)?.len().checked_add(1).unwrap(); |
| this.mem_copy(ptr_src, ptr_dest, Size::from_bytes(n), true)?; |
| this.write_pointer(ptr_dest, dest)?; |
| } |
| |
| // math functions (note that there are also intrinsics for some other functions) |
| #[rustfmt::skip] |
| | "cbrtf" |
| | "coshf" |
| | "sinhf" |
| | "tanf" |
| | "tanhf" |
| | "acosf" |
| | "asinf" |
| | "atanf" |
| | "log1pf" |
| | "expm1f" |
| | "tgammaf" |
| => { |
| let [f] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; |
| let f = this.read_scalar(f)?.to_f32()?; |
| // Using host floats (but it's fine, these operations do not have guaranteed precision). |
| let f_host = f.to_host(); |
| let res = match link_name.as_str() { |
| "cbrtf" => f_host.cbrt(), |
| "coshf" => f_host.cosh(), |
| "sinhf" => f_host.sinh(), |
| "tanf" => f_host.tan(), |
| "tanhf" => f_host.tanh(), |
| "acosf" => f_host.acos(), |
| "asinf" => f_host.asin(), |
| "atanf" => f_host.atan(), |
| "log1pf" => f_host.ln_1p(), |
| "expm1f" => f_host.exp_m1(), |
| "tgammaf" => f_host.gamma(), |
| _ => bug!(), |
| }; |
| let res = res.to_soft(); |
| let res = this.adjust_nan(res, &[f]); |
| this.write_scalar(res, dest)?; |
| } |
| #[rustfmt::skip] |
| | "_hypotf" |
| | "hypotf" |
| | "atan2f" |
| | "fdimf" |
| => { |
| let [f1, f2] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; |
| let f1 = this.read_scalar(f1)?.to_f32()?; |
| let f2 = this.read_scalar(f2)?.to_f32()?; |
| // underscore case for windows, here and below |
| // (see https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/floating-point-primitives?view=vs-2019) |
| // Using host floats (but it's fine, these operations do not have guaranteed precision). |
| let res = match link_name.as_str() { |
| "_hypotf" | "hypotf" => f1.to_host().hypot(f2.to_host()).to_soft(), |
| "atan2f" => f1.to_host().atan2(f2.to_host()).to_soft(), |
| #[allow(deprecated)] |
| "fdimf" => f1.to_host().abs_sub(f2.to_host()).to_soft(), |
| _ => bug!(), |
| }; |
| let res = this.adjust_nan(res, &[f1, f2]); |
| this.write_scalar(res, dest)?; |
| } |
| #[rustfmt::skip] |
| | "cbrt" |
| | "cosh" |
| | "sinh" |
| | "tan" |
| | "tanh" |
| | "acos" |
| | "asin" |
| | "atan" |
| | "log1p" |
| | "expm1" |
| | "tgamma" |
| => { |
| let [f] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; |
| let f = this.read_scalar(f)?.to_f64()?; |
| // Using host floats (but it's fine, these operations do not have guaranteed precision). |
| let f_host = f.to_host(); |
| let res = match link_name.as_str() { |
| "cbrt" => f_host.cbrt(), |
| "cosh" => f_host.cosh(), |
| "sinh" => f_host.sinh(), |
| "tan" => f_host.tan(), |
| "tanh" => f_host.tanh(), |
| "acos" => f_host.acos(), |
| "asin" => f_host.asin(), |
| "atan" => f_host.atan(), |
| "log1p" => f_host.ln_1p(), |
| "expm1" => f_host.exp_m1(), |
| "tgamma" => f_host.gamma(), |
| _ => bug!(), |
| }; |
| let res = res.to_soft(); |
| let res = this.adjust_nan(res, &[f]); |
| this.write_scalar(res, dest)?; |
| } |
| #[rustfmt::skip] |
| | "_hypot" |
| | "hypot" |
| | "atan2" |
| | "fdim" |
| => { |
| let [f1, f2] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; |
| let f1 = this.read_scalar(f1)?.to_f64()?; |
| let f2 = this.read_scalar(f2)?.to_f64()?; |
| // underscore case for windows, here and below |
| // (see https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/floating-point-primitives?view=vs-2019) |
| // Using host floats (but it's fine, these operations do not have guaranteed precision). |
| let res = match link_name.as_str() { |
| "_hypot" | "hypot" => f1.to_host().hypot(f2.to_host()).to_soft(), |
| "atan2" => f1.to_host().atan2(f2.to_host()).to_soft(), |
| #[allow(deprecated)] |
| "fdim" => f1.to_host().abs_sub(f2.to_host()).to_soft(), |
| _ => bug!(), |
| }; |
| let res = this.adjust_nan(res, &[f1, f2]); |
| this.write_scalar(res, dest)?; |
| } |
| #[rustfmt::skip] |
| | "_ldexp" |
| | "ldexp" |
| | "scalbn" |
| => { |
| let [x, exp] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; |
| // For radix-2 (binary) systems, `ldexp` and `scalbn` are the same. |
| let x = this.read_scalar(x)?.to_f64()?; |
| let exp = this.read_scalar(exp)?.to_i32()?; |
| |
| let res = x.scalbn(exp); |
| let res = this.adjust_nan(res, &[x]); |
| this.write_scalar(res, dest)?; |
| } |
| "lgammaf_r" => { |
| let [x, signp] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; |
| let x = this.read_scalar(x)?.to_f32()?; |
| let signp = this.deref_pointer(signp)?; |
| |
| // Using host floats (but it's fine, these operations do not have guaranteed precision). |
| let (res, sign) = x.to_host().ln_gamma(); |
| this.write_int(sign, &signp)?; |
| let res = this.adjust_nan(res.to_soft(), &[x]); |
| this.write_scalar(res, dest)?; |
| } |
| "lgamma_r" => { |
| let [x, signp] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; |
| let x = this.read_scalar(x)?.to_f64()?; |
| let signp = this.deref_pointer(signp)?; |
| |
| // Using host floats (but it's fine, these operations do not have guaranteed precision). |
| let (res, sign) = x.to_host().ln_gamma(); |
| this.write_int(sign, &signp)?; |
| let res = this.adjust_nan(res.to_soft(), &[x]); |
| this.write_scalar(res, dest)?; |
| } |
| |
| // LLVM intrinsics |
| "llvm.prefetch" => { |
| let [p, rw, loc, ty] = |
| this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; |
| |
| let _ = this.read_pointer(p)?; |
| let rw = this.read_scalar(rw)?.to_i32()?; |
| let loc = this.read_scalar(loc)?.to_i32()?; |
| let ty = this.read_scalar(ty)?.to_i32()?; |
| |
| if ty == 1 { |
| // Data cache prefetch. |
| // Notably, we do not have to check the pointer, this operation is never UB! |
| |
| if !matches!(rw, 0 | 1) { |
| throw_unsup_format!("invalid `rw` value passed to `llvm.prefetch`: {}", rw); |
| } |
| if !matches!(loc, 0..=3) { |
| throw_unsup_format!( |
| "invalid `loc` value passed to `llvm.prefetch`: {}", |
| loc |
| ); |
| } |
| } else { |
| throw_unsup_format!("unsupported `llvm.prefetch` type argument: {}", ty); |
| } |
| } |
| // Used to implement the x86 `_mm{,256,512}_popcnt_epi{8,16,32,64}` and wasm |
| // `{i,u}8x16_popcnt` functions. |
| name if name.starts_with("llvm.ctpop.v") => { |
| let [op] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; |
| |
| let (op, op_len) = this.operand_to_simd(op)?; |
| let (dest, dest_len) = this.mplace_to_simd(dest)?; |
| |
| assert_eq!(dest_len, op_len); |
| |
| for i in 0..dest_len { |
| let op = this.read_immediate(&this.project_index(&op, i)?)?; |
| // Use `to_uint` to get a zero-extended `u128`. Those |
| // extra zeros will not affect `count_ones`. |
| let res = op.to_scalar().to_uint(op.layout.size)?.count_ones(); |
| |
| this.write_scalar( |
| Scalar::from_uint(res, op.layout.size), |
| &this.project_index(&dest, i)?, |
| )?; |
| } |
| } |
| |
| // Target-specific shims |
| name if name.starts_with("llvm.x86.") |
| && (this.tcx.sess.target.arch == "x86" |
| || this.tcx.sess.target.arch == "x86_64") => |
| { |
| return shims::x86::EvalContextExt::emulate_x86_intrinsic( |
| this, link_name, abi, args, dest, |
| ); |
| } |
| // FIXME: Move these to an `arm` submodule. |
| "llvm.aarch64.isb" if this.tcx.sess.target.arch == "aarch64" => { |
| let [arg] = this.check_shim(abi, Abi::Unadjusted, link_name, args)?; |
| let arg = this.read_scalar(arg)?.to_i32()?; |
| match arg { |
| // SY ("full system scope") |
| 15 => { |
| this.yield_active_thread(); |
| } |
| _ => { |
| throw_unsup_format!("unsupported llvm.aarch64.isb argument {}", arg); |
| } |
| } |
| } |
| "llvm.arm.hint" if this.tcx.sess.target.arch == "arm" => { |
| let [arg] = this.check_shim(abi, Abi::Unadjusted, link_name, args)?; |
| let arg = this.read_scalar(arg)?.to_i32()?; |
| // Note that different arguments might have different target feature requirements. |
| match arg { |
| // YIELD |
| 1 => { |
| this.expect_target_feature_for_intrinsic(link_name, "v6")?; |
| this.yield_active_thread(); |
| } |
| _ => { |
| throw_unsup_format!("unsupported llvm.arm.hint argument {}", arg); |
| } |
| } |
| } |
| |
| // Platform-specific shims |
| _ => |
| return match this.tcx.sess.target.os.as_ref() { |
| _ if this.target_os_is_unix() => |
| shims::unix::foreign_items::EvalContextExt::emulate_foreign_item_inner( |
| this, link_name, abi, args, dest, |
| ), |
| "windows" => |
| shims::windows::foreign_items::EvalContextExt::emulate_foreign_item_inner( |
| this, link_name, abi, args, dest, |
| ), |
| _ => Ok(EmulateItemResult::NotSupported), |
| }, |
| }; |
| // We only fall through to here if we did *not* hit the `_` arm above, |
| // i.e., if we actually emulated the function with one of the shims. |
| Ok(EmulateItemResult::NeedsJumping) |
| } |
| } |
