blob: c1a9926d7ce2ce1f40d00763ff7370f9fc87b547 [file]
// Copyright 2025 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
use crate::mm::{MemoryManager, PAGE_SIZE};
use bitflags::bitflags;
use range_map::RangeMap;
use starnix_logging::track_stub;
use starnix_sync::{LockDepMutex, UserFaultInner};
use starnix_uapi::errors::Errno;
use starnix_uapi::user_address::UserAddress;
use starnix_uapi::{
UFFD_FEATURE_EVENT_FORK, UFFD_FEATURE_EVENT_REMAP, UFFD_FEATURE_EVENT_REMOVE,
UFFD_FEATURE_EVENT_UNMAP, UFFD_FEATURE_MINOR_HUGETLBFS, UFFD_FEATURE_MINOR_SHMEM,
UFFD_FEATURE_MISSING_HUGETLBFS, UFFD_FEATURE_MISSING_SHMEM, UFFD_FEATURE_SIGBUS,
UFFD_FEATURE_THREAD_ID, UFFDIO_CONTINUE_MODE_DONTWAKE, UFFDIO_COPY_MODE_DONTWAKE,
UFFDIO_COPY_MODE_WP, UFFDIO_REGISTER_MODE_MINOR, UFFDIO_REGISTER_MODE_MISSING,
UFFDIO_REGISTER_MODE_WP, UFFDIO_ZEROPAGE_MODE_DONTWAKE, errno, error,
};
use std::ops::Range;
use std::sync::{Arc, Weak};
#[derive(Debug)]
pub struct UserFault {
mm: Weak<MemoryManager>,
state: LockDepMutex<UserFaultState, UserFaultInner>,
}
#[derive(Debug, Clone)]
struct UserFaultState {
/// If initialized, contains features that this userfault was initialized with
features: Option<UserFaultFeatures>,
/// Pages that are currently registered with this userfault object, and whether they are
/// already populated.
userfault_pages: RangeMap<UserAddress, bool>,
}
impl UserFault {
pub fn new(mm: Weak<MemoryManager>) -> Self {
Self { mm, state: LockDepMutex::new(UserFaultState::new()) }
}
pub fn insert_pages(&self, range: Range<UserAddress>, value: bool) {
// RangeMap uses #[must_use] for its default usecase but this drop is trivial.
let _ = self.state.lock().userfault_pages.insert(range, value);
}
pub fn remove_pages(&self, range: Range<UserAddress>) -> bool {
!self.state.lock().userfault_pages.remove(range).is_empty()
}
pub fn get_registered_pages_overlapping_range(
&self,
range: Range<UserAddress>,
) -> Vec<Range<UserAddress>> {
self.state.lock().userfault_pages.get_keys(range).cloned().collect()
}
pub fn contains_addr(&self, addr: UserAddress) -> bool {
self.state.lock().userfault_pages.get(addr).is_some()
}
pub fn get_first_populated_page_after(&self, addr: UserAddress) -> Option<UserAddress> {
self.state.lock().userfault_pages.get(addr).map(|(affected_range, is_populated)| {
if *is_populated { addr } else { affected_range.end }
})
}
pub fn is_initialized(self: &Arc<Self>) -> bool {
self.state.lock().features.is_some()
}
pub fn has_features(self: &Arc<Self>, features: UserFaultFeatures) -> bool {
self.state.lock().features.map(|f| f.contains(features)).unwrap_or(false)
}
pub fn initialize(self: &Arc<Self>, features: UserFaultFeatures) {
self.state.lock().features = Some(features);
}
pub fn op_register(
self: &Arc<Self>,
start: UserAddress,
len: u64,
mode: FaultRegisterMode,
) -> Result<SupportedUserFaultIoctls, Errno> {
if !self.is_initialized() {
return error!(EINVAL);
}
if !self.has_features(UserFaultFeatures::SIGBUS) {
track_stub!(TODO("https://fxbug.dev/391599171"), "userfault without SIGBUS feature");
return error!(ENOTSUP);
}
check_op_range(start, len)?;
let mm = self.mm.upgrade().ok_or_else(|| errno!(EINVAL))?;
mm.register_with_uffd(start, len as usize, self, mode)?;
Ok(SupportedUserFaultIoctls::COPY | SupportedUserFaultIoctls::ZERO_PAGE)
}
pub fn op_unregister(self: &Arc<Self>, start: UserAddress, len: u64) -> Result<(), Errno> {
if !self.is_initialized() {
return error!(EINVAL);
}
check_op_range(start, len)?;
let mm = self.mm.upgrade().ok_or_else(|| errno!(EINVAL))?;
mm.unregister_range_from_uffd(self, start, len as usize)
}
pub fn op_copy(
self: &Arc<Self>,
mm_source: &MemoryManager,
source: UserAddress,
dest: UserAddress,
len: u64,
_mode: FaultCopyMode,
) -> Result<usize, Errno> {
if !self.is_initialized() {
return error!(EINVAL);
}
check_op_range(source, len)?;
check_op_range(dest, len)?;
let mm = self.mm.upgrade().ok_or_else(|| errno!(EINVAL))?;
// If the copy happens inside the same process, do it inside this process' memory manager
// so that the lock is held throughout the operation.
if Arc::as_ptr(&mm) == mm_source as *const MemoryManager {
mm.copy_from_uffd(source, dest, len as usize, self)
} else {
let mut buf = vec![std::mem::MaybeUninit::uninit(); len as usize];
let buf = mm_source.syscall_read_memory(source, &mut buf)?;
mm.fill_from_uffd(dest, buf, len as usize, self)
}
}
pub fn op_zero(
self: &Arc<Self>,
start: UserAddress,
len: u64,
_mode: FaultZeroMode,
) -> Result<usize, Errno> {
if !self.is_initialized() {
return error!(EINVAL);
}
check_op_range(start, len)?;
let mm = self.mm.upgrade().ok_or_else(|| errno!(EINVAL))?;
mm.zero_from_uffd(start, len as usize, self)
}
pub fn cleanup(self: &Arc<Self>) {
if let Some(mm) = self.mm.upgrade() {
mm.unregister_uffd(self);
}
}
}
impl UserFaultState {
pub fn new() -> Self {
Self { features: None, userfault_pages: RangeMap::default() }
}
}
bitflags! {
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
pub struct UserFaultFeatures: u32 {
const ALL_SUPPORTED = UFFD_FEATURE_SIGBUS;
const EVENT_FORK = UFFD_FEATURE_EVENT_FORK;
const EVENT_REMAP = UFFD_FEATURE_EVENT_REMAP;
const EVENT_REMOVE = UFFD_FEATURE_EVENT_REMOVE;
const EVENT_UNMAP = UFFD_FEATURE_EVENT_UNMAP;
const MISSING_HUGETLBFS = UFFD_FEATURE_MISSING_HUGETLBFS;
const MISSING_SHMEM = UFFD_FEATURE_MISSING_SHMEM;
const SIGBUS = UFFD_FEATURE_SIGBUS;
const THREAD_ID = UFFD_FEATURE_THREAD_ID;
const MINOR_HUGETLBFS = UFFD_FEATURE_MINOR_HUGETLBFS;
const MINOR_SHMEM = UFFD_FEATURE_MINOR_SHMEM;
}
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
pub struct FaultRegisterMode: u32 {
const MINOR = UFFDIO_REGISTER_MODE_MINOR;
const MISSING = UFFDIO_REGISTER_MODE_MISSING;
const WRITE_PROTECT = UFFDIO_REGISTER_MODE_WP;
}
pub struct FaultCopyMode: u32 {
const DONT_WAKE = UFFDIO_COPY_MODE_DONTWAKE;
const WRITE_PROTECT = UFFDIO_COPY_MODE_WP;
}
pub struct FaultZeroMode: u32 {
const DONT_WAKE = UFFDIO_ZEROPAGE_MODE_DONTWAKE;
}
pub struct FaultContinueMode: u32 {
const DONT_WAKE = UFFDIO_CONTINUE_MODE_DONTWAKE;
}
pub struct SupportedUserFaultIoctls: u64 {
const COPY = 1 << starnix_uapi::_UFFDIO_COPY;
const WAKE = 1 << starnix_uapi::_UFFDIO_WAKE;
const WRITE_PROTECT = 1 << starnix_uapi::_UFFDIO_WRITEPROTECT;
const ZERO_PAGE = 1 << starnix_uapi::_UFFDIO_ZEROPAGE;
const CONTINUE = 1 << starnix_uapi::_UFFDIO_CONTINUE;
}
}
fn check_op_range(addr: UserAddress, len: u64) -> Result<(), Errno> {
if addr.is_aligned(*PAGE_SIZE) && len % *PAGE_SIZE == 0 && len > 0 {
Ok(())
} else {
error!(EINVAL)
}
}