diff --git a/src/context/memory.rs b/src/context/memory.rs index 1c72958..fd6315c 100644 --- a/src/context/memory.rs +++ b/src/context/memory.rs @@ -70,7 +70,6 @@ pub struct AddrSpace { } impl AddrSpace { /// Attempt to clone an existing address space so that all mappings are copied (CoW). - // TODO: Actually use CoW! pub fn try_clone(&self) -> Result<(PtId, Arc>)> { let (id, mut new) = new_addrspace()?; @@ -87,20 +86,27 @@ impl AddrSpace { let mut new_mapper = unsafe { InactivePageTable::from_address(new.read().frame.utable.start_address().data()) }; for grant in self.grants.iter() { - // TODO: Fail if there are borrowed grants, rather than simply ignoring them? - if !grant.is_owned() { continue; } + if grant.desc_opt.is_some() { continue; } - let new_grant = Grant::zeroed(Page::containing_address(grant.start_address()), grant.size() / PAGE_SIZE, grant.flags(), &mut new_mapper.mapper(), ())?; + let new_grant; - for page in new_grant.pages() { - // FIXME: ENOMEM is wrong here, it cannot fail. - let current_frame = unsafe { RmmA::phys_to_virt(this_mapper.translate_page(page).ok_or(Error::new(ENOMEM))?.start_address()) }.data() as *const u8; - let new_frame = unsafe { RmmA::phys_to_virt(new_mapper.mapper().translate_page(page).ok_or(Error::new(ENOMEM))?.start_address()) }.data() as *mut u8; + // TODO: Replace this with CoW + if grant.owned { + new_grant = Grant::zeroed(Page::containing_address(grant.start_address()), grant.size() / PAGE_SIZE, grant.flags(), &mut new_mapper.mapper(), ())?; - // TODO: Replace this with CoW - unsafe { - new_frame.copy_from_nonoverlapping(current_frame, PAGE_SIZE); + for page in new_grant.pages() { + let current_frame = unsafe { RmmA::phys_to_virt(this_mapper.translate_page(page).expect("grant containing unmapped pages").start_address()) }.data() as *const u8; + let new_frame = unsafe { RmmA::phys_to_virt(new_mapper.mapper().translate_page(page).expect("grant containing unmapped pages").start_address()) }.data() as *mut u8; + + unsafe { + new_frame.copy_from_nonoverlapping(current_frame, PAGE_SIZE); + } } + } else { + // TODO: Remove reborrow? In that case, physmapped memory will need to either be + // remapped when cloning, or be backed by a file descriptor (like + // `memory:physical`). + new_grant = Grant::reborrow(&grant, Page::containing_address(grant.start_address()), &mut this_mapper, &mut new_mapper.mapper(), ()); } new.write().grants.insert(new_grant); @@ -508,35 +514,54 @@ impl Grant { } Ok(Grant { region: Region { start: dst.start_address(), size: page_count * PAGE_SIZE }, flags, mapped: true, owned: true, desc_opt: None }) } + pub fn borrow(src_base: Page, dst_base: Page, page_count: usize, flags: PageFlags, desc_opt: Option, src_mapper: &mut Mapper, dst_mapper: &mut Mapper, dst_flusher: impl Flusher) -> Grant { + Self::copy_inner(src_base, dst_base, page_count, flags, desc_opt, src_mapper, dst_mapper, (), dst_flusher, false, false) + } + pub fn reborrow(src_grant: &Grant, dst_base: Page, src_mapper: &mut Mapper, dst_mapper: &mut Mapper, dst_flusher: impl Flusher) -> Grant { + Self::borrow(Page::containing_address(src_grant.start_address()), dst_base, src_grant.size() / PAGE_SIZE, src_grant.flags(), src_grant.desc_opt.clone(), src_mapper, dst_mapper, dst_flusher) + } + pub fn transfer(mut src_grant: Grant, dst_base: Page, src_mapper: &mut Mapper, dst_mapper: &mut Mapper, src_flusher: impl Flusher, dst_flusher: impl Flusher) -> Grant { + assert!(core::mem::replace(&mut src_grant.mapped, false)); + let desc_opt = src_grant.desc_opt.take(); - pub fn map_inactive(src: VirtualAddress, dst: VirtualAddress, size: usize, flags: PageFlags, desc_opt: Option, inactive_table: &mut InactivePageTable) -> Grant { - let active_table = unsafe { ActivePageTable::new(src.kind()) }; - let mut inactive_mapper = inactive_table.mapper(); + Self::copy_inner(Page::containing_address(src_grant.start_address()), dst_base, src_grant.size() / PAGE_SIZE, src_grant.flags(), desc_opt, src_mapper, dst_mapper, src_flusher, dst_flusher, src_grant.owned, true) + } - let src_start_page = Page::containing_address(src); - let src_end_page = Page::containing_address(VirtualAddress::new(src.data() + size - 1)); - let src_range = Page::range_inclusive(src_start_page, src_end_page); + fn copy_inner( + src_base: Page, + dst_base: Page, + page_count: usize, + flags: PageFlags, + desc_opt: Option, + src_mapper: &mut Mapper, + dst_mapper: &mut Mapper, + mut src_flusher: impl Flusher, + mut dst_flusher: impl Flusher, + owned: bool, + unmap: bool, + ) -> Grant { + for index in 0..page_count { + let src_page = src_base.next_by(index); + let frame = if unmap { + let (flush, frame) = src_mapper.unmap_return(src_page, false); + src_flusher.consume(flush); + frame + } else { + src_mapper.translate_page(src_page).expect("grant references unmapped memory") + }; - let dst_start_page = Page::containing_address(dst); - let dst_end_page = Page::containing_address(VirtualAddress::new(dst.data() + size - 1)); - let dst_range = Page::range_inclusive(dst_start_page, dst_end_page); - - for (src_page, dst_page) in src_range.zip(dst_range) { - let frame = active_table.translate_page(src_page).expect("grant references unmapped memory"); - - let inactive_flush = inactive_mapper.map_to(dst_page, frame, flags); - // Ignore result due to mapping on inactive table - unsafe { inactive_flush.ignore(); } + let flush = dst_mapper.map_to(dst_base.next_by(index), frame, flags); + dst_flusher.consume(flush); } Grant { region: Region { - start: dst, - size, + start: dst_base.start_address(), + size: page_count * PAGE_SIZE, }, flags, - mapped: true, - owned: false, + mapped: !unmap, + owned, desc_opt, } } diff --git a/src/scheme/proc.rs b/src/scheme/proc.rs index 2d0c3cc..1441bf6 100644 --- a/src/scheme/proc.rs +++ b/src/scheme/proc.rs @@ -1,6 +1,6 @@ use crate::{ arch::paging::{ActivePageTable, Flusher, InactivePageTable, mapper::{InactiveFlusher, Mapper, PageFlushAll}, Page, RmmA, VirtualAddress}, - context::{self, Context, ContextId, Status, file::FileDescriptor, memory::{AddrSpace, Grant, new_addrspace, PtId, page_flags, Region}}, + context::{self, Context, ContextId, Status, file::{FileDescription, FileDescriptor}, memory::{AddrSpace, Grant, new_addrspace, PtId, page_flags, Region}}, memory::PAGE_SIZE, ptrace, scheme::{self, AtomicSchemeId, FileHandle, KernelScheme, SchemeId}, @@ -121,8 +121,14 @@ enum Operation { AddrSpace { addrspace: Arc> }, CurrentAddrSpace, CurrentFiletable, - // TODO: Any better interface to access newly created contexts? Openat? + // TODO: Remove this once openat is implemented, or allow openat-via-dup via e.g. the top-level + // directory. OpenViaDup, + // Allows calling fmap directly on a FileDescriptor (as opposed to a FileDescriptor). + // + // TODO: Remove this once cross-scheme links are merged. That would allow acquiring a new + // FD to access the file descriptor behind grants. + GrantHandle { description: Arc> }, } #[derive(Clone, Copy, PartialEq, Eq)] enum Attr { @@ -166,6 +172,7 @@ enum OperationData { Memory(MemData), Trace(TraceData), Static(StaticData), + Offset(usize), Other, } impl OperationData { @@ -298,6 +305,7 @@ impl ProcScheme { Operation::Static(_) => OperationData::Static(StaticData::new( target.name.read().clone().into() )), + Operation::AddrSpace { .. } => OperationData::Offset(0), _ => OperationData::Other, }; @@ -426,12 +434,19 @@ impl Scheme for ProcScheme { } } Operation::AddrSpace { addrspace } => { - let (new_addrspace, is_mem) = match buf { + let (operation, is_mem) = match buf { // TODO: Better way to obtain new empty address spaces, perhaps using SYS_OPEN. But // in that case, what scheme? - b"empty" => (new_addrspace()?.1, false), - b"exclusive" => (addrspace.read().try_clone()?.1, false), - b"mem" => (Arc::clone(&addrspace), true), + b"empty" => (Operation::AddrSpace { addrspace: new_addrspace()?.1 }, false), + b"exclusive" => (Operation::AddrSpace { addrspace: addrspace.read().try_clone()?.1 }, false), + b"mem" => (Operation::Memory { addrspace: Arc::clone(&addrspace) }, true), + + grant_handle if grant_handle.starts_with(b"grant-") => { + let start_addr = usize::from_str_radix(core::str::from_utf8(&grant_handle[6..]).map_err(|_| Error::new(EINVAL))?, 16).map_err(|_| Error::new(EINVAL))?; + (Operation::GrantHandle { + description: Arc::clone(&addrspace.read().grants.contains(VirtualAddress::new(start_addr)).ok_or(Error::new(EINVAL))?.desc_opt.as_ref().ok_or(Error::new(EINVAL))?.desc.description) + }, false) + } _ => return Err(Error::new(EINVAL)), }; @@ -439,9 +454,9 @@ impl Scheme for ProcScheme { info: Info { flags: 0, pid: info.pid, - operation: if is_mem { Operation::Memory { addrspace: new_addrspace } } else { Operation::AddrSpace { addrspace: new_addrspace } }, + operation, }, - data: if is_mem { OperationData::Memory(MemData { offset: VirtualAddress::new(0) }) } else { OperationData::Other }, + data: if is_mem { OperationData::Memory(MemData { offset: VirtualAddress::new(0) }) } else { OperationData::Offset(0) }, } } _ => return Err(Error::new(EINVAL)), @@ -504,8 +519,35 @@ impl Scheme for ProcScheme { data.offset = VirtualAddress::new(data.offset.data() + bytes_read); Ok(bytes_read) }, - // TODO: Support querying which grants exist and where - Operation::AddrSpace { .. } => return Err(Error::new(EBADF)), + // TODO: Support reading only a specific address range. Maybe using seek? + Operation::AddrSpace { addrspace } => { + let mut handles = self.handles.write(); + let offset = if let OperationData::Offset(ref mut offset) = handles.get_mut(&id).ok_or(Error::new(EBADF))?.data { + offset + } else { + return Err(Error::new(EBADFD)); + }; + + // TODO: Define a struct somewhere? + const RECORD_SIZE: usize = mem::size_of::() * 4; + let start = core::cmp::min(buf.len(), *offset); + let records = buf[start..].array_chunks_mut::(); + + let addrspace = addrspace.read(); + let mut bytes_read = 0; + + for (record_bytes, grant) in records.zip(addrspace.grants.iter()).skip(*offset / RECORD_SIZE) { + let mut qwords = record_bytes.array_chunks_mut::<{mem::size_of::()}>(); + qwords.next().unwrap().copy_from_slice(&usize::to_ne_bytes(grant.start_address().data())); + qwords.next().unwrap().copy_from_slice(&usize::to_ne_bytes(grant.size())); + qwords.next().unwrap().copy_from_slice(&usize::to_ne_bytes(grant.flags().data() | if grant.desc_opt.is_some() { 0x8000_0000 } else { 0 })); + qwords.next().unwrap().copy_from_slice(&usize::to_ne_bytes(grant.desc_opt.as_ref().map_or(0, |d| d.offset))); + bytes_read += RECORD_SIZE; + } + + *offset += bytes_read; + Ok(bytes_read) + } Operation::Regs(kind) => { union Output { @@ -641,7 +683,7 @@ impl Scheme for ProcScheme { // the instruction and stack pointer. Maybe remove `/regs` altogether and replace it // with `/ctx` Operation::CurrentAddrSpace | Operation::CurrentFiletable => return Err(Error::new(EBADF)), - Operation::OpenViaDup => return Err(Error::new(EBADF)), + Operation::OpenViaDup | Operation::GrantHandle { .. } => return Err(Error::new(EBADF)), } } @@ -695,7 +737,7 @@ impl Scheme for ProcScheme { let base = chunks.next().ok_or(Error::new(EINVAL))?; let size = chunks.next().ok_or(Error::new(EINVAL))?; let flags = chunks.next().and_then(|f| MapFlags::from_bits(f)).ok_or(Error::new(EINVAL))?; - let region = Region::new(VirtualAddress::new(base), size); + let src_address = chunks.next(); if base % PAGE_SIZE != 0 || size % PAGE_SIZE != 0 || base.saturating_add(size) > crate::USER_END_OFFSET { return Err(Error::new(EINVAL)); @@ -707,8 +749,6 @@ impl Scheme for ProcScheme { let callback = |addr_space: &mut AddrSpace| { let (mut inactive, mut active); - //let mut addr_space = context.addr_space()?.write(); - let (mut mapper, mut flusher) = if is_active { active = (unsafe { ActivePageTable::new(rmm::TableKind::User) }, PageFlushAll::new()); (active.0.mapper(), &mut active.1 as &mut dyn Flusher) @@ -717,6 +757,34 @@ impl Scheme for ProcScheme { (inactive.0.mapper(), &mut inactive.1 as &mut dyn Flusher) }; + if let Some(src_address) = src_address { + // Forbid transferring grants to the same address space! + if is_active { return Err(Error::new(EBUSY)); } + + let src_grant = current_addrspace()?.write().grants.take(&Region::new(VirtualAddress::new(src_address), size)).ok_or(Error::new(EINVAL))?; + + if src_address % PAGE_SIZE != 0 || src_address.saturating_add(size) > crate::USER_END_OFFSET { + return Err(Error::new(EINVAL)); + } + + // TODO: Allow downgrading flags? + + if let Some(grant) = addr_space.grants.conflicts(Region::new(VirtualAddress::new(base), size)).next() { + return Err(Error::new(EEXIST)); + } + + addr_space.grants.insert(Grant::transfer( + src_grant, + Page::containing_address(VirtualAddress::new(base)), + &mut *unsafe { ActivePageTable::new(rmm::TableKind::User) }, + &mut mapper, + PageFlushAll::new(), + flusher, + )); + return Ok(()); + } + + let region = Region::new(VirtualAddress::new(base), size); let conflicting = addr_space.grants.conflicts(region).map(|g| *g.region()).collect::>(); for conflicting_region in conflicting { let whole_grant = addr_space.grants.take(&conflicting_region).ok_or(Error::new(EBADFD))?; @@ -939,15 +1007,15 @@ impl Scheme for ProcScheme { let mut filetable = hopefully_this_scheme.as_filetable(number)?; - try_stop_context(info.pid, |context| { + let stopper = if info.pid == context::context_id() { with_context_mut } else { try_stop_context }; + + stopper(info.pid, |context: &mut Context| { context.files = filetable; Ok(()) })?; Ok(mem::size_of::()) } Operation::CurrentAddrSpace { .. } => { - println!("Setting current address space! ({} {})", info.pid.into(), context::context_id().into()); - let mut iter = buf.array_chunks::<{mem::size_of::()}>().copied().map(usize::from_ne_bytes); let addrspace_fd = iter.next().ok_or(Error::new(EINVAL))?; let sp = iter.next().ok_or(Error::new(EINVAL))?; @@ -974,7 +1042,7 @@ impl Scheme for ProcScheme { } Ok(3 * mem::size_of::()) } - Operation::OpenViaDup => return Err(Error::new(EBADF)), + Operation::OpenViaDup | Operation::GrantHandle { .. } => return Err(Error::new(EBADF)), } } @@ -1022,6 +1090,8 @@ impl Scheme for ProcScheme { Operation::CurrentAddrSpace => "current-addrspace", Operation::CurrentFiletable => "current-filetable", Operation::OpenViaDup => "open-via-dup", + + Operation::GrantHandle { .. } => return Err(Error::new(EOPNOTSUPP)), }); read_from(buf, &path.as_bytes(), &mut 0) @@ -1067,6 +1137,20 @@ impl Scheme for ProcScheme { } Ok(0) } + // TODO: Support borrowing someone else's memory. + fn fmap(&self, id: usize, map: &syscall::data::Map) -> Result { + let description_lock = match self.handles.read().get(&id) { + Some(Handle { info: Info { operation: Operation::GrantHandle { ref description }, .. }, .. }) => Arc::clone(description), + _ => return Err(Error::new(EBADF)), + }; + let (scheme_id, number) = { + let description = description_lock.read(); + + (description.scheme, description.number) + }; + let scheme = Arc::clone(scheme::schemes().get(scheme_id).ok_or(Error::new(EBADFD))?); + scheme.fmap(number, map) + } } impl KernelScheme for ProcScheme { fn as_addrspace(&self, number: usize) -> Result>> { @@ -1077,10 +1161,11 @@ impl KernelScheme for ProcScheme { } } fn as_filetable(&self, number: usize) -> Result>>>> { - if !matches!(self.handles.read().get(&number).ok_or(Error::new(EBADF))?.info.operation, Operation::Filetable { .. }) { - return Err(Error::new(EBADF)); + if let Operation::Filetable { ref filetable } = self.handles.read().get(&number).ok_or(Error::new(EBADF))?.info.operation { + Ok(Arc::clone(filetable)) + } else { + Err(Error::new(EBADF)) } - Ok(Arc::clone(&context::contexts().current().ok_or(Error::new(ESRCH))?.read().files)) } } extern "C" fn clone_handler() { diff --git a/src/scheme/user.rs b/src/scheme/user.rs index c87e694..96101db 100644 --- a/src/scheme/user.rs +++ b/src/scheme/user.rs @@ -8,9 +8,9 @@ use spin::{Mutex, RwLock}; use crate::context::{self, Context}; use crate::context::file::FileDescriptor; -use crate::context::memory::{DANGLING, page_flags, round_down_pages, Grant, Region, GrantFileRef}; +use crate::context::memory::{DANGLING, page_flags, round_down_pages, round_up_pages, Grant, Region, GrantFileRef}; use crate::event; -use crate::paging::{PAGE_SIZE, InactivePageTable, VirtualAddress}; +use crate::paging::{ActivePageTable, PAGE_SIZE, InactivePageTable, mapper::InactiveFlusher, Page, VirtualAddress}; use crate::scheme::{AtomicSchemeId, SchemeId}; use crate::sync::{WaitQueue, WaitMap}; use crate::syscall::data::{Map, Packet, Stat, StatVfs, TimeSpec}; @@ -123,6 +123,9 @@ impl UserInner { ).map(|addr| addr.data()) } + // TODO: Use an address space Arc over a context Arc. While contexts which share address spaces + // still can access borrowed scheme pages, it would both be cleaner and would handle the case + // where the initial context is closed. fn capture_inner(context_weak: &Weak>, dst_address: usize, address: usize, size: usize, flags: MapFlags, desc_opt: Option) -> Result { // TODO: More abstractions over grant creation! @@ -148,18 +151,21 @@ impl UserInner { let mut addr_space = context.addr_space()?.write(); let src_address = round_down_pages(address); + let dst_address = round_down_pages(dst_address); let offset = address - src_address; - let src_region = Region::new(VirtualAddress::new(src_address), offset + size).round(); - let dst_region = addr_space.grants.find_free_at(VirtualAddress::new(dst_address), src_region.size(), flags)?; + let aligned_size = round_up_pages(offset + size); + let dst_region = addr_space.grants.find_free_at(VirtualAddress::new(dst_address), aligned_size, flags)?; //TODO: Use syscall_head and syscall_tail to avoid leaking data - addr_space.grants.insert(Grant::map_inactive( - src_region.start_address(), - dst_region.start_address(), - src_region.size(), + addr_space.grants.insert(Grant::borrow( + Page::containing_address(VirtualAddress::new(src_address)), + Page::containing_address(dst_region.start_address()), + aligned_size / PAGE_SIZE, page_flags(flags), desc_opt, - &mut new_table, + &mut *unsafe { ActivePageTable::new(rmm::TableKind::User) }, + &mut new_table.mapper(), + InactiveFlusher::new(), )); Ok(VirtualAddress::new(dst_region.start_address().data() + offset)) diff --git a/src/syscall/debug.rs b/src/syscall/debug.rs index 993575f..717e1c3 100644 --- a/src/syscall/debug.rs +++ b/src/syscall/debug.rs @@ -170,10 +170,6 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize) - b, validate_slice_mut(c as *mut TimeSpec, 1) ), - SYS_CLONE => format!( - "clone({:?})", - CloneFlags::from_bits(b) - ), SYS_EXIT => format!( "exit({})", b diff --git a/src/syscall/process.rs b/src/syscall/process.rs index 418f464..1a1e725 100644 --- a/src/syscall/process.rs +++ b/src/syscall/process.rs @@ -652,7 +652,6 @@ pub fn usermode_bootstrap(mut data: Box<[u8]>) -> ! { .expect("expected bootstrap context to have an address space") .write().grants.insert(grant); } - log::info!("Usermode bootstrap"); drop(data);