Fix multi_core livelocks and add livelock debugging

This commit is contained in:
Jeremy Soller
2020-04-18 08:23:40 -06:00
parent 9d67e3dc28
commit fdf46d8043
16 changed files with 99 additions and 48 deletions

View File

@@ -31,7 +31,7 @@ version = "0.29.0"
default-features = false
[features]
default = ["serial_debug"]
default = ["acpi", "multi_core", "serial_debug"]
acpi = []
doc = []
graphical_debug = []

View File

@@ -118,6 +118,7 @@ pub struct Context {
pub umask: usize,
/// Status of context
pub status: Status,
pub status_reason: &'static str,
/// Context running or not
pub running: bool,
/// CPU ID, if locked
@@ -197,6 +198,7 @@ impl Context {
sigmask: [0; 2],
umask: 0o022,
status: Status::Blocked,
status_reason: "",
running: false,
cpu_id: None,
ticks: 0,
@@ -304,9 +306,10 @@ impl Context {
}
/// Block the context, and return true if it was runnable before being blocked
pub fn block(&mut self) -> bool {
pub fn block(&mut self, reason: &'static str) -> bool {
if self.status == Status::Runnable {
self.status = Status::Blocked;
self.status_reason = reason;
true
} else {
false
@@ -317,6 +320,7 @@ impl Context {
pub fn unblock(&mut self) -> bool {
if self.status == Status::Blocked {
self.status = Status::Runnable;
self.status_reason = "";
if let Some(cpu_id) = self.cpu_id {
if cpu_id != crate::cpu_id() {

View File

@@ -26,7 +26,7 @@ impl EventQueue {
}
pub fn read(&self, events: &mut [Event]) -> Result<usize> {
self.queue.receive_into(events, true).ok_or(Error::new(EINTR))
self.queue.receive_into(events, true, "EventQueue::read").ok_or(Error::new(EINTR))
}
pub fn write(&self, events: &[Event]) -> Result<usize> {

View File

@@ -39,7 +39,7 @@ use core::{
cmp,
sync::atomic::Ordering
};
use spin::{Once, RwLock, RwLockReadGuard, RwLockWriteGuard};
use spin::{Mutex, Once, RwLock, RwLockReadGuard, RwLockWriteGuard};
// ____ _
// / ___| ___ ___ ___(_) ___ _ __ ___
@@ -225,7 +225,9 @@ pub fn wait(pid: ContextId) -> Result<()> {
}
};
while !tracer.wait() {}
//TODO: proper wait_condition mutex
let m = Mutex::new(());
while !tracer.wait(m.lock(), "ptrace::wait") {}
let contexts = context::contexts();
let context = contexts.get(pid).ok_or(Error::new(ESRCH))?;
@@ -269,7 +271,9 @@ pub fn breakpoint_callback(match_flags: PtraceFlags, event: Option<PtraceEvent>)
)
};
while !tracee.wait() {}
//TODO: proper wait_condition mutex
let m = Mutex::new(());
while !tracee.wait(m.lock(), "ptrace::breakpoint_callback") {}
Some(flags)
}

View File

@@ -69,7 +69,7 @@ impl Scheme for DebugScheme {
};
INPUT.call_once(init_input)
.receive_into(buf, flags & O_NONBLOCK != O_NONBLOCK)
.receive_into(buf, flags & O_NONBLOCK != O_NONBLOCK, "DebugScheme::read")
.ok_or(Error::new(EINTR))
}

View File

@@ -180,31 +180,29 @@ impl PipeRead {
fn read(&self, buf: &mut [u8]) -> Result<usize> {
loop {
{
let mut vec = self.vec.lock();
let mut vec = self.vec.lock();
let mut i = 0;
while i < buf.len() {
if let Some(b) = vec.pop_front() {
buf[i] = b;
i += 1;
} else {
break;
}
let mut i = 0;
while i < buf.len() {
if let Some(b) = vec.pop_front() {
buf[i] = b;
i += 1;
} else {
break;
}
}
if i > 0 {
event::trigger(self.scheme_id, self.write_id, EVENT_WRITE);
if i > 0 {
event::trigger(self.scheme_id, self.write_id, EVENT_WRITE);
return Ok(i);
}
return Ok(i);
}
if Arc::weak_count(&self.vec) == 0 {
return Ok(0);
} else if self.flags.load(Ordering::SeqCst) & O_NONBLOCK == O_NONBLOCK {
return Err(Error::new(EAGAIN));
} else if ! self.condition.wait() {
} else if ! self.condition.wait(vec, "PipeRead::read") {
return Err(Error::new(EINTR));
}
}

36
src/scheme/sys/block.rs Normal file
View File

@@ -0,0 +1,36 @@
use alloc::string::String;
use alloc::vec::Vec;
use core::fmt::Write;
use core::str;
use crate::context;
use crate::syscall;
use crate::syscall::error::Result;
pub fn resource() -> Result<Vec<u8>> {
let mut string = String::new();
{
let mut rows = Vec::new();
{
let contexts = context::contexts();
for (id, context_lock) in contexts.iter() {
let context = context_lock.read();
rows.push((*id, context.name.lock().clone(), context.status_reason));
}
}
for row in rows.iter() {
let id: usize = row.0.into();
let name = str::from_utf8(&row.1).unwrap_or(".");
let _ = writeln!(string, "{}: {}", id, name);
if ! row.2.is_empty() {
let _ = writeln!(string, " {}", row.2);
}
}
}
Ok(string.into_bytes())
}

View File

@@ -10,6 +10,7 @@ use crate::syscall::error::{Error, EBADF, EINVAL, ENOENT, Result};
use crate::syscall::flag::{MODE_DIR, MODE_FILE, SEEK_CUR, SEEK_END, SEEK_SET};
use crate::syscall::scheme::Scheme;
mod block;
mod context;
mod cpu;
mod exe;
@@ -40,6 +41,7 @@ impl SysScheme {
pub fn new() -> SysScheme {
let mut files: BTreeMap<&'static [u8], Box<SysFn>> = BTreeMap::new();
files.insert(b"block", Box::new(block::resource));
files.insert(b"context", Box::new(context::resource));
files.insert(b"cpu", Box::new(cpu::resource));
files.insert(b"exe", Box::new(exe::resource));

View File

@@ -97,7 +97,7 @@ impl UserInner {
self.todo.send(packet);
event::trigger(self.root_id, self.handle_id, EVENT_READ);
Error::demux(self.done.receive(&id))
Error::demux(self.done.receive(&id, "UserInner::call_inner"))
}
pub fn capture(&self, buf: &[u8]) -> Result<usize> {
@@ -202,7 +202,7 @@ impl UserInner {
// If unmounting, do not block so that EOF can be returned immediately
let unmounting = self.unmounting.load(Ordering::SeqCst);
let block = !(nonblock || unmounting);
if let Some(count) = self.todo.receive_into(packet_buf, block) {
if let Some(count) = self.todo.receive_into(packet_buf, block, "UserInner::read") {
if count > 0 {
// If we received requests, return them to the scheme handler
Ok(count * mem::size_of::<Packet>())

View File

@@ -1,6 +1,6 @@
use alloc::sync::Arc;
use alloc::vec::Vec;
use spin::{Mutex, RwLock};
use spin::{Mutex, MutexGuard, RwLock};
use crate::context::{self, Context};
@@ -36,8 +36,8 @@ impl WaitCondition {
len
}
// Wait until notified. Returns false if resumed by a signal or the notify_signal function
pub fn wait(&self) -> bool {
// Wait until notified. Unlocks guard when blocking is ready. Returns false if resumed by a signal or the notify_signal function
pub fn wait<T>(&self, guard: MutexGuard<T>, reason: &'static str) -> bool {
let id;
{
let context_lock = {
@@ -49,10 +49,12 @@ impl WaitCondition {
{
let mut context = context_lock.write();
id = context.id;
context.block();
context.block(reason);
}
self.contexts.lock().push(context_lock);
drop(guard);
}
unsafe { context::switch(); }

View File

@@ -22,13 +22,14 @@ impl<K, V> WaitMap<K, V> where K: Clone + Ord {
self.inner.lock().remove(key)
}
pub fn receive(&self, key: &K) -> V {
pub fn receive(&self, key: &K, reason: &'static str) -> V {
loop {
if let Some(value) = self.receive_nonblock(key) {
let mut inner = self.inner.lock();
if let Some(value) = inner.remove(key) {
return value;
}
//TODO: use false from wait condition to indicate EINTR
let _ = self.condition.wait();
let _ = self.condition.wait(inner, reason);
}
}
@@ -41,12 +42,15 @@ impl<K, V> WaitMap<K, V> where K: Clone + Ord {
}
}
pub fn receive_any(&self) -> (K, V) {
pub fn receive_any(&self, reason: &'static str) -> (K, V) {
loop {
if let Some(entry) = self.receive_any_nonblock() {
return entry;
let mut inner = self.inner.lock();
if let Some(key) = inner.keys().next().cloned() {
if let Some(entry) = inner.remove(&key).map(|value| (key, value)) {
return entry;
}
}
let _ = self.condition.wait();
let _ = self.condition.wait(inner, reason);
}
}

View File

@@ -28,22 +28,23 @@ impl<T> WaitQueue<T> {
self.inner.lock().is_empty()
}
pub fn receive(&self) -> Option<T> {
pub fn receive(&self, reason: &'static str) -> Option<T> {
loop {
if let Some(value) = self.inner.lock().pop_front() {
let mut inner = self.inner.lock();
if let Some(value) = inner.pop_front() {
return Some(value);
}
if ! self.condition.wait() {
if ! self.condition.wait(inner, reason) {
return None;
}
}
}
pub fn receive_into(&self, buf: &mut [T], block: bool) -> Option<usize> {
pub fn receive_into(&self, buf: &mut [T], block: bool, reason: &'static str) -> Option<usize> {
let mut i = 0;
if i < buf.len() && block {
buf[i] = self.receive()?;
buf[i] = self.receive(reason)?;
i += 1;
}

View File

@@ -22,7 +22,7 @@ impl<'a> ::core::fmt::Debug for ByteStr<'a> {
}
}
//TODO: calling format_call with arguments from another process space will not work
pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize) -> String {
match a {
SYS_OPEN => format!(

View File

@@ -66,7 +66,7 @@ pub fn futex(addr: &mut i32, op: usize, val: i32, val2: usize, addr2: *mut i32)
context.wake = Some(end);
}
context.block();
context.block("futex");
}
futexes.push_back((addr as *mut i32 as usize, context_lock));

View File

@@ -355,7 +355,7 @@ pub fn clone(flags: CloneFlags, stack_base: usize) -> Result<ContextId> {
let contexts = context::contexts();
let context_lock = contexts.current().ok_or(Error::new(ESRCH))?;
let mut context = context_lock.write();
context.block();
context.block("vfork");
vfork = true;
} else {
vfork = false;
@@ -1429,7 +1429,7 @@ pub fn sigreturn() -> Result<usize> {
let context_lock = contexts.current().ok_or(Error::new(ESRCH))?;
let mut context = context_lock.write();
context.ksig_restore = true;
context.block();
context.block("sigreturn");
}
let _ = unsafe { context::switch() };
@@ -1538,7 +1538,7 @@ pub fn waitpid(pid: ContextId, status_ptr: usize, flags: WaitFlags) -> Result<Co
Some(Ok(ContextId::from(0)))
}
} else {
let (_wid, (w_pid, status)) = waitpid.receive_any();
let (_wid, (w_pid, status)) = waitpid.receive_any("waitpid any");
grim_reaper(w_pid, status)
}
} else if (pid.into() as isize) < 0 {
@@ -1575,7 +1575,7 @@ pub fn waitpid(pid: ContextId, status_ptr: usize, flags: WaitFlags) -> Result<Co
let (w_pid, status) = waitpid.receive(&WaitpidKey {
pid: None,
pgid: Some(pgid)
});
}, "waitpid pgid");
grim_reaper(w_pid, status)
}
} else {
@@ -1612,7 +1612,7 @@ pub fn waitpid(pid: ContextId, status_ptr: usize, flags: WaitFlags) -> Result<Co
let (w_pid, status) = waitpid.receive(&WaitpidKey {
pid: Some(pid),
pgid: None
});
}, "waitpid pid");
grim_reaper(w_pid, status)
}
};

View File

@@ -29,7 +29,7 @@ pub fn nanosleep(req: &TimeSpec, rem_opt: Option<&mut TimeSpec>) -> Result<usize
let mut context = context_lock.write();
context.wake = Some(end);
context.block();
context.block("nanosleep");
}
unsafe { context::switch(); }