diff --git a/src/arch/aarch64/init/pre_kstart/helpers/vectors.S b/src/arch/aarch64/init/pre_kstart/helpers/vectors.S index dec303b..0406506 100644 --- a/src/arch/aarch64/init/pre_kstart/helpers/vectors.S +++ b/src/arch/aarch64/init/pre_kstart/helpers/vectors.S @@ -18,21 +18,21 @@ __vec_00: .align 7 __vec_01: mov x18, #0xb0b1 - b do_exception_irq + b irq_at_el1 b __vec_01 // FIQ .align 7 __vec_02: mov x18, #0xb0b2 - b do_exception_unhandled + b unhandled_exception b __vec_02 // SError .align 7 __vec_03: mov x18, #0xb0b3 - b do_exception_unhandled + b unhandled_exception b __vec_03 // Synchronous @@ -46,21 +46,21 @@ __vec_04: .align 7 __vec_05: mov x18, #0xb0b5 - b do_exception_irq + b irq_at_el1 b __vec_05 // FIQ .align 7 __vec_06: mov x18, #0xb0b6 - b do_exception_unhandled + b unhandled_exception b __vec_06 // SError .align 7 __vec_07: mov x18, #0xb0b7 - b do_exception_unhandled + b unhandled_exception b __vec_07 // Synchronous @@ -74,49 +74,49 @@ __vec_08: .align 7 __vec_09: mov x18, #0xb0b9 - b do_exception_irq + b irq_at_el0 b __vec_09 // FIQ .align 7 __vec_10: mov x18, #0xb0ba - b do_exception_unhandled + b unhandled_exception b __vec_10 // SError .align 7 __vec_11: mov x18, #0xb0bb - b do_exception_unhandled + b unhandled_exception b __vec_11 // Synchronous .align 7 __vec_12: mov x18, #0xb0bc - b do_exception_unhandled + b unhandled_exception b __vec_12 // IRQ .align 7 __vec_13: mov x18, #0xb0bd - b do_exception_unhandled + b unhandled_exception b __vec_13 // FIQ .align 7 __vec_14: mov x18, #0xb0be - b do_exception_unhandled + b unhandled_exception b __vec_14 // SError .align 7 __vec_15: mov x18, #0xb0bf - b do_exception_unhandled + b unhandled_exception b __vec_15 .align 7 diff --git a/src/arch/aarch64/interrupt/irq.rs b/src/arch/aarch64/interrupt/irq.rs index 1ad13f6..27636e4 100644 --- a/src/arch/aarch64/interrupt/irq.rs +++ b/src/arch/aarch64/interrupt/irq.rs @@ -7,127 +7,26 @@ use crate::device::{gic}; use crate::device::serial::{COM1}; use crate::time; +use crate::{exception_stack}; + //resets to 0 in context::switch() pub static PIT_TICKS: AtomicUsize = ATOMIC_USIZE_INIT; -#[naked] -#[no_mangle] -pub unsafe extern fn do_exception_irq() { - #[inline(never)] - unsafe fn inner() { - irq_demux(); +exception_stack!(irq_at_el0, |stack| { + match gic::irq_ack() { + 30 => irq_handler_gentimer(30), + 33 => irq_handler_com1(33), + _ => panic!("irq_demux: unregistered IRQ"), } +}); - llvm_asm!("str x0, [sp, #-8]! - str x1, [sp, #-8]! - str x2, [sp, #-8]! - str x3, [sp, #-8]! - str x4, [sp, #-8]! - str x5, [sp, #-8]! - str x6, [sp, #-8]! - str x7, [sp, #-8]! - str x8, [sp, #-8]! - str x9, [sp, #-8]! - str x10, [sp, #-8]! - str x11, [sp, #-8]! - str x12, [sp, #-8]! - str x13, [sp, #-8]! - str x14, [sp, #-8]! - str x15, [sp, #-8]! - str x16, [sp, #-8]! - str x17, [sp, #-8]! - str x18, [sp, #-8]! - str x19, [sp, #-8]! - str x20, [sp, #-8]! - str x21, [sp, #-8]! - str x22, [sp, #-8]! - str x23, [sp, #-8]! - str x24, [sp, #-8]! - str x25, [sp, #-8]! - str x26, [sp, #-8]! - str x27, [sp, #-8]! - str x28, [sp, #-8]! - str x29, [sp, #-8]! - str x30, [sp, #-8]! - - mrs x18, sp_el0 - str x18, [sp, #-8]! - - mrs x18, esr_el1 - str x18, [sp, #-8]! - - mrs x18, spsr_el1 - str x18, [sp, #-8]! - - mrs x18, tpidrro_el0 - str x18, [sp, #-8]! - - mrs x18, tpidr_el0 - str x18, [sp, #-8]! - - str x18, [sp, #-8]! - - mrs x18, elr_el1 - str x18, [sp, #-8]!" - : : : : "volatile"); - - inner(); - - llvm_asm!("ldr x18, [sp], #8 - msr elr_el1, x18 - - ldr x18, [sp], #8 - - ldr x18, [sp], #8 - msr tpidr_el0, x18 - - ldr x18, [sp], #8 - msr tpidrro_el0, x18 - - ldr x18, [sp], #8 - msr spsr_el1, x18 - - ldr x18, [sp], #8 - msr esr_el1, x18 - - ldr x18, [sp], #8 - msr sp_el0, x18 - - ldr x30, [sp], #8 - ldr x29, [sp], #8 - ldr x28, [sp], #8 - ldr x27, [sp], #8 - ldr x26, [sp], #8 - ldr x25, [sp], #8 - ldr x24, [sp], #8 - ldr x23, [sp], #8 - ldr x22, [sp], #8 - ldr x21, [sp], #8 - ldr x20, [sp], #8 - ldr x19, [sp], #8 - ldr x18, [sp], #8 - ldr x17, [sp], #8 - ldr x16, [sp], #8 - ldr x15, [sp], #8 - ldr x14, [sp], #8 - ldr x13, [sp], #8 - ldr x12, [sp], #8 - ldr x11, [sp], #8 - ldr x10, [sp], #8 - ldr x9, [sp], #8 - ldr x8, [sp], #8 - ldr x7, [sp], #8 - ldr x6, [sp], #8 - ldr x5, [sp], #8 - ldr x4, [sp], #8 - ldr x3, [sp], #8 - ldr x2, [sp], #8 - ldr x1, [sp], #8 - ldr x0, [sp], #8" - : : : : "volatile"); - - llvm_asm!("eret" :::: "volatile"); -} +exception_stack!(irq_at_el1, |stack| { + match gic::irq_ack() { + 30 => irq_handler_gentimer(30), + 33 => irq_handler_com1(33), + _ => panic!("irq_demux: unregistered IRQ"), + } +}); unsafe fn trigger(irq: u32) { extern { diff --git a/src/arch/aarch64/interrupt/syscall.rs b/src/arch/aarch64/interrupt/syscall.rs index afd1416..948cd5d 100644 --- a/src/arch/aarch64/interrupt/syscall.rs +++ b/src/arch/aarch64/interrupt/syscall.rs @@ -8,232 +8,9 @@ use crate::{ #[no_mangle] pub unsafe extern fn do_exception_unhandled() {} -/* -#[naked] -#[no_mangle] -pub unsafe extern fn do_exception_unhandled() { - #[inline(never)] - unsafe fn inner(stack: &mut InterruptStack) -> usize { - println!("do_exception_unhandled: ELR: 0x{:016x}", stack.elr_el1); - loop {} - } - - llvm_asm!("str x0, [sp, #-8]! - str x1, [sp, #-8]! - str x2, [sp, #-8]! - str x3, [sp, #-8]! - str x4, [sp, #-8]! - str x5, [sp, #-8]! - str x6, [sp, #-8]! - str x7, [sp, #-8]! - str x8, [sp, #-8]! - str x9, [sp, #-8]! - str x10, [sp, #-8]! - str x11, [sp, #-8]! - str x12, [sp, #-8]! - str x13, [sp, #-8]! - str x14, [sp, #-8]! - str x15, [sp, #-8]! - str x16, [sp, #-8]! - str x17, [sp, #-8]! - str x18, [sp, #-8]! - str x19, [sp, #-8]! - str x20, [sp, #-8]! - str x21, [sp, #-8]! - str x22, [sp, #-8]! - str x23, [sp, #-8]! - str x24, [sp, #-8]! - str x25, [sp, #-8]! - str x26, [sp, #-8]! - str x27, [sp, #-8]! - str x28, [sp, #-8]! - str x29, [sp, #-8]! - str x30, [sp, #-8]! - - mrs x18, sp_el0 - str x18, [sp, #-8]! - - mrs x18, esr_el1 - str x18, [sp, #-8]! - - mrs x18, spsr_el1 - str x18, [sp, #-8]! - - mrs x18, tpidrro_el0 - str x18, [sp, #-8]! - - mrs x18, tpidr_el0 - str x18, [sp, #-8]! - - str x18, [sp, #-8]! - - mrs x18, elr_el1 - str x18, [sp, #-8]!" - : : : : "volatile"); - - let sp: usize; - llvm_asm!("" : "={sp}"(sp) : : : "volatile"); - llvm_asm!("mov x29, sp" : : : : "volatile"); - - let a = inner(&mut *(sp as *mut InterruptStack)); -} -*/ - #[no_mangle] pub unsafe extern fn do_exception_synchronous() {} -/* -#[naked] -#[no_mangle] -pub unsafe extern fn do_exception_synchronous() { - #[inline(never)] - unsafe fn inner(stack: &mut InterruptStack) -> usize { - let exception_code = (stack.esr_el1 & (0x3f << 26)) >> 26; - if exception_code != 0b010101 { - println!("do_exception_synchronous: Non-SVC!!!"); - loop {} - } else { - println!("do_exception_synchronous: SVC: x8: 0x{:016x}", stack.scratch.x8); - } - - llvm_asm!("nop": : : : "volatile"); - llvm_asm!("nop": : : : "volatile"); - llvm_asm!("nop": : : : "volatile"); - llvm_asm!("nop": : : : "volatile"); - - let fp; - llvm_asm!("" : "={fp}"(fp) : : : "volatile"); - - syscall::syscall( - stack.scratch.x8, - stack.scratch.x0, - stack.scratch.x1, - stack.scratch.x2, - stack.scratch.x3, - stack.scratch.x4, - fp, - stack - ) - } - - llvm_asm!("str x0, [sp, #-8]! - str x1, [sp, #-8]! - str x2, [sp, #-8]! - str x3, [sp, #-8]! - str x4, [sp, #-8]! - str x5, [sp, #-8]! - str x6, [sp, #-8]! - str x7, [sp, #-8]! - str x8, [sp, #-8]! - str x9, [sp, #-8]! - str x10, [sp, #-8]! - str x11, [sp, #-8]! - str x12, [sp, #-8]! - str x13, [sp, #-8]! - str x14, [sp, #-8]! - str x15, [sp, #-8]! - str x16, [sp, #-8]! - str x17, [sp, #-8]! - str x18, [sp, #-8]! - str x19, [sp, #-8]! - str x20, [sp, #-8]! - str x21, [sp, #-8]! - str x22, [sp, #-8]! - str x23, [sp, #-8]! - str x24, [sp, #-8]! - str x25, [sp, #-8]! - str x26, [sp, #-8]! - str x27, [sp, #-8]! - str x28, [sp, #-8]! - str x29, [sp, #-8]! - str x30, [sp, #-8]! - - mrs x18, sp_el0 - str x18, [sp, #-8]! - - mrs x18, esr_el1 - str x18, [sp, #-8]! - - mrs x18, spsr_el1 - str x18, [sp, #-8]! - - mrs x18, tpidrro_el0 - str x18, [sp, #-8]! - - mrs x18, tpidr_el0 - str x18, [sp, #-8]! - - str x18, [sp, #-8]! - - mrs x18, elr_el1 - str x18, [sp, #-8]!" - : : : : "volatile"); - - let sp: usize; - llvm_asm!("" : "={sp}"(sp) : : : "volatile"); - llvm_asm!("mov x29, sp" : : : : "volatile"); - - let a = inner(&mut *(sp as *mut InterruptStack)); - - llvm_asm!("" : : "{x0}"(a) : : "volatile"); - - llvm_asm!("ldr x18, [sp], #8 - msr elr_el1, x18 - - ldr x18, [sp], #8 - - ldr x18, [sp], #8 - msr tpidr_el0, x18 - - ldr x18, [sp], #8 - msr tpidrro_el0, x18 - - ldr x18, [sp], #8 - msr spsr_el1, x18 - - ldr x18, [sp], #8 - msr esr_el1, x18 - - ldr x18, [sp], #8 - msr sp_el0, x18 - - ldr x30, [sp], #8 - ldr x29, [sp], #8 - ldr x28, [sp], #8 - ldr x27, [sp], #8 - ldr x26, [sp], #8 - ldr x25, [sp], #8 - ldr x24, [sp], #8 - ldr x23, [sp], #8 - ldr x22, [sp], #8 - ldr x21, [sp], #8 - ldr x20, [sp], #8 - ldr x19, [sp], #8 - ldr x18, [sp], #8 - ldr x17, [sp], #8 - ldr x16, [sp], #8 - ldr x15, [sp], #8 - ldr x14, [sp], #8 - ldr x13, [sp], #8 - ldr x12, [sp], #8 - ldr x11, [sp], #8 - ldr x10, [sp], #8 - ldr x9, [sp], #8 - ldr x8, [sp], #8 - ldr x7, [sp], #8 - ldr x6, [sp], #8 - ldr x5, [sp], #8 - ldr x4, [sp], #8 - ldr x3, [sp], #8 - ldr x2, [sp], #8 - ldr x1, [sp], #8 - add sp, sp, #8" /* Skip over x0 - it's got the retval of inner already */ - : : : : "volatile"); - - llvm_asm!("eret" :::: "volatile"); -} -*/ - #[allow(dead_code)] #[repr(packed)] pub struct SyscallStack { @@ -285,45 +62,6 @@ macro_rules! with_exception_stack { }} } -#[no_mangle] -pub unsafe extern "C" fn __inner_syscall_instruction(stack: *mut InterruptStack) { - with_exception_stack!(|stack| { - // Set a restore point for clone - let fp; - asm!("mov {}, fp", out(reg) fp); - - let scratch = &stack.scratch; - syscall::syscall(scratch.x8, scratch.x0, scratch.x1, scratch.x2, scratch.x3, scratch.x4, fp, stack) - }); -} - -function!(syscall_instruction => { - " - nop - ", - - // Push context registers - push_preserved!(), - push_scratch!(), - push_special!(), - - // TODO: Map PTI - - // Call inner function - "mov x0, sp\n", - "bl __inner_syscall_instruction\n", - - // TODO: Unmap PTI - - // Pop context registers - pop_special!(), - pop_scratch!(), - pop_preserved!(), - - // Return - "eret\n", -}); - function!(clone_ret => { "ldp x29, x30, [sp], #16\n", "mov sp, x29\n", diff --git a/src/context/arch/aarch64.rs b/src/context/arch/aarch64.rs index 3af07fd..a8af274 100644 --- a/src/context/arch/aarch64.rs +++ b/src/context/arch/aarch64.rs @@ -1,13 +1,14 @@ use core::mem; -use core::sync::atomic::{AtomicBool, AtomicUsize, ATOMIC_BOOL_INIT, ATOMIC_USIZE_INIT, Ordering}; +use core::sync::atomic::{AtomicBool, Ordering}; use crate::device::cpu::registers::{control_regs, tlb}; +use crate::syscall::FloatRegisters; /// This must be used by the kernel to ensure that context switches are done atomically /// Compare and exchange this to true when beginning a context switch on any CPU /// The `Context::switch_to` function will set it back to false, allowing other CPU's to switch /// This must be done, as no locks can be held on the stack during switch -pub static CONTEXT_SWITCH_LOCK: AtomicBool = ATOMIC_BOOL_INIT; +pub static CONTEXT_SWITCH_LOCK: AtomicBool = AtomicBool::new(false); #[derive(Clone, Debug)] pub struct Context { @@ -17,9 +18,10 @@ pub struct Context { ttbr1_el1: usize, /* Pointer to P4 translation table for this Context */ tpidr_el0: usize, /* Pointer to TLS region for this Context */ tpidrro_el0: usize, /* Pointer to TLS (read-only) region for this Context */ - rflags: usize, + spsr_el1: usize, esr_el1: usize, - padding: usize, + fx_loadable: bool, + fx_address: usize, sp: usize, /* Stack Pointer (x31) */ lr: usize, /* Link Register (x30) */ fp: usize, /* Frame pointer Register (x29) */ @@ -46,8 +48,6 @@ pub struct Context { x8: usize, /* Indirect location Register */ } -static CONTEXT_COUNT: AtomicUsize = ATOMIC_USIZE_INIT; - impl Context { pub fn new() -> Context { Context { @@ -57,9 +57,10 @@ impl Context { ttbr1_el1: 0, tpidr_el0: 0, tpidrro_el0: 0, - rflags: 0, /* spsr_el1 */ + spsr_el1: 0, esr_el1: 0, - padding: 0xbeef0000 | CONTEXT_COUNT.fetch_add(1, Ordering::SeqCst), + fx_loadable: false, + fx_address: 0, sp: 0, lr: 0, fp: 0, @@ -95,9 +96,6 @@ impl Context { self.ttbr1_el1 } - pub fn set_fx(&mut self, _address: usize) { - } - pub fn set_page_utable(&mut self, address: usize) { self.ttbr0_el1 = address; } @@ -150,6 +148,44 @@ impl Context { value } + pub fn get_fx_regs(&self) -> Option { + if !self.fx_loadable { + return None; + } + let mut regs = unsafe { *(self.fx_address as *const FloatRegisters) }; + let mut new_st = regs.fp_simd_regs; + regs.fp_simd_regs = new_st; + Some(regs) + } + + pub fn set_fx_regs(&mut self, mut new: FloatRegisters) -> bool { + if !self.fx_loadable { + return false; + } + + { + let old = unsafe { &*(self.fx_address as *const FloatRegisters) }; + let old_st = new.fp_simd_regs; + let mut new_st = new.fp_simd_regs; + for (new_st, old_st) in new_st.iter_mut().zip(&old_st) { + *new_st = *old_st; + } + new.fp_simd_regs = new_st; + + // Make sure we don't use `old` from now on + } + + unsafe { + *(self.fx_address as *mut FloatRegisters) = new; + } + true + } + + pub fn set_fx(&mut self, address: usize) { + self.fx_address = address; + } + + pub fn dump(&self) { println!("elr_el1: 0x{:016x}", self.elr_el1); println!("sp_el0: 0x{:016x}", self.sp_el0); @@ -157,9 +193,8 @@ impl Context { println!("ttbr1_el1: 0x{:016x}", self.ttbr1_el1); println!("tpidr_el0: 0x{:016x}", self.tpidr_el0); println!("tpidrro_el0: 0x{:016x}", self.tpidrro_el0); - println!("rflags: 0x{:016x}", self.rflags); + println!("spsr_el1: 0x{:016x}", self.spsr_el1); println!("esr_el1: 0x{:016x}", self.esr_el1); - println!("padding: 0x{:016x}", self.padding); println!("sp: 0x{:016x}", self.sp); println!("lr: 0x{:016x}", self.lr); println!("fp: 0x{:016x}", self.fp); @@ -190,6 +225,59 @@ impl Context { #[inline(never)] #[naked] pub unsafe fn switch_to(&mut self, next: &mut Context) { + let mut float_regs = self.fx_address as *mut FloatRegisters; + asm!( + "stp q0, q1, [{0}, #16 * 0]", + "stp q2, q3, [{0}, #16 * 2]", + "stp q4, q5, [{0}, #16 * 4]", + "stp q6, q7, [{0}, #16 * 6]", + "stp q8, q9, [{0}, #16 * 8]", + "stp q10, q11, [{0}, #16 * 10]", + "stp q12, q13, [{0}, #16 * 12]", + "stp q14, q15, [{0}, #16 * 14]", + "stp q16, q17, [{0}, #16 * 16]", + "stp q18, q19, [{0}, #16 * 18]", + "stp q20, q21, [{0}, #16 * 20]", + "stp q22, q23, [{0}, #16 * 22]", + "stp q24, q25, [{0}, #16 * 24]", + "stp q26, q27, [{0}, #16 * 26]", + "stp q28, q29, [{0}, #16 * 28]", + "stp q30, q31, [{0}, #16 * 30]", + "mrs {1}, fpcr", + "mrs {2}, fpsr", + in(reg) (&(*(float_regs)).fp_simd_regs), + out(reg) ((*(float_regs)).fpcr), + out(reg) ((*(float_regs)).fpsr) + ); + + self.fx_loadable = true; + + if next.fx_loadable { + asm!( + "ldp q0, q1, [{0}, #16 * 0]", + "ldp q2, q3, [{0}, #16 * 2]", + "ldp q4, q5, [{0}, #16 * 4]", + "ldp q6, q7, [{0}, #16 * 6]", + "ldp q8, q9, [{0}, #16 * 8]", + "ldp q10, q11, [{0}, #16 * 10]", + "ldp q12, q13, [{0}, #16 * 12]", + "ldp q14, q15, [{0}, #16 * 14]", + "ldp q16, q17, [{0}, #16 * 16]", + "ldp q18, q19, [{0}, #16 * 18]", + "ldp q20, q21, [{0}, #16 * 20]", + "ldp q22, q23, [{0}, #16 * 22]", + "ldp q24, q25, [{0}, #16 * 24]", + "ldp q26, q27, [{0}, #16 * 26]", + "ldp q28, q29, [{0}, #16 * 28]", + "ldp q30, q31, [{0}, #16 * 30]", + "msr fpcr, {1}", + "msr fpsr, {2}", + in(reg) (&(*(float_regs)).fp_simd_regs), + in(reg) ((*(float_regs)).fpcr), + in(reg) ((*(float_regs)).fpsr) + ); + } + self.ttbr0_el1 = control_regs::ttbr0_el1() as usize; if next.ttbr0_el1 != self.ttbr0_el1 { control_regs::ttbr0_el1_write(next.ttbr0_el1 as u64); @@ -277,8 +365,8 @@ impl Context { llvm_asm!("mrs $0, tpidrro_el0" : "=r"(self.tpidrro_el0) : : "memory" : "volatile"); llvm_asm!("msr tpidrro_el0, $0" : : "r"(next.tpidrro_el0) : "memory" : "volatile"); - llvm_asm!("mrs $0, spsr_el1" : "=r"(self.rflags) : : "memory" : "volatile"); - llvm_asm!("msr spsr_el1, $0" : : "r"(next.rflags) : "memory" : "volatile"); + llvm_asm!("mrs $0, spsr_el1" : "=r"(self.spsr_el1) : : "memory" : "volatile"); + llvm_asm!("msr spsr_el1, $0" : : "r"(next.spsr_el1) : "memory" : "volatile"); llvm_asm!("mrs $0, esr_el1" : "=r"(self.esr_el1) : : "memory" : "volatile"); llvm_asm!("msr esr_el1, $0" : : "r"(next.esr_el1) : "memory" : "volatile"); diff --git a/src/syscall/process.rs b/src/syscall/process.rs index 5e7dd27..df1b24d 100644 --- a/src/syscall/process.rs +++ b/src/syscall/process.rs @@ -519,7 +519,7 @@ pub fn clone(flags: CloneFlags, stack_base: usize) -> Result { // Update the pointer to the InterruptStack to reflect the new process' // stack. (Without this the pointer would be InterruptStack on the parent - // process' stack. + // process' stack). *(new_sp as *mut u64) = new_sp as u64 + istack_offset; // Update tpidr_el0 in the new process' InterruptStack