From fbdfcd2ba012b644500ea4d2170542d2e9160ac8 Mon Sep 17 00:00:00 2001 From: jD91mZM2 Date: Sat, 11 Jul 2020 16:03:57 +0200 Subject: [PATCH] WIP: Rewrite interrupts as global assembly Because the way we were using inline assembly was technically incorrect and breaking the laws of rust This *finally* compiles. That doesn't mean it works! --- Cargo.lock | 26 ++ Cargo.toml | 1 + src/arch/x86_64/idt.rs | 4 +- src/arch/x86_64/interrupt/exception.rs | 45 +-- src/arch/x86_64/interrupt/handler.rs | 421 +++++++++++++++++++++++++ src/arch/x86_64/interrupt/ipi.rs | 8 +- src/arch/x86_64/interrupt/irq.rs | 49 +-- src/arch/x86_64/interrupt/mod.rs | 4 + src/arch/x86_64/interrupt/syscall.rs | 104 +++--- src/arch/x86_64/macros.rs | 397 ----------------------- src/context/context.rs | 2 +- src/lib.rs | 3 +- src/ptrace.rs | 2 +- src/syscall/driver.rs | 2 +- src/syscall/mod.rs | 2 +- 15 files changed, 558 insertions(+), 512 deletions(-) create mode 100644 src/arch/x86_64/interrupt/handler.rs diff --git a/Cargo.lock b/Cargo.lock index b34799f..b9f2271 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -31,6 +31,7 @@ dependencies = [ "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "goblin 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "linked_list_allocator 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)", + "paste 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", "raw-cpuid 8.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.56", "rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", @@ -63,11 +64,33 @@ dependencies = [ "scopeguard 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "paste" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "paste-impl 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-hack 0.5.16 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "paste-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro-hack 0.5.16 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "plain" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "proc-macro-hack" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "raw-cpuid" version = "7.0.3" @@ -175,7 +198,10 @@ dependencies = [ "checksum linked_list_allocator 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "47de1a43fad0250ee197e9e124e5b5deab3d7b39d4428ae8a6d741ceb340c362" "checksum linked_list_allocator 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)" = "e70e46c13c0e8374c26cec5752e3347ca1087d9711de8f45aa513a7700efd73d" "checksum lock_api 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" +"checksum paste 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)" = "45ca20c77d80be666aef2b45486da86238fabe33e38306bd3118fe4af33fa880" +"checksum paste-impl 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)" = "d95a7db200b97ef370c8e6de0088252f7e0dfff7d047a28528e47456c0fc98b6" "checksum plain 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" +"checksum proc-macro-hack 0.5.16 (registry+https://github.com/rust-lang/crates.io-index)" = "7e0456befd48169b9f13ef0f0ad46d492cf9d2dbb918bcf38e01eed4ce3ec5e4" "checksum raw-cpuid 7.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b4a349ca83373cfa5d6dbb66fd76e58b2cca08da71a5f6400de0a0a6a9bceeaf" "checksum raw-cpuid 8.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9e9c0f2091b865a94bc3c9d34896cc4bbda04453453c391f7eb224491be9ae1d" "checksum rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" diff --git a/Cargo.toml b/Cargo.toml index 09bc6e3..d38edb7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ raw-cpuid = "8.0.0" redox_syscall = { path = "syscall" } slab_allocator = { path = "slab_allocator", optional = true } spin = "0.5.2" +paste = "0.1.18" [dependencies.goblin] version = "0.2.1" diff --git a/src/arch/x86_64/idt.rs b/src/arch/x86_64/idt.rs index e1c0095..bb0eb29 100644 --- a/src/arch/x86_64/idt.rs +++ b/src/arch/x86_64/idt.rs @@ -172,7 +172,7 @@ pub unsafe fn init_generic(is_bsp: bool, idt: &mut Idt) { current_idt[13].set_func(exception::protection); current_idt[14].set_func(exception::page); // 15 reserved - current_idt[16].set_func(exception::fpu); + current_idt[16].set_func(exception::fpu_fault); current_idt[17].set_func(exception::alignment_check); current_idt[18].set_func(exception::machine_check); current_idt[19].set_func(exception::simd); @@ -186,7 +186,7 @@ pub unsafe fn init_generic(is_bsp: bool, idt: &mut Idt) { if is_bsp { // Set up IRQs - current_idt[32].set_func(irq::pit); + current_idt[32].set_func(irq::pit_stack); current_idt[33].set_func(irq::keyboard); current_idt[34].set_func(irq::cascade); current_idt[35].set_func(irq::com2); diff --git a/src/arch/x86_64/interrupt/exception.rs b/src/arch/x86_64/interrupt/exception.rs index b55a13e..09ba6d2 100644 --- a/src/arch/x86_64/interrupt/exception.rs +++ b/src/arch/x86_64/interrupt/exception.rs @@ -1,21 +1,24 @@ use crate::{ interrupt::stack_trace, ptrace, - syscall::flag::* + syscall::flag::*, + + interrupt_stack, + interrupt_error, }; extern { fn ksignal(signal: usize); } -interrupt_stack!(divide_by_zero, stack, { +interrupt_stack!(divide_by_zero, |stack| { println!("Divide by zero"); stack.dump(); stack_trace(); ksignal(SIGFPE); }); -interrupt_stack!(debug, stack, { +interrupt_stack!(debug, |stack| { let mut handled = false; let guard = ptrace::set_process_regs(stack); @@ -42,12 +45,12 @@ interrupt_stack!(debug, stack, { } }); -interrupt_stack!(non_maskable, stack, { +interrupt_stack!(non_maskable, |stack| { println!("Non-maskable interrupt"); stack.dump(); }); -interrupt_stack!(breakpoint, stack, { +interrupt_stack!(breakpoint, |stack| { // The processor lets RIP point to the instruction *after* int3, so // unhandled breakpoint interrupt don't go in an infinite loop. But we // throw SIGTRAP anyway, so that's not a problem. @@ -72,70 +75,70 @@ interrupt_stack!(breakpoint, stack, { } }); -interrupt_stack!(overflow, stack, { +interrupt_stack!(overflow, |stack| { println!("Overflow trap"); stack.dump(); stack_trace(); ksignal(SIGFPE); }); -interrupt_stack!(bound_range, stack, { +interrupt_stack!(bound_range, |stack| { println!("Bound range exceeded fault"); stack.dump(); stack_trace(); ksignal(SIGSEGV); }); -interrupt_stack!(invalid_opcode, stack, { +interrupt_stack!(invalid_opcode, |stack| { println!("Invalid opcode fault"); stack.dump(); stack_trace(); ksignal(SIGILL); }); -interrupt_stack!(device_not_available, stack, { +interrupt_stack!(device_not_available, |stack| { println!("Device not available fault"); stack.dump(); stack_trace(); ksignal(SIGILL); }); -interrupt_error!(double_fault, stack, { +interrupt_error!(double_fault, |stack| { println!("Double fault"); stack.dump(); stack_trace(); ksignal(SIGSEGV); }); -interrupt_error!(invalid_tss, stack, { +interrupt_error!(invalid_tss, |stack| { println!("Invalid TSS fault"); stack.dump(); stack_trace(); ksignal(SIGSEGV); }); -interrupt_error!(segment_not_present, stack, { +interrupt_error!(segment_not_present, |stack| { println!("Segment not present fault"); stack.dump(); stack_trace(); ksignal(SIGSEGV); }); -interrupt_error!(stack_segment, stack, { +interrupt_error!(stack_segment, |stack| { println!("Stack segment fault"); stack.dump(); stack_trace(); ksignal(SIGSEGV); }); -interrupt_error!(protection, stack, { +interrupt_error!(protection, |stack| { println!("Protection fault"); stack.dump(); stack_trace(); ksignal(SIGSEGV); }); -interrupt_error!(page, stack, { +interrupt_error!(page, |stack| { let cr2: usize; asm!("mov rax, cr2" : "={rax}"(cr2) : : : "intel", "volatile"); println!("Page fault: {:>016X}", cr2); @@ -144,42 +147,42 @@ interrupt_error!(page, stack, { ksignal(SIGSEGV); }); -interrupt_stack!(fpu, stack, { +interrupt_stack!(fpu_fault, |stack| { println!("FPU floating point fault"); stack.dump(); stack_trace(); ksignal(SIGFPE); }); -interrupt_error!(alignment_check, stack, { +interrupt_error!(alignment_check, |stack| { println!("Alignment check fault"); stack.dump(); stack_trace(); ksignal(SIGBUS); }); -interrupt_stack!(machine_check, stack, { +interrupt_stack!(machine_check, |stack| { println!("Machine check fault"); stack.dump(); stack_trace(); ksignal(SIGBUS); }); -interrupt_stack!(simd, stack, { +interrupt_stack!(simd, |stack| { println!("SIMD floating point fault"); stack.dump(); stack_trace(); ksignal(SIGFPE); }); -interrupt_stack!(virtualization, stack, { +interrupt_stack!(virtualization, |stack| { println!("Virtualization fault"); stack.dump(); stack_trace(); ksignal(SIGBUS); }); -interrupt_error!(security, stack, { +interrupt_error!(security, |stack| { println!("Security exception"); stack.dump(); stack_trace(); diff --git a/src/arch/x86_64/interrupt/handler.rs b/src/arch/x86_64/interrupt/handler.rs new file mode 100644 index 0000000..b96d666 --- /dev/null +++ b/src/arch/x86_64/interrupt/handler.rs @@ -0,0 +1,421 @@ +use core::mem; +use syscall::IntRegisters; + +#[allow(dead_code)] +#[derive(Default)] +#[repr(packed)] +pub struct ScratchRegisters { + pub r11: usize, + pub r10: usize, + pub r9: usize, + pub r8: usize, + pub rsi: usize, + pub rdi: usize, + pub rdx: usize, + pub rcx: usize, + pub rax: usize, +} + +impl ScratchRegisters { + pub fn dump(&self) { + println!("RAX: {:>016X}", { self.rax }); + println!("RCX: {:>016X}", { self.rcx }); + println!("RDX: {:>016X}", { self.rdx }); + println!("RDI: {:>016X}", { self.rdi }); + println!("RSI: {:>016X}", { self.rsi }); + println!("R8: {:>016X}", { self.r8 }); + println!("R9: {:>016X}", { self.r9 }); + println!("R10: {:>016X}", { self.r10 }); + println!("R11: {:>016X}", { self.r11 }); + } +} + +#[allow(dead_code)] +#[derive(Default)] +#[repr(packed)] +pub struct PreservedRegisters { + pub r15: usize, + pub r14: usize, + pub r13: usize, + pub r12: usize, + pub rbp: usize, + pub rbx: usize, +} + +impl PreservedRegisters { + pub fn dump(&self) { + println!("RBX: {:>016X}", { self.rbx }); + println!("RBP: {:>016X}", { self.rbp }); + println!("R12: {:>016X}", { self.r12 }); + println!("R13: {:>016X}", { self.r13 }); + println!("R14: {:>016X}", { self.r14 }); + println!("R15: {:>016X}", { self.r15 }); + } +} + +#[allow(dead_code)] +#[derive(Default)] +#[repr(packed)] +pub struct IretRegisters { + pub rip: usize, + pub cs: usize, + pub rflags: usize, + // Will only be present if interrupt is raised from another + // privilege ring + pub rsp: usize, + pub ss: usize +} + +impl IretRegisters { + pub fn dump(&self) { + println!("RFLAG: {:>016X}", { self.rflags }); + println!("CS: {:>016X}", { self.cs }); + println!("RIP: {:>016X}", { self.rip }); + } +} + +#[allow(dead_code)] +#[derive(Default)] +#[repr(packed)] +pub struct InterruptStack { + pub fs: usize, + pub preserved: PreservedRegisters, + pub scratch: ScratchRegisters, + pub iret: IretRegisters, +} + +impl InterruptStack { + pub fn dump(&self) { + self.iret.dump(); + self.scratch.dump(); + self.preserved.dump(); + println!("FS: {:>016X}", { self.fs }); + } + /// Saves all registers to a struct used by the proc: + /// scheme to read/write registers. + pub fn save(&self, all: &mut IntRegisters) { + all.fs = self.fs; + + all.r15 = self.preserved.r15; + all.r14 = self.preserved.r14; + all.r13 = self.preserved.r13; + all.r12 = self.preserved.r12; + all.rbp = self.preserved.rbp; + all.rbx = self.preserved.rbx; + all.r11 = self.scratch.r11; + all.r10 = self.scratch.r10; + all.r9 = self.scratch.r9; + all.r8 = self.scratch.r8; + all.rsi = self.scratch.rsi; + all.rdi = self.scratch.rdi; + all.rdx = self.scratch.rdx; + all.rcx = self.scratch.rcx; + all.rax = self.scratch.rax; + all.rip = self.iret.rip; + all.cs = self.iret.cs; + all.rflags = self.iret.rflags; + + // Set rsp and ss: + + const CPL_MASK: usize = 0b11; + + let cs: usize; + unsafe { + asm!("mov $0, cs" : "=r"(cs) ::: "intel"); + } + + if self.iret.cs & CPL_MASK == cs & CPL_MASK { + // Privilege ring didn't change, so neither did the stack + all.rsp = self as *const Self as usize // rsp after Self was pushed to the stack + + mem::size_of::() // disregard Self + - mem::size_of::() * 2; // well, almost: rsp and ss need to be excluded as they aren't present + unsafe { + asm!("mov $0, ss" : "=r"(all.ss) ::: "intel"); + } + } else { + all.rsp = self.iret.rsp; + all.ss = self.iret.ss; + } + } + /// Loads all registers from a struct used by the proc: + /// scheme to read/write registers. + pub fn load(&mut self, all: &IntRegisters) { + // TODO: Which of these should be allowed to change? + + // self.fs = all.fs; + self.preserved.r15 = all.r15; + self.preserved.r14 = all.r14; + self.preserved.r13 = all.r13; + self.preserved.r12 = all.r12; + self.preserved.rbp = all.rbp; + self.preserved.rbx = all.rbx; + self.scratch.r11 = all.r11; + self.scratch.r10 = all.r10; + self.scratch.r9 = all.r9; + self.scratch.r8 = all.r8; + self.scratch.rsi = all.rsi; + self.scratch.rdi = all.rdi; + self.scratch.rdx = all.rdx; + self.scratch.rcx = all.rcx; + self.scratch.rax = all.rax; + self.iret.rip = all.rip; + + // These should probably be restricted + // self.iret.cs = all.cs; + // self.iret.rflags = all.eflags; + } + /// Enables the "Trap Flag" in the FLAGS register, causing the CPU + /// to send a Debug exception after the next instruction. This is + /// used for singlestep in the proc: scheme. + pub fn set_singlestep(&mut self, enabled: bool) { + if enabled { + self.iret.rflags |= 1 << 8; + } else { + self.iret.rflags &= !(1 << 8); + } + } + /// Checks if the trap flag is enabled, see `set_singlestep` + pub fn is_singlestep(&self) -> bool { + self.iret.rflags & 1 << 8 == 1 << 8 + } +} + +#[allow(dead_code)] +#[derive(Default)] +#[repr(packed)] +pub struct InterruptErrorStack { + pub code: usize, + pub inner: InterruptStack, +} + +impl InterruptErrorStack { + pub fn dump(&self) { + println!("CODE: {:>016X}", { self.code }); + self.inner.dump(); + } +} + +#[macro_export] +macro_rules! intel_asm { + ($($strings:expr,)+) => { + global_asm!(concat!( + ".intel_syntax noprefix\n", + $($strings),+, + ".att_syntax prefix\n", + )); + }; +} +#[macro_export] +macro_rules! function { + ($name:expr => { $($body:expr,)+ }) => { + intel_asm!( + ".global ", $name, "\n", + $name, ":\n", + $($body,)+ + ); + } +} + +#[macro_export] +macro_rules! push_scratch { + () => { " + // Push scratch registers + push rcx + push rdx + push rdi + push rsi + push r8 + push r9 + push r10 + push r11 + " }; +} +#[macro_export] +macro_rules! push_preserved { + () => { " + // Push preserved registers + push rbx + push rbp + push r12 + push r13 + push r14 + push r15 + " }; +} +#[macro_export] +macro_rules! push_fs { + () => { " + // Push fs + push fs + + // Load kernel tls + mov rax, 0x18 + mov fs, ax // can't load value directly into `fs` + " }; +} + +#[macro_export] +macro_rules! pop_scratch { + () => { " + // Pop scratch registers + pop r11 + pop r10 + pop r9 + pop r8 + pop rsi + pop rdi + pop rdx + pop rcx + " }; +} +#[macro_export] +macro_rules! pop_preserved { + () => { " + // Pop preserved registers + pop r15 + pop r14 + pop r13 + pop r12 + pop rbp + pop rbx + " }; +} +#[macro_export] +macro_rules! pop_fs { + () => { " + // Pop fs + pop fs + " }; +} + +#[macro_export] +macro_rules! interrupt_stack { + ($name:ident, |$stack:ident| $code:block) => { + paste::item! { + #[no_mangle] + unsafe extern "C" fn [<__interrupt_ $name>]($stack: *mut $crate::arch::x86_64::interrupt::InterruptStack) { + #[no_mangle] + unsafe extern "C" fn [<__interrupt_inner_ $name>]($stack: &mut $crate::arch::x86_64::interrupt::InterruptStack) { + $code + } + + $crate::arch::x86_64::pti::map(); + [<__interrupt_inner_ $name>](&mut *$stack); + $crate::arch::x86_64::pti::unmap(); + } + + function!(stringify!($name) => { + // Backup all userspace registers to stack + "push rax\n", + push_scratch!(), + push_preserved!(), + push_fs!(), + + // Call inner function with pointer to stack + "mov rdi, rsp\n", + "call __interrupt_", stringify!($name), "\n", + + // Restore all userspace registers + pop_fs!(), + pop_preserved!(), + pop_scratch!(), + + "iretq\n", + }); + + extern "C" { + pub fn $name(); + } + } + }; +} + +#[macro_export] +macro_rules! interrupt { + ($name:ident, || $code:block) => { + paste::item! { + #[no_mangle] + unsafe extern "C" fn [<__interrupt_ $name>]() { + #[no_mangle] + unsafe extern "C" fn [<__interrupt_inner_ $name>]() { + $code + } + + $crate::arch::x86_64::pti::map(); + [<__interrupt_inner_ $name>](); + $crate::arch::x86_64::pti::unmap(); + } + + function!(stringify!($name) => { + // Backup all userspace registers to stack + "push rax\n", + push_scratch!(), + push_fs!(), + + // Call inner function with pointer to stack + "call __interrupt_", stringify!($name), "\n", + + // Restore all userspace registers + pop_fs!(), + pop_scratch!(), + + "iretq\n", + }); + + extern "C" { + pub fn $name(); + } + } + }; +} + +#[macro_export] +macro_rules! interrupt_error { + ($name:ident, |$stack:ident| $code:block) => { + paste::item! { + #[no_mangle] + unsafe extern "C" fn [<__interrupt_ $name>]($stack: *mut $crate::arch::x86_64::interrupt::handler::InterruptErrorStack) { + #[no_mangle] + unsafe extern "C" fn [<__interrupt_inner_ $name>]($stack: &mut $crate::arch::x86_64::interrupt::handler::InterruptErrorStack) { + $code + } + + $crate::arch::x86_64::pti::map(); + [<__interrupt_inner_ $name>](&mut *$stack); + $crate::arch::x86_64::pti::unmap(); + } + + function!(stringify!($name) => { + // Move rax into code's place, put code in last instead (to be + // compatible with InterruptStack) + "xchg [rsp], rax\n", + + // Push all userspace registers + push_scratch!(), + push_preserved!(), + push_fs!(), + + // Put code in, it's now in rax + "push rax\n", + + // Call inner function with pointer to stack + "mov rdi, rsp\n", + "call __interrupt_", stringify!($name), "\n", + + // Pop code + "add rsp, 8\n", + + // Restore all userspace registers + pop_fs!(), + pop_preserved!(), + pop_scratch!(), + + "iretq\n", + }); + + extern "C" { + pub fn $name(); + } + } + }; +} diff --git a/src/arch/x86_64/interrupt/ipi.rs b/src/arch/x86_64/interrupt/ipi.rs index eb747e6..52aad2a 100644 --- a/src/arch/x86_64/interrupt/ipi.rs +++ b/src/arch/x86_64/interrupt/ipi.rs @@ -5,23 +5,23 @@ use crate::context; use crate::device::local_apic::LOCAL_APIC; use super::irq::PIT_TICKS; -interrupt!(wakeup, { +interrupt!(wakeup, || { LOCAL_APIC.eoi(); }); -interrupt!(tlb, { +interrupt!(tlb, || { LOCAL_APIC.eoi(); tlb::flush_all(); }); -interrupt!(switch, { +interrupt!(switch, || { LOCAL_APIC.eoi(); let _ = context::switch(); }); -interrupt!(pit, { +interrupt!(pit, || { LOCAL_APIC.eoi(); if PIT_TICKS.fetch_add(1, Ordering::SeqCst) >= 10 { diff --git a/src/arch/x86_64/interrupt/irq.rs b/src/arch/x86_64/interrupt/irq.rs index a01196f..b0856e8 100644 --- a/src/arch/x86_64/interrupt/irq.rs +++ b/src/arch/x86_64/interrupt/irq.rs @@ -2,6 +2,7 @@ use core::sync::atomic::{AtomicUsize, Ordering}; use alloc::vec::Vec; +use crate::{interrupt, interrupt_stack}; use crate::context::timeout; use crate::device::{local_apic, ioapic, pic}; use crate::device::serial::{COM1, COM2}; @@ -31,10 +32,10 @@ unsafe fn ps2_interrupt(_index: usize) { mov ah, al in al, 0x60 " - : "={al}"(data), "={ah}"(status) - : - : "memory" - : "intel", "volatile" + : "={al}"(data), "={ah}"(status) + : + : "memory" + : "intel", "volatile" ); if status & 1 != 0 { @@ -168,7 +169,7 @@ unsafe fn ioapic_unmask(irq: usize) { ioapic::unmask(irq as u8); } -interrupt_stack!(pit, stack, { +interrupt_stack!(pit_stack, |stack| { // Saves CPU time by not sending IRQ event irq_trigger(0); const PIT_RATE: u64 = 2_250_286; @@ -194,41 +195,41 @@ interrupt_stack!(pit, stack, { } }); -interrupt!(keyboard, { +interrupt!(keyboard, || { ps2_interrupt(0); eoi(1); }); -interrupt!(cascade, { +interrupt!(cascade, || { // No need to do any operations on cascade eoi(2); }); -interrupt!(com2, { +interrupt!(com2, || { while let Some(c) = COM2.lock().receive() { debug_input(c); } eoi(3); }); -interrupt!(com1, { +interrupt!(com1, || { while let Some(c) = COM1.lock().receive() { debug_input(c); } eoi(4); }); -interrupt!(lpt2, { +interrupt!(lpt2, || { trigger(5); eoi(5); }); -interrupt!(floppy, { +interrupt!(floppy, || { trigger(6); eoi(6); }); -interrupt!(lpt1, { +interrupt!(lpt1, || { if irq_method() == IrqMethod::Pic && pic::MASTER.isr() & (1 << 7) == 0 { // the IRQ was spurious, ignore it but increment a counter. SPURIOUS_COUNT_IRQ7.fetch_add(1, Ordering::Relaxed); @@ -238,42 +239,42 @@ interrupt!(lpt1, { eoi(7); }); -interrupt!(rtc, { +interrupt!(rtc, || { trigger(8); eoi(8); }); -interrupt!(pci1, { +interrupt!(pci1, || { trigger(9); eoi(9); }); -interrupt!(pci2, { +interrupt!(pci2, || { trigger(10); eoi(10); }); -interrupt!(pci3, { +interrupt!(pci3, || { trigger(11); eoi(11); }); -interrupt!(mouse, { +interrupt!(mouse, || { ps2_interrupt(1); eoi(12); }); -interrupt!(fpu, { +interrupt!(fpu, || { trigger(13); eoi(13); }); -interrupt!(ata1, { +interrupt!(ata1, || { trigger(14); eoi(14); }); -interrupt!(ata2, { +interrupt!(ata2, || { if irq_method() == IrqMethod::Pic && pic::SLAVE.isr() & (1 << 7) == 0 { SPURIOUS_COUNT_IRQ15.fetch_add(1, Ordering::Relaxed); pic::MASTER.ack(); @@ -283,17 +284,17 @@ interrupt!(ata2, { eoi(15); }); -interrupt!(lapic_timer, { +interrupt!(lapic_timer, || { println!("Local apic timer interrupt"); lapic_eoi(); }); -interrupt!(lapic_error, { +interrupt!(lapic_error, || { println!("Local apic internal error: ESR={:#0x}", local_apic::LOCAL_APIC.esr()); lapic_eoi(); }); -interrupt!(calib_pit, { +interrupt!(calib_pit, || { const PIT_RATE: u64 = 2_250_286; { @@ -309,7 +310,7 @@ interrupt!(calib_pit, { macro_rules! allocatable_irq( ( $idt:expr, $number:literal, $name:ident ) => { - interrupt!($name, { + interrupt!($name, || { allocatable_irq_generic($number); }); } diff --git a/src/arch/x86_64/interrupt/mod.rs b/src/arch/x86_64/interrupt/mod.rs index fcf75d1..ef2588b 100644 --- a/src/arch/x86_64/interrupt/mod.rs +++ b/src/arch/x86_64/interrupt/mod.rs @@ -1,11 +1,15 @@ //! Interrupt instructions +#[macro_use] +pub mod handler; + pub mod exception; pub mod ipi; pub mod irq; pub mod syscall; pub mod trace; +pub use self::handler::InterruptStack; pub use self::trace::stack_trace; pub use super::idt::{available_irqs_iter, is_reserved, set_reserved}; diff --git a/src/arch/x86_64/interrupt/syscall.rs b/src/arch/x86_64/interrupt/syscall.rs index 9f5ed74..56d3ac6 100644 --- a/src/arch/x86_64/interrupt/syscall.rs +++ b/src/arch/x86_64/interrupt/syscall.rs @@ -1,4 +1,4 @@ -use crate::arch::macros::InterruptStack; +use crate::arch::interrupt::InterruptStack; use crate::arch::{gdt, pti}; use crate::syscall::flag::{PTRACE_FLAG_IGNORE, PTRACE_STOP_PRE_SYSCALL, PTRACE_STOP_POST_SYSCALL}; use crate::{ptrace, syscall}; @@ -36,8 +36,8 @@ macro_rules! with_interrupt_stack { } } -#[naked] -pub unsafe extern fn syscall_instruction() { +#[no_mangle] +pub unsafe extern "C" fn __inner_syscall_instruction(stack: *mut InterruptStack) { with_interrupt_stack! { unsafe fn inner(stack) -> usize { let rbp; @@ -48,46 +48,50 @@ pub unsafe extern fn syscall_instruction() { } } - // Yes, this is magic. No, you don't need to understand - asm!(" - swapgs // Set gs segment to TSS - mov gs:[28], rsp // Save userspace rsp - mov rsp, gs:[4] // Load kernel rsp - push 5 * 8 + 3 // Push userspace data segment - push qword ptr gs:[28] // Push userspace rsp - mov qword ptr gs:[28], 0 // Clear userspace rsp - push r11 // Push rflags - push 4 * 8 + 3 // Push userspace code segment - push rcx // Push userspace return pointer - swapgs // Restore gs - " - : - : - : - : "intel", "volatile"); - - // Push scratch registers - interrupt_push!(); - - // Get reference to stack variables - let rsp: usize; - asm!("" : "={rsp}"(rsp) : : : "intel", "volatile"); - - // Map kernel pti::map(); - - inner(rsp as *mut InterruptStack); - - // Unmap kernel + inner(stack); pti::unmap(); - - // Interrupt return - interrupt_pop!(); - iret!() } -#[naked] -pub unsafe extern fn syscall() { +function!("syscall_instruction" => { + // Yes, this is magic. No, you don't need to understand + " + swapgs // Set gs segment to TSS + mov gs:[28], rsp // Save userspace rsp + mov rsp, gs:[4] // Load kernel rsp + push 5 * 8 + 3 // Push userspace data segment + push qword ptr gs:[28] // Push userspace rsp + mov qword ptr gs:[28], 0 // Clear userspace rsp + push r11 // Push rflags + push 4 * 8 + 3 // Push userspace code segment + push rcx // Push userspace return pointer + swapgs // Restore gs + ", + + // Push context registers + "push rax\n", + push_scratch!(), + push_preserved!(), + push_fs!(), + + // Call inner funtion + "mov rdi, rsp\n", + "call __inner_syscall_instruction\n", + + // Pop context registers + pop_fs!(), + pop_preserved!(), + pop_scratch!(), + + // Return + "iretq\n", +}); + +extern "C" { + pub fn syscall_instruction(); +} + +interrupt_stack!(syscall, |stack| { with_interrupt_stack! { unsafe fn inner(stack) -> usize { let rbp; @@ -97,26 +101,8 @@ pub unsafe extern fn syscall() { syscall::syscall(scratch.rax, stack.preserved.rbx, scratch.rcx, scratch.rdx, scratch.rsi, scratch.rdi, rbp, stack) } } - - // Push scratch registers - interrupt_push!(); - - // Get reference to stack variables - let rsp: usize; - asm!("" : "={rsp}"(rsp) : : : "intel", "volatile"); - - // Map kernel - pti::map(); - - inner(rsp as *mut InterruptStack); - - // Unmap kernel - pti::unmap(); - - // Interrupt return - interrupt_pop!(); - iret!(); -} + inner(stack); +}); #[naked] pub unsafe extern "C" fn clone_ret() { diff --git a/src/arch/x86_64/macros.rs b/src/arch/x86_64/macros.rs index 6271229..5f888cb 100644 --- a/src/arch/x86_64/macros.rs +++ b/src/arch/x86_64/macros.rs @@ -1,6 +1,3 @@ -use core::mem; -use syscall::data::IntRegisters; - /// Print to console #[macro_export] macro_rules! print { @@ -18,400 +15,6 @@ macro_rules! println { ($fmt:expr, $($arg:tt)*) => (print!(concat!($fmt, "\n"), $($arg)*)); } -#[allow(dead_code)] -#[derive(Default)] -#[repr(packed)] -pub struct ScratchRegisters { - pub r11: usize, - pub r10: usize, - pub r9: usize, - pub r8: usize, - pub rsi: usize, - pub rdi: usize, - pub rdx: usize, - pub rcx: usize, - pub rax: usize, -} - -impl ScratchRegisters { - pub fn dump(&self) { - println!("RAX: {:>016X}", { self.rax }); - println!("RCX: {:>016X}", { self.rcx }); - println!("RDX: {:>016X}", { self.rdx }); - println!("RDI: {:>016X}", { self.rdi }); - println!("RSI: {:>016X}", { self.rsi }); - println!("R8: {:>016X}", { self.r8 }); - println!("R9: {:>016X}", { self.r9 }); - println!("R10: {:>016X}", { self.r10 }); - println!("R11: {:>016X}", { self.r11 }); - } -} - -macro_rules! scratch_push { - () => (asm!( - "push rax - push rcx - push rdx - push rdi - push rsi - push r8 - push r9 - push r10 - push r11" - : : : : "intel", "volatile" - )); -} - -macro_rules! scratch_pop { - () => (asm!( - "pop r11 - pop r10 - pop r9 - pop r8 - pop rsi - pop rdi - pop rdx - pop rcx - pop rax" - : : : : "intel", "volatile" - )); -} - -#[allow(dead_code)] -#[derive(Default)] -#[repr(packed)] -pub struct PreservedRegisters { - pub r15: usize, - pub r14: usize, - pub r13: usize, - pub r12: usize, - pub rbp: usize, - pub rbx: usize, -} - -impl PreservedRegisters { - pub fn dump(&self) { - println!("RBX: {:>016X}", { self.rbx }); - println!("RBP: {:>016X}", { self.rbp }); - println!("R12: {:>016X}", { self.r12 }); - println!("R13: {:>016X}", { self.r13 }); - println!("R14: {:>016X}", { self.r14 }); - println!("R15: {:>016X}", { self.r15 }); - } -} - -macro_rules! preserved_push { - () => (asm!( - "push rbx - push rbp - push r12 - push r13 - push r14 - push r15" - : : : : "intel", "volatile" - )); -} - -macro_rules! preserved_pop { - () => (asm!( - "pop r15 - pop r14 - pop r13 - pop r12 - pop rbp - pop rbx" - : : : : "intel", "volatile" - )); -} - -macro_rules! fs_push { - () => (asm!( - " - push fs - - // Load kernel tls - mov rax, 0x18 - mov fs, ax // can't load value directly into `fs` - " - : : : : "intel", "volatile" - )); -} - -macro_rules! fs_pop { - () => (asm!( - "pop fs" - : : : : "intel", "volatile" - )); -} - -#[allow(dead_code)] -#[derive(Default)] -#[repr(packed)] -pub struct IretRegisters { - pub rip: usize, - pub cs: usize, - pub rflags: usize, - // Will only be present if interrupt is raised from another - // privilege ring - pub rsp: usize, - pub ss: usize -} - -impl IretRegisters { - pub fn dump(&self) { - println!("RFLAG: {:>016X}", { self.rflags }); - println!("CS: {:>016X}", { self.cs }); - println!("RIP: {:>016X}", { self.rip }); - } -} - -macro_rules! iret { - () => (asm!( - "iretq" - : : : : "intel", "volatile" - )); -} - -/// Create an interrupt function that can safely run rust code -#[macro_export] -macro_rules! interrupt { - ($name:ident, $func:block) => { - #[naked] - pub unsafe extern fn $name () { - #[inline(never)] - unsafe fn inner() { - $func - } - - // Push scratch registers - scratch_push!(); - fs_push!(); - - // Map kernel - $crate::arch::x86_64::pti::map(); - - // Call inner rust function - inner(); - - // Unmap kernel - $crate::arch::x86_64::pti::unmap(); - - // Pop scratch registers and return - fs_pop!(); - scratch_pop!(); - iret!(); - } - }; -} - -#[allow(dead_code)] -#[derive(Default)] -#[repr(packed)] -pub struct InterruptStack { - pub fs: usize, - pub preserved: PreservedRegisters, - pub scratch: ScratchRegisters, - pub iret: IretRegisters, -} - -impl InterruptStack { - pub fn dump(&self) { - self.iret.dump(); - self.scratch.dump(); - self.preserved.dump(); - println!("FS: {:>016X}", { self.fs }); - } - /// Saves all registers to a struct used by the proc: - /// scheme to read/write registers. - pub fn save(&self, all: &mut IntRegisters) { - all.fs = self.fs; - - all.r15 = self.preserved.r15; - all.r14 = self.preserved.r14; - all.r13 = self.preserved.r13; - all.r12 = self.preserved.r12; - all.rbp = self.preserved.rbp; - all.rbx = self.preserved.rbx; - all.r11 = self.scratch.r11; - all.r10 = self.scratch.r10; - all.r9 = self.scratch.r9; - all.r8 = self.scratch.r8; - all.rsi = self.scratch.rsi; - all.rdi = self.scratch.rdi; - all.rdx = self.scratch.rdx; - all.rcx = self.scratch.rcx; - all.rax = self.scratch.rax; - all.rip = self.iret.rip; - all.cs = self.iret.cs; - all.rflags = self.iret.rflags; - - // Set rsp and ss: - - const CPL_MASK: usize = 0b11; - - let cs: usize; - unsafe { - asm!("mov $0, cs" : "=r"(cs) ::: "intel"); - } - - if self.iret.cs & CPL_MASK == cs & CPL_MASK { - // Privilege ring didn't change, so neither did the stack - all.rsp = self as *const Self as usize // rsp after Self was pushed to the stack - + mem::size_of::() // disregard Self - - mem::size_of::() * 2; // well, almost: rsp and ss need to be excluded as they aren't present - unsafe { - asm!("mov $0, ss" : "=r"(all.ss) ::: "intel"); - } - } else { - all.rsp = self.iret.rsp; - all.ss = self.iret.ss; - } - } - /// Loads all registers from a struct used by the proc: - /// scheme to read/write registers. - pub fn load(&mut self, all: &IntRegisters) { - // TODO: Which of these should be allowed to change? - - // self.fs = all.fs; - self.preserved.r15 = all.r15; - self.preserved.r14 = all.r14; - self.preserved.r13 = all.r13; - self.preserved.r12 = all.r12; - self.preserved.rbp = all.rbp; - self.preserved.rbx = all.rbx; - self.scratch.r11 = all.r11; - self.scratch.r10 = all.r10; - self.scratch.r9 = all.r9; - self.scratch.r8 = all.r8; - self.scratch.rsi = all.rsi; - self.scratch.rdi = all.rdi; - self.scratch.rdx = all.rdx; - self.scratch.rcx = all.rcx; - self.scratch.rax = all.rax; - self.iret.rip = all.rip; - - // These should probably be restricted - // self.iret.cs = all.cs; - // self.iret.rflags = all.eflags; - } - /// Enables the "Trap Flag" in the FLAGS register, causing the CPU - /// to send a Debug exception after the next instruction. This is - /// used for singlestep in the proc: scheme. - pub fn set_singlestep(&mut self, enabled: bool) { - if enabled { - self.iret.rflags |= 1 << 8; - } else { - self.iret.rflags &= !(1 << 8); - } - } - /// Checks if the trap flag is enabled, see `set_singlestep` - pub fn is_singlestep(&self) -> bool { - self.iret.rflags & 1 << 8 == 1 << 8 - } -} - -macro_rules! interrupt_push { - () => { - scratch_push!(); - preserved_push!(); - fs_push!(); - }; -} -macro_rules! interrupt_pop { - () => { - fs_pop!(); - preserved_pop!(); - scratch_pop!(); - }; -} - -#[macro_export] -macro_rules! interrupt_stack { - ($name:ident, $stack:ident, $func:block) => { - #[naked] - pub unsafe extern fn $name () { - #[inline(never)] - unsafe fn inner($stack: &mut $crate::arch::x86_64::macros::InterruptStack) { - $func - } - - // Push scratch registers - interrupt_push!(); - - // Get reference to stack variables - let rsp: usize; - asm!("" : "={rsp}"(rsp) : : : "intel", "volatile"); - - // Map kernel - $crate::arch::x86_64::pti::map(); - - // Call inner rust function - inner(&mut *(rsp as *mut $crate::arch::x86_64::macros::InterruptStack)); - - // Unmap kernel - $crate::arch::x86_64::pti::unmap(); - - // Pop scratch registers and return - interrupt_pop!(); - iret!(); - } - }; -} - -#[allow(dead_code)] -#[derive(Default)] -#[repr(packed)] -pub struct InterruptErrorStack { - pub fs: usize, - pub preserved: PreservedRegisters, - pub scratch: ScratchRegisters, - pub code: usize, - pub iret: IretRegisters, -} - -impl InterruptErrorStack { - pub fn dump(&self) { - self.iret.dump(); - println!("CODE: {:>016X}", { self.code }); - self.scratch.dump(); - self.preserved.dump(); - println!("FS: {:>016X}", { self.fs }); - } -} - -#[macro_export] -macro_rules! interrupt_error { - ($name:ident, $stack:ident, $func:block) => { - #[naked] - pub unsafe extern fn $name () { - #[inline(never)] - unsafe fn inner($stack: &$crate::arch::x86_64::macros::InterruptErrorStack) { - $func - } - - // Push scratch registers - interrupt_push!(); - - // Get reference to stack variables - let rsp: usize; - asm!("" : "={rsp}"(rsp) : : : "intel", "volatile"); - - // Map kernel - $crate::arch::x86_64::pti::map(); - - // Call inner rust function - inner(&*(rsp as *const $crate::arch::x86_64::macros::InterruptErrorStack)); - - // Unmap kernel - $crate::arch::x86_64::pti::unmap(); - - // Pop scratch registers, error code, and return - interrupt_pop!(); - asm!("add rsp, 8" : : : : "intel", "volatile"); // pop error code - iret!(); - } - }; -} #[macro_export] macro_rules! irqs( ( [ $( ($idt:expr, $number:literal, $name:ident) ,)* ], $submac:ident ) => { diff --git a/src/context/context.rs b/src/context/context.rs index 1627b6e..0814ee6 100644 --- a/src/context/context.rs +++ b/src/context/context.rs @@ -7,7 +7,7 @@ use core::cmp::Ordering; use core::mem; use spin::Mutex; -use crate::arch::{macros::InterruptStack, paging::PAGE_SIZE}; +use crate::arch::{interrupt::InterruptStack, paging::PAGE_SIZE}; use crate::common::unique::Unique; use crate::context::arch; use crate::context::file::{FileDescriptor, FileDescription}; diff --git a/src/lib.rs b/src/lib.rs index 8627940..4df4412 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -45,10 +45,11 @@ #![feature(concat_idents)] #![feature(const_fn)] #![feature(core_intrinsics)] +#![feature(global_asm)] #![feature(integer_atomics)] #![feature(lang_items)] -#![feature(naked_functions)] #![feature(matches_macro)] // stable in current Rust +#![feature(naked_functions)] #![feature(ptr_internals)] #![feature(thread_local)] #![no_std] diff --git a/src/ptrace.rs b/src/ptrace.rs index d098d9e..6158df1 100644 --- a/src/ptrace.rs +++ b/src/ptrace.rs @@ -4,7 +4,7 @@ use crate::{ arch::{ - macros::InterruptStack, + interrupt::InterruptStack, paging::{ entry::EntryFlags, mapper::MapperFlushAll, diff --git a/src/syscall/driver.rs b/src/syscall/driver.rs index 7cf168d..6318426 100644 --- a/src/syscall/driver.rs +++ b/src/syscall/driver.rs @@ -1,4 +1,4 @@ -use crate::macros::InterruptStack; +use crate::interrupt::InterruptStack; use crate::memory::{allocate_frames_complex, deallocate_frames, Frame}; use crate::paging::{ActivePageTable, PhysicalAddress, VirtualAddress}; use crate::paging::entry::EntryFlags; diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index bb01f60..b21b9d2 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -20,7 +20,7 @@ use self::flag::{CloneFlags, MapFlags, PhysmapFlags, WaitFlags}; use self::number::*; use crate::context::ContextId; -use crate::macros::InterruptStack; +use crate::interrupt::InterruptStack; use crate::scheme::{FileHandle, SchemeNamespace}; /// Debug