diff --git a/src/arch/x86_64/gdt.rs b/src/arch/x86_64/gdt.rs index a285d9a..e999abe 100644 --- a/src/arch/x86_64/gdt.rs +++ b/src/arch/x86_64/gdt.rs @@ -193,13 +193,16 @@ pub unsafe fn init_paging(tcb_offset: usize, stack_offset: usize) { segmentation::load_fs(SegmentSelector::new(GDT_KERNEL_TLS as u16, Ring::Ring0)); segmentation::load_gs(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); - // Ensure that GS always points to the TSS segment in kernel space. - //x86::msr::wrmsr(x86::msr::IA32_GS_BASE, &TSS as *const _ as usize as u64); - segmentation::load_ss(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); // Load the task register task::load_tr(SegmentSelector::new(GDT_TSS as u16, Ring::Ring0)); + + // Ensure that GS always points to the TSS segment in kernel space. + x86::msr::wrmsr(x86::msr::IA32_GS_BASE, &TSS as *const _ as usize as u64); + // Inside kernel space, GS should _always_ point to the TSS. When leaving userspace, `swapgs` + // is called again, making the userspace GS always point to user data. + x86::msr::wrmsr(x86::msr::IA32_KERNEL_GSBASE, 0); } #[derive(Copy, Clone, Debug)] diff --git a/src/arch/x86_64/interrupt/exception.rs b/src/arch/x86_64/interrupt/exception.rs index 18b4f06..621379f 100644 --- a/src/arch/x86_64/interrupt/exception.rs +++ b/src/arch/x86_64/interrupt/exception.rs @@ -18,7 +18,7 @@ interrupt_stack!(divide_by_zero, |stack| { ksignal(SIGFPE); }); -interrupt_stack!(debug, |stack| { +interrupt_stack!(debug, super_atomic: swapgs_iff_ring3_slow!, |stack| { let mut handled = false; // Disable singlestep before there is a breakpoint, since the breakpoint @@ -41,7 +41,7 @@ interrupt_stack!(debug, |stack| { } }); -interrupt_stack!(non_maskable, |stack| { +interrupt_stack!(non_maskable, super_atomic: swapgs_iff_ring3_slow!, |stack| { println!("Non-maskable interrupt"); stack.dump(); }); @@ -153,7 +153,7 @@ interrupt_error!(alignment_check, |stack| { ksignal(SIGBUS); }); -interrupt_stack!(machine_check, |stack| { +interrupt_stack!(machine_check, super_atomic: swapgs_iff_ring3_slow!, |stack| { println!("Machine check fault"); stack.dump(); stack_trace(); diff --git a/src/arch/x86_64/interrupt/handler.rs b/src/arch/x86_64/interrupt/handler.rs index 776b98d..626e1e3 100644 --- a/src/arch/x86_64/interrupt/handler.rs +++ b/src/arch/x86_64/interrupt/handler.rs @@ -309,8 +309,8 @@ macro_rules! pop_fs { " }; } -macro_rules! swapgs_iff_ring3 { - (error_code: true) => { " +macro_rules! swapgs_iff_ring3_fast { + () => { " // Check whether the last two bits RSP+8 (code segment) are equal to zero. test QWORD PTR [rsp + 8], 0x3 // Skip the SWAPGS instruction if CS & 0b11 == 0b00. @@ -318,19 +318,22 @@ macro_rules! swapgs_iff_ring3 { swapgs 1: " }; - (error_code: false) => { " +} +macro_rules! swapgs_iff_ring3_fast_errorcode { + () => { " test QWORD PTR [rsp + 16], 0x3 jz 1f swapgs 1: " }; } +#[macro_export] macro_rules! swapgs_iff_ring3_slow { () => { " - push eax - push edx - push ecx - mov ecx, 0xC000_0102 + push rax + push rdx + push rcx + mov ecx, 0xC0000102 rdmsr shl rdx, 32 or eax, edx @@ -338,15 +341,17 @@ macro_rules! swapgs_iff_ring3_slow { jnz 1f swapgs 1: - pop ecx - pop edx - pop eax + pop rcx + pop rdx + pop rax " } } #[macro_export] macro_rules! interrupt_stack { - ($name:ident, |$stack:ident| $code:block) => { + // XXX: Apparently we cannot use $expr and check for bool exhaustiveness, so we will have to + // use idents directly instead. + ($name:ident, super_atomic: $is_super_atomic:ident!, |$stack:ident| $code:block) => { paste::item! { #[no_mangle] unsafe extern "C" fn [<__interrupt_ $name>](stack: *mut $crate::arch::x86_64::interrupt::InterruptStack) { @@ -362,7 +367,7 @@ macro_rules! interrupt_stack { function!($name => { // Backup all userspace registers to stack - //swapgs_if_ring3!(error_code: false), + $is_super_atomic!(), "push rax\n", push_scratch!(), push_preserved!(), @@ -383,11 +388,12 @@ macro_rules! interrupt_stack { pop_preserved!(), pop_scratch!(), - //swapgs_if_ring3!(error_code: false), + $is_super_atomic!(), "iretq\n", }); } }; + ($name:ident, |$stack:ident| $code:block) => { interrupt_stack!($name, super_atomic: swapgs_iff_ring3_fast!, |$stack| $code); }; } #[macro_export] @@ -401,7 +407,7 @@ macro_rules! interrupt { function!($name => { // Backup all userspace registers to stack - //swapgs_if_ring3!(error_code: false), + swapgs_iff_ring3_fast!(), "push rax\n", push_scratch!(), push_fs!(), @@ -419,7 +425,7 @@ macro_rules! interrupt { pop_fs!(), pop_scratch!(), - //swapgs_if_ring3!(error_code: false), + swapgs_iff_ring3_fast!(), "iretq\n", }); } @@ -443,7 +449,7 @@ macro_rules! interrupt_error { } function!($name => { - //swapgs_if_ring3!(error_code: true), + swapgs_iff_ring3_fast_errorcode!(), // Move rax into code's place, put code in last instead (to be // compatible with InterruptStack) "xchg [rsp], rax\n", @@ -474,7 +480,7 @@ macro_rules! interrupt_error { pop_preserved!(), pop_scratch!(), - //swapgs_if_ring3!(error_code: true), + swapgs_iff_ring3_fast_errorcode!(), "iretq\n", }); } diff --git a/src/arch/x86_64/interrupt/syscall.rs b/src/arch/x86_64/interrupt/syscall.rs index 1df5906..ef8ec59 100644 --- a/src/arch/x86_64/interrupt/syscall.rs +++ b/src/arch/x86_64/interrupt/syscall.rs @@ -23,10 +23,6 @@ pub unsafe fn init() { msr::wrmsr(msr::IA32_LSTAR, syscall_instruction as u64); msr::wrmsr(msr::IA32_FMASK, 0x0300); // Clear trap flag and interrupt enable - // Inside kernel space, GS should _always_ point to the TSS. When leaving userspace, `swapgs` - // is called again, making the userspace GS always point to user data. - x86::msr::wrmsr(x86::msr::IA32_KERNEL_GSBASE, &gdt::TSS as *const _ as usize as u64); - let efer = msr::rdmsr(msr::IA32_EFER); msr::wrmsr(msr::IA32_EFER, efer | 1); } @@ -73,7 +69,6 @@ function!(syscall_instruction => { push r11 // Push rflags push QWORD PTR 6 * 8 + 3 // Push fake userspace CS (resembling iret frame) push rcx // Push userspace return pointer - swapgs ", // Push context registers @@ -102,7 +97,6 @@ function!(syscall_instruction => { pop rcx // Pop userspace return pointer add rsp, 8 // Pop CS pop r11 // Pop rflags - swapgs pop QWORD PTR gs:[0x70] // Pop userspace stack pointer add rsp, 8 // Pop SS mov rsp, gs:[0x70] // Restore userspace stack pointer diff --git a/src/arch/x86_64/start.rs b/src/arch/x86_64/start.rs index f13aeb9..3ebe195 100644 --- a/src/arch/x86_64/start.rs +++ b/src/arch/x86_64/start.rs @@ -262,6 +262,7 @@ pub unsafe extern "C" fn usermode(_ip: usize, _sp: usize, _arg: usize, _singlest call {pti_unmap} // Go to usermode + swapgs mov r8, {user_data_seg_selector} mov r9, {user_tls_seg_selector} mov ds, r8d