Only swapgs when leaving/entering userspace code.

src/arch/x86_64/gdt.rs

@@ -193,13 +193,16 @@ pub unsafe fn init_paging(tcb_offset: usize, stack_offset: usize) {
     segmentation::load_fs(SegmentSelector::new(GDT_KERNEL_TLS as u16, Ring::Ring0));
 
     segmentation::load_gs(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
-    // Ensure that GS always points to the TSS segment in kernel space.
-    //x86::msr::wrmsr(x86::msr::IA32_GS_BASE, &TSS as *const _ as usize as u64);
-
     segmentation::load_ss(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
 
     // Load the task register
     task::load_tr(SegmentSelector::new(GDT_TSS as u16, Ring::Ring0));
+
+    // Ensure that GS always points to the TSS segment in kernel space.
+    x86::msr::wrmsr(x86::msr::IA32_GS_BASE, &TSS as *const _ as usize as u64);
+    // Inside kernel space, GS should _always_ point to the TSS. When leaving userspace, `swapgs`
+    // is called again, making the userspace GS always point to user data.
+    x86::msr::wrmsr(x86::msr::IA32_KERNEL_GSBASE, 0);
 }
 
 #[derive(Copy, Clone, Debug)]

src/arch/x86_64/interrupt/exception.rs

@@ -18,7 +18,7 @@ interrupt_stack!(divide_by_zero, |stack| {
     ksignal(SIGFPE);
 });
 
-interrupt_stack!(debug, |stack| {
+interrupt_stack!(debug, super_atomic: swapgs_iff_ring3_slow!, |stack| {
     let mut handled = false;
 
     // Disable singlestep before there is a breakpoint, since the breakpoint
@@ -41,7 +41,7 @@ interrupt_stack!(debug, |stack| {
     }
 });
 
-interrupt_stack!(non_maskable, |stack| {
+interrupt_stack!(non_maskable, super_atomic: swapgs_iff_ring3_slow!, |stack| {
     println!("Non-maskable interrupt");
     stack.dump();
 });
@@ -153,7 +153,7 @@ interrupt_error!(alignment_check, |stack| {
     ksignal(SIGBUS);
 });
 
-interrupt_stack!(machine_check, |stack| {
+interrupt_stack!(machine_check, super_atomic: swapgs_iff_ring3_slow!, |stack| {
     println!("Machine check fault");
     stack.dump();
     stack_trace();

src/arch/x86_64/interrupt/handler.rs

@@ -309,8 +309,8 @@ macro_rules! pop_fs {
     " };
 }
 
-macro_rules! swapgs_iff_ring3 {
-    (error_code: true) => { "
+macro_rules! swapgs_iff_ring3_fast {
+    () => { "
         // Check whether the last two bits RSP+8 (code segment) are equal to zero.
         test QWORD PTR [rsp + 8], 0x3
         // Skip the SWAPGS instruction if CS & 0b11 == 0b00.
@@ -318,19 +318,22 @@ macro_rules! swapgs_iff_ring3 {
         swapgs
         1:
     " };
-    (error_code: false) => { "
+}
+macro_rules! swapgs_iff_ring3_fast_errorcode {
+    () => { "
         test QWORD PTR [rsp + 16], 0x3
         jz 1f
         swapgs
         1:
     " };
 }
+#[macro_export]
 macro_rules! swapgs_iff_ring3_slow {
     () => { "
-    push eax
-    push edx
-    push ecx
-    mov ecx, 0xC000_0102
+    push rax
+    push rdx
+    push rcx
+    mov ecx, 0xC0000102
     rdmsr
     shl rdx, 32
     or eax, edx
@@ -338,15 +341,17 @@ macro_rules! swapgs_iff_ring3_slow {
     jnz 1f
     swapgs
     1:
-    pop ecx
-    pop edx
-    pop eax
+    pop rcx
+    pop rdx
+    pop rax
     " }
 }
 
 #[macro_export]
 macro_rules! interrupt_stack {
-    ($name:ident, |$stack:ident| $code:block) => {
+    // XXX: Apparently we cannot use $expr and check for bool exhaustiveness, so we will have to
+    // use idents directly instead.
+    ($name:ident, super_atomic: $is_super_atomic:ident!, |$stack:ident| $code:block) => {
         paste::item! {
             #[no_mangle]
             unsafe extern "C" fn [<__interrupt_ $name>](stack: *mut $crate::arch::x86_64::interrupt::InterruptStack) {
@@ -362,7 +367,7 @@ macro_rules! interrupt_stack {
 
             function!($name => {
                 // Backup all userspace registers to stack
-                //swapgs_if_ring3!(error_code: false),
+                $is_super_atomic!(),
                 "push rax\n",
                 push_scratch!(),
                 push_preserved!(),
@@ -383,11 +388,12 @@ macro_rules! interrupt_stack {
                 pop_preserved!(),
                 pop_scratch!(),
 
-                //swapgs_if_ring3!(error_code: false),
+                $is_super_atomic!(),
                 "iretq\n",
             });
         }
     };
+    ($name:ident, |$stack:ident| $code:block) => { interrupt_stack!($name, super_atomic: swapgs_iff_ring3_fast!, |$stack| $code); };
 }
 
 #[macro_export]
@@ -401,7 +407,7 @@ macro_rules! interrupt {
 
             function!($name => {
                 // Backup all userspace registers to stack
-                //swapgs_if_ring3!(error_code: false),
+                swapgs_iff_ring3_fast!(),
                 "push rax\n",
                 push_scratch!(),
                 push_fs!(),
@@ -419,7 +425,7 @@ macro_rules! interrupt {
                 pop_fs!(),
                 pop_scratch!(),
 
-                //swapgs_if_ring3!(error_code: false),
+                swapgs_iff_ring3_fast!(),
                 "iretq\n",
             });
         }
@@ -443,7 +449,7 @@ macro_rules! interrupt_error {
             }
 
             function!($name => {
-                //swapgs_if_ring3!(error_code: true),
+                swapgs_iff_ring3_fast_errorcode!(),
                 // Move rax into code's place, put code in last instead (to be
                 // compatible with InterruptStack)
                 "xchg [rsp], rax\n",
@@ -474,7 +480,7 @@ macro_rules! interrupt_error {
                 pop_preserved!(),
                 pop_scratch!(),
 
-                //swapgs_if_ring3!(error_code: true),
+                swapgs_iff_ring3_fast_errorcode!(),
                 "iretq\n",
             });
         }

src/arch/x86_64/interrupt/syscall.rs

@@ -23,10 +23,6 @@ pub unsafe fn init() {
     msr::wrmsr(msr::IA32_LSTAR, syscall_instruction as u64);
     msr::wrmsr(msr::IA32_FMASK, 0x0300); // Clear trap flag and interrupt enable
 
-    // Inside kernel space, GS should _always_ point to the TSS. When leaving userspace, `swapgs`
-    // is called again, making the userspace GS always point to user data.
-    x86::msr::wrmsr(x86::msr::IA32_KERNEL_GSBASE, &gdt::TSS as *const _ as usize as u64);
-
     let efer = msr::rdmsr(msr::IA32_EFER);
     msr::wrmsr(msr::IA32_EFER, efer | 1);
 }
@@ -73,7 +69,6 @@ function!(syscall_instruction => {
         push r11                  // Push rflags
         push QWORD PTR 6 * 8 + 3  // Push fake userspace CS (resembling iret frame)
         push rcx                  // Push userspace return pointer
-        swapgs
     ",
 
     // Push context registers
@@ -102,7 +97,6 @@ function!(syscall_instruction => {
         pop rcx                 // Pop userspace return pointer
         add rsp, 8              // Pop CS
         pop r11                 // Pop rflags
-        swapgs
         pop QWORD PTR gs:[0x70] // Pop userspace stack pointer
         add rsp, 8              // Pop SS
         mov rsp, gs:[0x70]      // Restore userspace stack pointer

src/arch/x86_64/start.rs

@@ -262,6 +262,7 @@ pub unsafe extern "C" fn usermode(_ip: usize, _sp: usize, _arg: usize, _singlest
             call {pti_unmap}
 
             // Go to usermode
+            swapgs
             mov r8, {user_data_seg_selector}
             mov r9, {user_tls_seg_selector}
             mov ds, r8d