From 8f507857816d573a647cdba60b69f061be48c59c Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Mon, 3 May 2021 21:57:45 -0600 Subject: [PATCH] Fixes for building aarch64 --- src/arch/aarch64/consts.rs | 5 + src/arch/aarch64/device/gic.rs | 13 +- src/arch/aarch64/device/rtc.rs | 9 +- src/arch/aarch64/device/serial.rs | 13 +- src/arch/aarch64/paging/mod.rs | 97 +++---- src/arch/aarch64/paging/temporary_page.rs | 15 +- src/arch/aarch64/rmm.rs | 18 +- src/arch/aarch64/start.rs | 5 +- src/arch/x86_64/paging/mapper.rs | 6 +- src/context/arch/aarch64.rs | 304 +++++++++++----------- 10 files changed, 243 insertions(+), 242 deletions(-) diff --git a/src/arch/aarch64/consts.rs b/src/arch/aarch64/consts.rs index a348714..12a49bb 100644 --- a/src/arch/aarch64/consts.rs +++ b/src/arch/aarch64/consts.rs @@ -48,6 +48,11 @@ /// Size of kernel percpu variables pub const KERNEL_PERCPU_SIZE: usize = 64 * 1024; // 64 KB + /// Offset of physmap + // This needs to match RMM's PHYS_OFFSET + pub const PHYS_OFFSET: usize = 0xFFFF_FE00_0000_0000; + pub const PHYS_PML4: usize = (PHYS_OFFSET & PML4_MASK)/PML4_SIZE; + /// Offset to user image pub const USER_OFFSET: usize = 0; pub const USER_PML4: usize = (USER_OFFSET & PML4_MASK)/PML4_SIZE; diff --git a/src/arch/aarch64/device/gic.rs b/src/arch/aarch64/device/gic.rs index 92c3724..fcf32db 100644 --- a/src/arch/aarch64/device/gic.rs +++ b/src/arch/aarch64/device/gic.rs @@ -1,8 +1,7 @@ use core::intrinsics::{volatile_load, volatile_store}; use crate::memory::Frame; -use crate::paging::{ActivePageTable, PhysicalAddress, Page, PageTableType, VirtualAddress}; -use crate::paging::entry::EntryFlags; +use crate::paging::{ActivePageTable, PhysicalAddress, Page, PageFlags, TableKind, VirtualAddress}; static GICD_CTLR: u32 = 0x000; static GICD_TYPER: u32 = 0x004; @@ -57,14 +56,14 @@ pub struct GicDistIf { impl GicDistIf { unsafe fn init(&mut self) { // Map in the Distributor interface - let mut active_table = ActivePageTable::new(PageTableType::Kernel); + let mut active_table = ActivePageTable::new(TableKind::Kernel); let start_frame = Frame::containing_address(PhysicalAddress::new(0x08000000)); let end_frame = Frame::containing_address(PhysicalAddress::new(0x08000000 + 0x10000 - 1)); for frame in Frame::range_inclusive(start_frame, end_frame) { let page = Page::containing_address(VirtualAddress::new(frame.start_address().data() + crate::KERNEL_DEVMAP_OFFSET)); - let result = active_table.map_to(page, frame, EntryFlags::PRESENT | EntryFlags::WRITABLE | EntryFlags::NO_EXECUTE); - result.flush(&mut active_table); + let result = active_table.map_to(page, frame, PageFlags::new().write(true)); + result.flush(); } self.address = crate::KERNEL_DEVMAP_OFFSET + 0x08000000; @@ -74,8 +73,8 @@ impl GicDistIf { let end_frame = Frame::containing_address(PhysicalAddress::new(0x08010000 + 0x10000 - 1)); for frame in Frame::range_inclusive(start_frame, end_frame) { let page = Page::containing_address(VirtualAddress::new(frame.start_address().data() + crate::KERNEL_DEVMAP_OFFSET)); - let result = active_table.map_to(page, frame, EntryFlags::PRESENT | EntryFlags::WRITABLE | EntryFlags::NO_EXECUTE); - result.flush(&mut active_table); + let result = active_table.map_to(page, frame, PageFlags::new().write(true)); + result.flush(); } GIC_CPU_IF.address = crate::KERNEL_DEVMAP_OFFSET + 0x08010000; diff --git a/src/arch/aarch64/device/rtc.rs b/src/arch/aarch64/device/rtc.rs index e547523..c035e41 100644 --- a/src/arch/aarch64/device/rtc.rs +++ b/src/arch/aarch64/device/rtc.rs @@ -1,8 +1,7 @@ use core::intrinsics::{volatile_load, volatile_store}; use crate::memory::Frame; -use crate::paging::{ActivePageTable, PhysicalAddress, Page, PageTableType, VirtualAddress}; -use crate::paging::entry::EntryFlags; +use crate::paging::{ActivePageTable, PhysicalAddress, Page, PageFlags, TableKind, VirtualAddress}; use crate::time; static RTC_DR: u32 = 0x000; @@ -29,15 +28,15 @@ struct Pl031rtc { impl Pl031rtc { unsafe fn init(&mut self) { - let mut active_table = ActivePageTable::new(PageTableType::Kernel); + let mut active_table = ActivePageTable::new(TableKind::Kernel); let start_frame = Frame::containing_address(PhysicalAddress::new(0x09010000)); let end_frame = Frame::containing_address(PhysicalAddress::new(0x09010000 + 0x1000 - 1)); for frame in Frame::range_inclusive(start_frame, end_frame) { let page = Page::containing_address(VirtualAddress::new(frame.start_address().data() + crate::KERNEL_DEVMAP_OFFSET)); - let result = active_table.map_to(page, frame, EntryFlags::PRESENT | EntryFlags::WRITABLE | EntryFlags::NO_EXECUTE); - result.flush(&mut active_table); + let result = active_table.map_to(page, frame, PageFlags::new().write(true)); + result.flush(); } self.address = crate::KERNEL_DEVMAP_OFFSET + 0x09010000; diff --git a/src/arch/aarch64/device/serial.rs b/src/arch/aarch64/device/serial.rs index 1bba91e..a6dd8d6 100644 --- a/src/arch/aarch64/device/serial.rs +++ b/src/arch/aarch64/device/serial.rs @@ -4,9 +4,8 @@ use spin::Mutex; use crate::device::uart_pl011::SerialPort; use crate::init::device_tree; use crate::memory::Frame; -use crate::paging::mapper::{MapperFlushAll, MapperType}; -use crate::paging::{ActivePageTable, Page, PageTableType, PhysicalAddress, VirtualAddress}; -use crate::paging::entry::EntryFlags; +use crate::paging::mapper::PageFlushAll; +use crate::paging::{ActivePageTable, Page, PageFlags, PhysicalAddress, TableKind, VirtualAddress}; pub static COM1: Mutex> = Mutex::new(None); @@ -16,17 +15,17 @@ pub unsafe fn init() { } let (base, size) = device_tree::diag_uart_range(crate::KERNEL_DTB_OFFSET, crate::KERNEL_DTB_MAX_SIZE).unwrap(); - let mut active_ktable = unsafe { ActivePageTable::new(PageTableType::Kernel) }; - let mut flush_all = MapperFlushAll::new(); + let mut active_ktable = unsafe { ActivePageTable::new(TableKind::Kernel) }; + let mut flush_all = PageFlushAll::new(); let start_frame = Frame::containing_address(PhysicalAddress::new(base)); let end_frame = Frame::containing_address(PhysicalAddress::new(base + size - 1)); for frame in Frame::range_inclusive(start_frame, end_frame) { let page = Page::containing_address(VirtualAddress::new(frame.start_address().data() + crate::KERNEL_DEVMAP_OFFSET)); - let result = active_ktable.map_to(page, frame, EntryFlags::PRESENT | EntryFlags::NO_EXECUTE | EntryFlags::WRITABLE); + let result = active_ktable.map_to(page, frame, PageFlags::new().write(true)); flush_all.consume(result); }; - flush_all.flush(&mut active_ktable); + flush_all.flush(); let start_frame = Frame::containing_address(PhysicalAddress::new(base)); let vaddr = start_frame.start_address().data() + crate::KERNEL_DEVMAP_OFFSET; diff --git a/src/arch/aarch64/paging/mod.rs b/src/arch/aarch64/paging/mod.rs index 1d3f5e2..4b10d6c 100644 --- a/src/arch/aarch64/paging/mod.rs +++ b/src/arch/aarch64/paging/mod.rs @@ -8,11 +8,17 @@ use spin::Mutex; use crate::device::cpu::registers::{control_regs, tlb}; use crate::memory::{allocate_frames, Frame}; -use self::entry::{EntryFlags, TableDescriptorFlags}; -use self::mapper::{Mapper, MapperFlushAll}; +use self::mapper::{Mapper, PageFlushAll}; use self::temporary_page::TemporaryPage; -pub use rmm::{PhysicalAddress, TableKind, VirtualAddress}; +pub use rmm::{ + AArch64Arch as RmmA, + Arch as RmmArch, + PageFlags, + PhysicalAddress, + TableKind, + VirtualAddress, +}; pub mod entry; pub mod mapper; @@ -68,7 +74,7 @@ unsafe fn init_mair() { } /// Map TSS -unsafe fn map_tss(cpu_id: usize, mapper: &mut Mapper) -> MapperFlushAll { +unsafe fn map_tss(cpu_id: usize, mapper: &mut Mapper) -> PageFlushAll { extern "C" { /// The starting byte of the thread data segment static mut __tdata_start: u8; @@ -84,17 +90,11 @@ unsafe fn map_tss(cpu_id: usize, mapper: &mut Mapper) -> MapperFlushAll { let start = crate::KERNEL_PERCPU_OFFSET + crate::KERNEL_PERCPU_SIZE * cpu_id; let end = start + size; - let mut flush_all = MapperFlushAll::new(); + let flush_all = PageFlushAll::new(); let start_page = Page::containing_address(VirtualAddress::new(start)); let end_page = Page::containing_address(VirtualAddress::new(end - 1)); for page in Page::range_inclusive(start_page, end_page) { - let result = mapper.map( - page, - EntryFlags::PRESENT - | EntryFlags::GLOBAL - | EntryFlags::NO_EXECUTE - | EntryFlags::WRITABLE, - ); + let result = mapper.map(page, PageFlags::new().write(true)); flush_all.consume(result); } flush_all @@ -171,10 +171,10 @@ pub unsafe fn init( init_mair(); - let mut active_table = ActivePageTable::new_unlocked(PageTableType::Kernel); + let mut active_table = ActivePageTable::new_unlocked(TableKind::Kernel); let flush_all = map_tss(cpu_id, &mut active_table); - flush_all.flush(&mut active_table); + flush_all.flush(); return (active_table, init_tcb(cpu_id)); } @@ -185,7 +185,7 @@ pub unsafe fn init_ap( ) -> usize { init_mair(); - let mut active_table = ActivePageTable::new_unlocked(PageTableType::Kernel); + let mut active_table = ActivePageTable::new_unlocked(TableKind::Kernel); let mut new_table = InactivePageTable::from_address(bsp_table); @@ -206,6 +206,7 @@ pub unsafe fn init_ap( init_tcb(cpu_id) } +#[derive(Debug)] pub struct ActivePageTable { mapper: Mapper, locked: bool, @@ -226,7 +227,6 @@ impl DerefMut for ActivePageTable { } impl ActivePageTable { - //TODO: table_type argument pub unsafe fn new(table_kind: TableKind) -> ActivePageTable { page_table_lock(); ActivePageTable { @@ -235,7 +235,6 @@ impl ActivePageTable { } } - //TODO: table_type argument pub unsafe fn new_unlocked(table_kind: TableKind) -> ActivePageTable { ActivePageTable { mapper: Mapper::new(table_kind), @@ -246,14 +245,14 @@ impl ActivePageTable { pub fn switch(&mut self, new_table: InactivePageTable) -> InactivePageTable { let old_table: InactivePageTable; - match self.mapper.mapper_type { - MapperType::User => { - old_table = InactivePageTable { p4_frame: Frame::containing_address(PhysicalAddress::new(unsafe { control_regs::ttbr0_el1() } as usize)) }; - unsafe { control_regs::ttbr0_el1_write(new_table.p4_frame.start_address().data() as u64) }; + match self.mapper.table_kind { + TableKind::User => { + old_table = InactivePageTable { frame: Frame::containing_address(PhysicalAddress::new(unsafe { control_regs::ttbr0_el1() } as usize)) }; + unsafe { control_regs::ttbr0_el1_write(new_table.frame.start_address().data() as u64) }; }, - MapperType::Kernel => { - old_table = InactivePageTable { p4_frame: Frame::containing_address(PhysicalAddress::new(unsafe { control_regs::ttbr1_el1() } as usize)) }; - unsafe { control_regs::ttbr1_el1_write(new_table.p4_frame.start_address().data() as u64) }; + TableKind::Kernel => { + old_table = InactivePageTable { frame: Frame::containing_address(PhysicalAddress::new(unsafe { control_regs::ttbr1_el1() } as usize)) }; + unsafe { control_regs::ttbr1_el1_write(new_table.frame.start_address().data() as u64) }; } } @@ -277,20 +276,24 @@ impl ActivePageTable { where F: FnOnce(&mut Mapper) { { - let backup: Frame; + let backup = Frame::containing_address(PhysicalAddress::new(unsafe { + match self.mapper.table_kind { + TableKind::User => control_regs::ttbr0_el1() as usize, + TableKind::Kernel => control_regs::ttbr1_el1() as usize, + } + })); - match self.mapper.mapper_type { - MapperType::User => backup = Frame::containing_address(PhysicalAddress::new(unsafe { control_regs::ttbr0_el1() as usize })), - MapperType::Kernel => backup = Frame::containing_address(PhysicalAddress::new(unsafe { control_regs::ttbr1_el1() as usize })) - } - - // map temporary_kpage to current p4 table - let p4_table = temporary_page.map_table_frame(backup.clone(), EntryFlags::PRESENT | EntryFlags::WRITABLE | EntryFlags::NO_EXECUTE, self); + // map temporary_page to current p4 table + let p4_table = temporary_page.map_table_frame( + backup.clone(), + PageFlags::new_table().write(true), //TODO: RISC-V will not like this + self, + ); // overwrite recursive mapping - self.p4_mut()[crate::RECURSIVE_PAGE_PML4].page_table_entry_set( - table.p4_frame.clone(), - TableDescriptorFlags::VALID | TableDescriptorFlags::TABLE, + self.p4_mut()[crate::RECURSIVE_PAGE_PML4].set( + table.frame.clone(), + PageFlags::new_table().write(true), //TODO: RISC-V will not like this ); self.flush_all(); @@ -298,9 +301,9 @@ impl ActivePageTable { f(self); // restore recursive mapping to original p4 table - p4_table[crate::RECURSIVE_PAGE_PML4].page_table_entry_set( + p4_table[crate::RECURSIVE_PAGE_PML4].set( backup, - TableDescriptorFlags::VALID | TableDescriptorFlags::TABLE, + PageFlags::new_table().write(true), //TODO: RISC-V will not like this ); self.flush_all(); } @@ -309,9 +312,9 @@ impl ActivePageTable { } pub unsafe fn address(&self) -> usize { - match self.mapper.mapper_type { - MapperType::User => control_regs::ttbr0_el1() as usize, - MapperType::Kernel => control_regs::ttbr1_el1() as usize, + match self.mapper.table_kind { + TableKind::User => control_regs::ttbr0_el1() as usize, + TableKind::Kernel => control_regs::ttbr1_el1() as usize, } } } @@ -326,7 +329,7 @@ impl Drop for ActivePageTable { } pub struct InactivePageTable { - p4_frame: Frame, + frame: Frame, } impl InactivePageTable { @@ -338,30 +341,30 @@ impl InactivePageTable { { let table = temporary_page.map_table_frame( frame.clone(), - EntryFlags::PRESENT | EntryFlags::WRITABLE | EntryFlags::NO_EXECUTE, + PageFlags::new_table().write(true), //TODO: RISC-V will not like this active_table, ); // now we are able to zero the table table.zero(); // set up recursive mapping for the table - table[crate::RECURSIVE_PAGE_PML4].page_table_entry_set( + table[crate::RECURSIVE_PAGE_PML4].set( frame.clone(), - TableDescriptorFlags::VALID | TableDescriptorFlags::TABLE + PageFlags::new_table().write(true), //TODO: RISC-V will not like this ); } temporary_page.unmap(active_table); - InactivePageTable { p4_frame: frame } + InactivePageTable { frame: frame } } pub unsafe fn from_address(address: usize) -> InactivePageTable { InactivePageTable { - p4_frame: Frame::containing_address(PhysicalAddress::new(address)), + frame: Frame::containing_address(PhysicalAddress::new(address)), } } pub unsafe fn address(&self) -> usize { - self.p4_frame.start_address().data() + self.frame.start_address().data() } } diff --git a/src/arch/aarch64/paging/temporary_page.rs b/src/arch/aarch64/paging/temporary_page.rs index 8ccf441..c8427cc 100644 --- a/src/arch/aarch64/paging/temporary_page.rs +++ b/src/arch/aarch64/paging/temporary_page.rs @@ -3,8 +3,7 @@ use crate::memory::Frame; -use super::{ActivePageTable, Page, VirtualAddress}; -use super::entry::EntryFlags; +use super::{ActivePageTable, Page, PageFlags, RmmA, VirtualAddress}; use super::table::{Table, Level1}; pub struct TemporaryPage { @@ -13,9 +12,7 @@ pub struct TemporaryPage { impl TemporaryPage { pub fn new(page: Page) -> TemporaryPage { - TemporaryPage { - page: page, - } + TemporaryPage { page } } pub fn start_address (&self) -> VirtualAddress { @@ -24,22 +21,22 @@ impl TemporaryPage { /// Maps the temporary page to the given frame in the active table. /// Returns the start address of the temporary page. - pub fn map(&mut self, frame: Frame, flags: EntryFlags, active_table: &mut ActivePageTable) -> VirtualAddress { + pub fn map(&mut self, frame: Frame, flags: PageFlags, active_table: &mut ActivePageTable) -> VirtualAddress { assert!(active_table.translate_page(self.page).is_none(), "temporary page is already mapped"); let result = active_table.map_to(self.page, frame, flags); - result.flush(active_table); + result.flush(); self.page.start_address() } /// Maps the temporary page to the given page table frame in the active /// table. Returns a reference to the now mapped table. - pub fn map_table_frame(&mut self, frame: Frame, flags: EntryFlags, active_table: &mut ActivePageTable) -> &mut Table { + pub fn map_table_frame(&mut self, frame: Frame, flags: PageFlags, active_table: &mut ActivePageTable) -> &mut Table { unsafe { &mut *(self.map(frame, flags, active_table).data() as *mut Table) } } /// Unmaps the temporary page in the active table. pub fn unmap(&mut self, active_table: &mut ActivePageTable) { let (result, _frame) = active_table.unmap_return(self.page, true); - result.flush(active_table); + result.flush(); } } diff --git a/src/arch/aarch64/rmm.rs b/src/arch/aarch64/rmm.rs index 3f7c179..8cff242 100644 --- a/src/arch/aarch64/rmm.rs +++ b/src/arch/aarch64/rmm.rs @@ -1,7 +1,7 @@ use rmm::{ KILOBYTE, MEGABYTE, - AArch64Arch, + AArch64Arch as RmmA, Arch, BuddyAllocator, BumpAllocator, @@ -42,13 +42,13 @@ unsafe fn page_flags(virt: VirtualAddress) -> PageFlags { if in_section!(text) { // Remap text read-only, execute - PageFlags::new().write(false).execute(true) + PageFlags::new().execute(true) } else if in_section!(rodata) { // Remap rodata read-only, no execute - PageFlags::new().write(false).execute(false) + PageFlags::new() } else { // Remap everything else read-write, no execute - PageFlags::new().write(true).execute(false) + PageFlags::new().write(true) } } @@ -181,7 +181,7 @@ unsafe fn inner(areas: &'static [MemoryArea], kernel_base: usize, kerne } pub struct LockedAllocator { - inner: Mutex>>, + inner: Mutex>>, } impl LockedAllocator { @@ -223,21 +223,21 @@ static mut AREAS: [MemoryArea; 512] = [MemoryArea { pub static mut FRAME_ALLOCATOR: LockedAllocator = LockedAllocator::new(); -pub unsafe fn mapper_new(table_addr: PhysicalAddress) -> PageMapper<'static, AArch64Arch, LockedAllocator> { +pub unsafe fn mapper_new(table_addr: PhysicalAddress) -> PageMapper<'static, RmmA, LockedAllocator> { PageMapper::new(table_addr, &mut FRAME_ALLOCATOR) } //TODO: global paging lock? -pub unsafe fn mapper_create() -> Option> { +pub unsafe fn mapper_create() -> Option> { PageMapper::create(&mut FRAME_ALLOCATOR) } -pub unsafe fn mapper_current() -> PageMapper<'static, AArch64Arch, LockedAllocator> { +pub unsafe fn mapper_current() -> PageMapper<'static, RmmA, LockedAllocator> { PageMapper::current(&mut FRAME_ALLOCATOR) } pub unsafe fn init(kernel_base: usize, kernel_size: usize) { - type A = AArch64Arch; + type A = RmmA; let kernel_size_aligned = ((kernel_size + (A::PAGE_SIZE - 1))/A::PAGE_SIZE) * A::PAGE_SIZE; let kernel_end = kernel_base + kernel_size_aligned; diff --git a/src/arch/aarch64/start.rs b/src/arch/aarch64/start.rs index 13d1b2d..b30c649 100644 --- a/src/arch/aarch64/start.rs +++ b/src/arch/aarch64/start.rs @@ -7,8 +7,7 @@ use core::slice; use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use crate::memory::{Frame}; -use crate::paging::{ActivePageTable, PageTableType, Page, PAGE_SIZE, PhysicalAddress, VirtualAddress}; -use crate::paging::entry::{EntryFlags}; +use crate::paging::{ActivePageTable, Page, PAGE_SIZE, PhysicalAddress, VirtualAddress}; use crate::allocator; use crate::device; @@ -174,7 +173,7 @@ pub unsafe extern fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! { } #[naked] -pub unsafe fn usermode(ip: usize, sp: usize, arg: usize, singlestep: bool) -> ! { +pub unsafe fn usermode(ip: usize, sp: usize, arg: usize, _singlestep: u32) -> ! { let cpu_id: usize = 0; let spsr: u32 = 0; diff --git a/src/arch/x86_64/paging/mapper.rs b/src/arch/x86_64/paging/mapper.rs index c2308e4..da45a06 100644 --- a/src/arch/x86_64/paging/mapper.rs +++ b/src/arch/x86_64/paging/mapper.rs @@ -91,7 +91,7 @@ impl Mapper { } if let Some(p1_frame) = p2[page.p2_index()].pointed_frame() { - //println!("Free p1 {:?}", p1_frame); + //println!("unmap_inner: Free p1 {:?}", p1_frame); p2.decrement_entry_count(); p2[page.p2_index()].set_unused(); deallocate_frames(p1_frame, 1); @@ -107,7 +107,7 @@ impl Mapper { } if let Some(p2_frame) = p3[page.p3_index()].pointed_frame() { - //println!("Free p2 {:?}", p2_frame); + //println!("unmap_inner: Free p2 {:?}", p2_frame); p3.decrement_entry_count(); p3[page.p3_index()].set_unused(); deallocate_frames(p2_frame, 1); @@ -123,7 +123,7 @@ impl Mapper { } if let Some(p3_frame) = p4[page.p4_index()].pointed_frame() { - //println!("Free p3 {:?}", p3_frame); + //println!("unmap_inner: Free p3 {:?}", p3_frame); p4.decrement_entry_count(); p4[page.p4_index()].set_unused(); deallocate_frames(p3_frame, 1); diff --git a/src/context/arch/aarch64.rs b/src/context/arch/aarch64.rs index 2b850a7..39f45c0 100644 --- a/src/context/arch/aarch64.rs +++ b/src/context/arch/aarch64.rs @@ -220,163 +220,163 @@ impl Context { println!("x9: 0x{:016x}", self.x9); println!("x8: 0x{:016x}", self.x8); } +} - #[cold] - #[inline(never)] - #[naked] - pub unsafe fn switch_to(&mut self, next: &mut Context) { - let mut float_regs = &mut *(self.fx_address as *mut FloatRegisters); +#[cold] +#[inline(never)] +#[naked] +pub unsafe fn switch_to(prev: &mut Context, next: &mut Context) { + let mut float_regs = &mut *(prev.fx_address as *mut FloatRegisters); + asm!( + "stp q0, q1, [{0}, #16 * 0]", + "stp q2, q3, [{0}, #16 * 2]", + "stp q4, q5, [{0}, #16 * 4]", + "stp q6, q7, [{0}, #16 * 6]", + "stp q8, q9, [{0}, #16 * 8]", + "stp q10, q11, [{0}, #16 * 10]", + "stp q12, q13, [{0}, #16 * 12]", + "stp q14, q15, [{0}, #16 * 14]", + "stp q16, q17, [{0}, #16 * 16]", + "stp q18, q19, [{0}, #16 * 18]", + "stp q20, q21, [{0}, #16 * 20]", + "stp q22, q23, [{0}, #16 * 22]", + "stp q24, q25, [{0}, #16 * 24]", + "stp q26, q27, [{0}, #16 * 26]", + "stp q28, q29, [{0}, #16 * 28]", + "stp q30, q31, [{0}, #16 * 30]", + "mrs {1}, fpcr", + "mrs {2}, fpsr", + in(reg) &mut float_regs.fp_simd_regs, + out(reg) float_regs.fpcr, + out(reg) float_regs.fpsr + ); + + prev.fx_loadable = true; + + if next.fx_loadable { + let mut float_regs = &mut *(next.fx_address as *mut FloatRegisters); asm!( - "stp q0, q1, [{0}, #16 * 0]", - "stp q2, q3, [{0}, #16 * 2]", - "stp q4, q5, [{0}, #16 * 4]", - "stp q6, q7, [{0}, #16 * 6]", - "stp q8, q9, [{0}, #16 * 8]", - "stp q10, q11, [{0}, #16 * 10]", - "stp q12, q13, [{0}, #16 * 12]", - "stp q14, q15, [{0}, #16 * 14]", - "stp q16, q17, [{0}, #16 * 16]", - "stp q18, q19, [{0}, #16 * 18]", - "stp q20, q21, [{0}, #16 * 20]", - "stp q22, q23, [{0}, #16 * 22]", - "stp q24, q25, [{0}, #16 * 24]", - "stp q26, q27, [{0}, #16 * 26]", - "stp q28, q29, [{0}, #16 * 28]", - "stp q30, q31, [{0}, #16 * 30]", - "mrs {1}, fpcr", - "mrs {2}, fpsr", + "ldp q0, q1, [{0}, #16 * 0]", + "ldp q2, q3, [{0}, #16 * 2]", + "ldp q4, q5, [{0}, #16 * 4]", + "ldp q6, q7, [{0}, #16 * 6]", + "ldp q8, q9, [{0}, #16 * 8]", + "ldp q10, q11, [{0}, #16 * 10]", + "ldp q12, q13, [{0}, #16 * 12]", + "ldp q14, q15, [{0}, #16 * 14]", + "ldp q16, q17, [{0}, #16 * 16]", + "ldp q18, q19, [{0}, #16 * 18]", + "ldp q20, q21, [{0}, #16 * 20]", + "ldp q22, q23, [{0}, #16 * 22]", + "ldp q24, q25, [{0}, #16 * 24]", + "ldp q26, q27, [{0}, #16 * 26]", + "ldp q28, q29, [{0}, #16 * 28]", + "ldp q30, q31, [{0}, #16 * 30]", + "msr fpcr, {1}", + "msr fpsr, {2}", in(reg) &mut float_regs.fp_simd_regs, - out(reg) float_regs.fpcr, - out(reg) float_regs.fpsr + in(reg) float_regs.fpcr, + in(reg) float_regs.fpsr ); - - self.fx_loadable = true; - - if next.fx_loadable { - let mut float_regs = &mut *(next.fx_address as *mut FloatRegisters); - asm!( - "ldp q0, q1, [{0}, #16 * 0]", - "ldp q2, q3, [{0}, #16 * 2]", - "ldp q4, q5, [{0}, #16 * 4]", - "ldp q6, q7, [{0}, #16 * 6]", - "ldp q8, q9, [{0}, #16 * 8]", - "ldp q10, q11, [{0}, #16 * 10]", - "ldp q12, q13, [{0}, #16 * 12]", - "ldp q14, q15, [{0}, #16 * 14]", - "ldp q16, q17, [{0}, #16 * 16]", - "ldp q18, q19, [{0}, #16 * 18]", - "ldp q20, q21, [{0}, #16 * 20]", - "ldp q22, q23, [{0}, #16 * 22]", - "ldp q24, q25, [{0}, #16 * 24]", - "ldp q26, q27, [{0}, #16 * 26]", - "ldp q28, q29, [{0}, #16 * 28]", - "ldp q30, q31, [{0}, #16 * 30]", - "msr fpcr, {1}", - "msr fpsr, {2}", - in(reg) &mut float_regs.fp_simd_regs, - in(reg) float_regs.fpcr, - in(reg) float_regs.fpsr - ); - } - - self.ttbr0_el1 = control_regs::ttbr0_el1() as usize; - if next.ttbr0_el1 != self.ttbr0_el1 { - control_regs::ttbr0_el1_write(next.ttbr0_el1 as u64); - tlb::flush_all(); - } - - llvm_asm!("mov $0, x8" : "=r"(self.x8) : : "memory" : "volatile"); - llvm_asm!("mov x8, $0" : : "r"(next.x8) :"memory" : "volatile"); - - llvm_asm!("mov $0, x9" : "=r"(self.x9) : : "memory" : "volatile"); - llvm_asm!("mov x9, $0" : : "r"(next.x9) :"memory" : "volatile"); - - llvm_asm!("mov $0, x10" : "=r"(self.x10) : : "memory" : "volatile"); - llvm_asm!("mov x10, $0" : : "r"(next.x10) :"memory" : "volatile"); - - llvm_asm!("mov $0, x11" : "=r"(self.x11) : : "memory" : "volatile"); - llvm_asm!("mov x11, $0" : : "r"(next.x11) :"memory" : "volatile"); - - llvm_asm!("mov $0, x12" : "=r"(self.x12) : : "memory" : "volatile"); - llvm_asm!("mov x12, $0" : : "r"(next.x12) :"memory" : "volatile"); - - llvm_asm!("mov $0, x13" : "=r"(self.x13) : : "memory" : "volatile"); - llvm_asm!("mov x13, $0" : : "r"(next.x13) :"memory" : "volatile"); - - llvm_asm!("mov $0, x14" : "=r"(self.x14) : : "memory" : "volatile"); - llvm_asm!("mov x14, $0" : : "r"(next.x14) :"memory" : "volatile"); - - llvm_asm!("mov $0, x15" : "=r"(self.x15) : : "memory" : "volatile"); - llvm_asm!("mov x15, $0" : : "r"(next.x15) :"memory" : "volatile"); - - llvm_asm!("mov $0, x16" : "=r"(self.x16) : : "memory" : "volatile"); - llvm_asm!("mov x16, $0" : : "r"(next.x16) :"memory" : "volatile"); - - llvm_asm!("mov $0, x17" : "=r"(self.x17) : : "memory" : "volatile"); - llvm_asm!("mov x17, $0" : : "r"(next.x17) :"memory" : "volatile"); - - llvm_asm!("mov $0, x18" : "=r"(self.x18) : : "memory" : "volatile"); - llvm_asm!("mov x18, $0" : : "r"(next.x18) :"memory" : "volatile"); - - llvm_asm!("mov $0, x19" : "=r"(self.x19) : : "memory" : "volatile"); - llvm_asm!("mov x19, $0" : : "r"(next.x19) :"memory" : "volatile"); - - llvm_asm!("mov $0, x20" : "=r"(self.x20) : : "memory" : "volatile"); - llvm_asm!("mov x20, $0" : : "r"(next.x20) :"memory" : "volatile"); - - llvm_asm!("mov $0, x21" : "=r"(self.x21) : : "memory" : "volatile"); - llvm_asm!("mov x21, $0" : : "r"(next.x21) :"memory" : "volatile"); - - llvm_asm!("mov $0, x22" : "=r"(self.x22) : : "memory" : "volatile"); - llvm_asm!("mov x22, $0" : : "r"(next.x22) :"memory" : "volatile"); - - llvm_asm!("mov $0, x23" : "=r"(self.x23) : : "memory" : "volatile"); - llvm_asm!("mov x23, $0" : : "r"(next.x23) :"memory" : "volatile"); - - llvm_asm!("mov $0, x24" : "=r"(self.x24) : : "memory" : "volatile"); - llvm_asm!("mov x24, $0" : : "r"(next.x24) :"memory" : "volatile"); - - llvm_asm!("mov $0, x25" : "=r"(self.x25) : : "memory" : "volatile"); - llvm_asm!("mov x25, $0" : : "r"(next.x25) :"memory" : "volatile"); - - llvm_asm!("mov $0, x26" : "=r"(self.x26) : : "memory" : "volatile"); - llvm_asm!("mov x26, $0" : : "r"(next.x26) :"memory" : "volatile"); - - llvm_asm!("mov $0, x27" : "=r"(self.x27) : : "memory" : "volatile"); - llvm_asm!("mov x27, $0" : : "r"(next.x27) :"memory" : "volatile"); - - llvm_asm!("mov $0, x28" : "=r"(self.x28) : : "memory" : "volatile"); - llvm_asm!("mov x28, $0" : : "r"(next.x28) :"memory" : "volatile"); - - llvm_asm!("mov $0, x29" : "=r"(self.fp) : : "memory" : "volatile"); - llvm_asm!("mov x29, $0" : : "r"(next.fp) :"memory" : "volatile"); - - llvm_asm!("mov $0, x30" : "=r"(self.lr) : : "memory" : "volatile"); - llvm_asm!("mov x30, $0" : : "r"(next.lr) :"memory" : "volatile"); - - llvm_asm!("mrs $0, elr_el1" : "=r"(self.elr_el1) : : "memory" : "volatile"); - llvm_asm!("msr elr_el1, $0" : : "r"(next.elr_el1) : "memory" : "volatile"); - - llvm_asm!("mrs $0, sp_el0" : "=r"(self.sp_el0) : : "memory" : "volatile"); - llvm_asm!("msr sp_el0, $0" : : "r"(next.sp_el0) : "memory" : "volatile"); - - llvm_asm!("mrs $0, tpidr_el0" : "=r"(self.tpidr_el0) : : "memory" : "volatile"); - llvm_asm!("msr tpidr_el0, $0" : : "r"(next.tpidr_el0) : "memory" : "volatile"); - - llvm_asm!("mrs $0, tpidrro_el0" : "=r"(self.tpidrro_el0) : : "memory" : "volatile"); - llvm_asm!("msr tpidrro_el0, $0" : : "r"(next.tpidrro_el0) : "memory" : "volatile"); - - llvm_asm!("mrs $0, spsr_el1" : "=r"(self.spsr_el1) : : "memory" : "volatile"); - llvm_asm!("msr spsr_el1, $0" : : "r"(next.spsr_el1) : "memory" : "volatile"); - - llvm_asm!("mrs $0, esr_el1" : "=r"(self.esr_el1) : : "memory" : "volatile"); - llvm_asm!("msr esr_el1, $0" : : "r"(next.esr_el1) : "memory" : "volatile"); - - llvm_asm!("mov $0, sp" : "=r"(self.sp) : : "memory" : "volatile"); - llvm_asm!("mov sp, $0" : : "r"(next.sp) : "memory" : "volatile"); - - CONTEXT_SWITCH_LOCK.store(false, Ordering::SeqCst); } + + prev.ttbr0_el1 = control_regs::ttbr0_el1() as usize; + if next.ttbr0_el1 != prev.ttbr0_el1 { + control_regs::ttbr0_el1_write(next.ttbr0_el1 as u64); + tlb::flush_all(); + } + + llvm_asm!("mov $0, x8" : "=r"(prev.x8) : : "memory" : "volatile"); + llvm_asm!("mov x8, $0" : : "r"(next.x8) :"memory" : "volatile"); + + llvm_asm!("mov $0, x9" : "=r"(prev.x9) : : "memory" : "volatile"); + llvm_asm!("mov x9, $0" : : "r"(next.x9) :"memory" : "volatile"); + + llvm_asm!("mov $0, x10" : "=r"(prev.x10) : : "memory" : "volatile"); + llvm_asm!("mov x10, $0" : : "r"(next.x10) :"memory" : "volatile"); + + llvm_asm!("mov $0, x11" : "=r"(prev.x11) : : "memory" : "volatile"); + llvm_asm!("mov x11, $0" : : "r"(next.x11) :"memory" : "volatile"); + + llvm_asm!("mov $0, x12" : "=r"(prev.x12) : : "memory" : "volatile"); + llvm_asm!("mov x12, $0" : : "r"(next.x12) :"memory" : "volatile"); + + llvm_asm!("mov $0, x13" : "=r"(prev.x13) : : "memory" : "volatile"); + llvm_asm!("mov x13, $0" : : "r"(next.x13) :"memory" : "volatile"); + + llvm_asm!("mov $0, x14" : "=r"(prev.x14) : : "memory" : "volatile"); + llvm_asm!("mov x14, $0" : : "r"(next.x14) :"memory" : "volatile"); + + llvm_asm!("mov $0, x15" : "=r"(prev.x15) : : "memory" : "volatile"); + llvm_asm!("mov x15, $0" : : "r"(next.x15) :"memory" : "volatile"); + + llvm_asm!("mov $0, x16" : "=r"(prev.x16) : : "memory" : "volatile"); + llvm_asm!("mov x16, $0" : : "r"(next.x16) :"memory" : "volatile"); + + llvm_asm!("mov $0, x17" : "=r"(prev.x17) : : "memory" : "volatile"); + llvm_asm!("mov x17, $0" : : "r"(next.x17) :"memory" : "volatile"); + + llvm_asm!("mov $0, x18" : "=r"(prev.x18) : : "memory" : "volatile"); + llvm_asm!("mov x18, $0" : : "r"(next.x18) :"memory" : "volatile"); + + llvm_asm!("mov $0, x19" : "=r"(prev.x19) : : "memory" : "volatile"); + llvm_asm!("mov x19, $0" : : "r"(next.x19) :"memory" : "volatile"); + + llvm_asm!("mov $0, x20" : "=r"(prev.x20) : : "memory" : "volatile"); + llvm_asm!("mov x20, $0" : : "r"(next.x20) :"memory" : "volatile"); + + llvm_asm!("mov $0, x21" : "=r"(prev.x21) : : "memory" : "volatile"); + llvm_asm!("mov x21, $0" : : "r"(next.x21) :"memory" : "volatile"); + + llvm_asm!("mov $0, x22" : "=r"(prev.x22) : : "memory" : "volatile"); + llvm_asm!("mov x22, $0" : : "r"(next.x22) :"memory" : "volatile"); + + llvm_asm!("mov $0, x23" : "=r"(prev.x23) : : "memory" : "volatile"); + llvm_asm!("mov x23, $0" : : "r"(next.x23) :"memory" : "volatile"); + + llvm_asm!("mov $0, x24" : "=r"(prev.x24) : : "memory" : "volatile"); + llvm_asm!("mov x24, $0" : : "r"(next.x24) :"memory" : "volatile"); + + llvm_asm!("mov $0, x25" : "=r"(prev.x25) : : "memory" : "volatile"); + llvm_asm!("mov x25, $0" : : "r"(next.x25) :"memory" : "volatile"); + + llvm_asm!("mov $0, x26" : "=r"(prev.x26) : : "memory" : "volatile"); + llvm_asm!("mov x26, $0" : : "r"(next.x26) :"memory" : "volatile"); + + llvm_asm!("mov $0, x27" : "=r"(prev.x27) : : "memory" : "volatile"); + llvm_asm!("mov x27, $0" : : "r"(next.x27) :"memory" : "volatile"); + + llvm_asm!("mov $0, x28" : "=r"(prev.x28) : : "memory" : "volatile"); + llvm_asm!("mov x28, $0" : : "r"(next.x28) :"memory" : "volatile"); + + llvm_asm!("mov $0, x29" : "=r"(prev.fp) : : "memory" : "volatile"); + llvm_asm!("mov x29, $0" : : "r"(next.fp) :"memory" : "volatile"); + + llvm_asm!("mov $0, x30" : "=r"(prev.lr) : : "memory" : "volatile"); + llvm_asm!("mov x30, $0" : : "r"(next.lr) :"memory" : "volatile"); + + llvm_asm!("mrs $0, elr_el1" : "=r"(prev.elr_el1) : : "memory" : "volatile"); + llvm_asm!("msr elr_el1, $0" : : "r"(next.elr_el1) : "memory" : "volatile"); + + llvm_asm!("mrs $0, sp_el0" : "=r"(prev.sp_el0) : : "memory" : "volatile"); + llvm_asm!("msr sp_el0, $0" : : "r"(next.sp_el0) : "memory" : "volatile"); + + llvm_asm!("mrs $0, tpidr_el0" : "=r"(prev.tpidr_el0) : : "memory" : "volatile"); + llvm_asm!("msr tpidr_el0, $0" : : "r"(next.tpidr_el0) : "memory" : "volatile"); + + llvm_asm!("mrs $0, tpidrro_el0" : "=r"(prev.tpidrro_el0) : : "memory" : "volatile"); + llvm_asm!("msr tpidrro_el0, $0" : : "r"(next.tpidrro_el0) : "memory" : "volatile"); + + llvm_asm!("mrs $0, spsr_el1" : "=r"(prev.spsr_el1) : : "memory" : "volatile"); + llvm_asm!("msr spsr_el1, $0" : : "r"(next.spsr_el1) : "memory" : "volatile"); + + llvm_asm!("mrs $0, esr_el1" : "=r"(prev.esr_el1) : : "memory" : "volatile"); + llvm_asm!("msr esr_el1, $0" : : "r"(next.esr_el1) : "memory" : "volatile"); + + llvm_asm!("mov $0, sp" : "=r"(prev.sp) : : "memory" : "volatile"); + llvm_asm!("mov sp, $0" : : "r"(next.sp) : "memory" : "volatile"); + + CONTEXT_SWITCH_LOCK.store(false, Ordering::SeqCst); } #[allow(dead_code)]