Jika/redox-kernel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixes for building aarch64

Browse Source
This commit is contained in:
Jeremy Soller
2021-05-03 21:57:45 -06:00
parent c646cb76e5
commit 8f50785781
10 changed files with 243 additions and 242 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/arch/aarch64/consts.rs
View File

@@ -48,6 +48,11 @@
/// Size of kernel percpu variables
pub const KERNEL_PERCPU_SIZE: usize = 64 * 1024; // 64 KB
/// Offset of physmap
// This needs to match RMM's PHYS_OFFSET
pub const PHYS_OFFSET: usize = 0xFFFF_FE00_0000_0000;
pub const PHYS_PML4: usize = (PHYS_OFFSET & PML4_MASK)/PML4_SIZE;
/// Offset to user image
pub const USER_OFFSET: usize = 0;
pub const USER_PML4: usize = (USER_OFFSET & PML4_MASK)/PML4_SIZE;

13
src/arch/aarch64/device/gic.rs
View File

@@ -1,8 +1,7 @@
use core::intrinsics::{volatile_load, volatile_store};
use crate::memory::Frame;
use crate::paging::{ActivePageTable, PhysicalAddress, Page, PageTableType, VirtualAddress};
use crate::paging::entry::EntryFlags;
use crate::paging::{ActivePageTable, PhysicalAddress, Page, PageFlags, TableKind, VirtualAddress};
static GICD_CTLR: u32 = 0x000;
static GICD_TYPER: u32 = 0x004;
@@ -57,14 +56,14 @@ pub struct GicDistIf {
impl GicDistIf {
unsafe fn init(&mut self) {
// Map in the Distributor interface
let mut active_table = ActivePageTable::new(PageTableType::Kernel);
let mut active_table = ActivePageTable::new(TableKind::Kernel);
let start_frame = Frame::containing_address(PhysicalAddress::new(0x08000000));
let end_frame = Frame::containing_address(PhysicalAddress::new(0x08000000 + 0x10000 - 1));
for frame in Frame::range_inclusive(start_frame, end_frame) {
let page = Page::containing_address(VirtualAddress::new(frame.start_address().data() + crate::KERNEL_DEVMAP_OFFSET));
let result = active_table.map_to(page, frame, EntryFlags::PRESENT | EntryFlags::WRITABLE | EntryFlags::NO_EXECUTE);
result.flush(&mut active_table);
let result = active_table.map_to(page, frame, PageFlags::new().write(true));
result.flush();
}
self.address = crate::KERNEL_DEVMAP_OFFSET + 0x08000000;
@@ -74,8 +73,8 @@ impl GicDistIf {
let end_frame = Frame::containing_address(PhysicalAddress::new(0x08010000 + 0x10000 - 1));
for frame in Frame::range_inclusive(start_frame, end_frame) {
let page = Page::containing_address(VirtualAddress::new(frame.start_address().data() + crate::KERNEL_DEVMAP_OFFSET));
let result = active_table.map_to(page, frame, EntryFlags::PRESENT | EntryFlags::WRITABLE | EntryFlags::NO_EXECUTE);
result.flush(&mut active_table);
let result = active_table.map_to(page, frame, PageFlags::new().write(true));
result.flush();
}
GIC_CPU_IF.address = crate::KERNEL_DEVMAP_OFFSET + 0x08010000;

9
src/arch/aarch64/device/rtc.rs
View File

@@ -1,8 +1,7 @@
use core::intrinsics::{volatile_load, volatile_store};
use crate::memory::Frame;
use crate::paging::{ActivePageTable, PhysicalAddress, Page, PageTableType, VirtualAddress};
use crate::paging::entry::EntryFlags;
use crate::paging::{ActivePageTable, PhysicalAddress, Page, PageFlags, TableKind, VirtualAddress};
use crate::time;
static RTC_DR: u32 = 0x000;
@@ -29,15 +28,15 @@ struct Pl031rtc {
impl Pl031rtc {
unsafe fn init(&mut self) {
let mut active_table = ActivePageTable::new(PageTableType::Kernel);
let mut active_table = ActivePageTable::new(TableKind::Kernel);
let start_frame = Frame::containing_address(PhysicalAddress::new(0x09010000));
let end_frame = Frame::containing_address(PhysicalAddress::new(0x09010000 + 0x1000 - 1));
for frame in Frame::range_inclusive(start_frame, end_frame) {
let page = Page::containing_address(VirtualAddress::new(frame.start_address().data() + crate::KERNEL_DEVMAP_OFFSET));
let result = active_table.map_to(page, frame, EntryFlags::PRESENT | EntryFlags::WRITABLE | EntryFlags::NO_EXECUTE);
result.flush(&mut active_table);
let result = active_table.map_to(page, frame, PageFlags::new().write(true));
result.flush();
}
self.address = crate::KERNEL_DEVMAP_OFFSET + 0x09010000;

13
src/arch/aarch64/device/serial.rs
View File

@@ -4,9 +4,8 @@ use spin::Mutex;
use crate::device::uart_pl011::SerialPort;
use crate::init::device_tree;
use crate::memory::Frame;
use crate::paging::mapper::{MapperFlushAll, MapperType};
use crate::paging::{ActivePageTable, Page, PageTableType, PhysicalAddress, VirtualAddress};
use crate::paging::entry::EntryFlags;
use crate::paging::mapper::PageFlushAll;
use crate::paging::{ActivePageTable, Page, PageFlags, PhysicalAddress, TableKind, VirtualAddress};
pub static COM1: Mutex<Option<SerialPort>> = Mutex::new(None);
@@ -16,17 +15,17 @@ pub unsafe fn init() {
}
let (base, size) = device_tree::diag_uart_range(crate::KERNEL_DTB_OFFSET, crate::KERNEL_DTB_MAX_SIZE).unwrap();
let mut active_ktable = unsafe { ActivePageTable::new(PageTableType::Kernel) };
let mut flush_all = MapperFlushAll::new();
let mut active_ktable = unsafe { ActivePageTable::new(TableKind::Kernel) };
let mut flush_all = PageFlushAll::new();
let start_frame = Frame::containing_address(PhysicalAddress::new(base));
let end_frame = Frame::containing_address(PhysicalAddress::new(base + size - 1));
for frame in Frame::range_inclusive(start_frame, end_frame) {
let page = Page::containing_address(VirtualAddress::new(frame.start_address().data() + crate::KERNEL_DEVMAP_OFFSET));
let result = active_ktable.map_to(page, frame, EntryFlags::PRESENT | EntryFlags::NO_EXECUTE | EntryFlags::WRITABLE);
let result = active_ktable.map_to(page, frame, PageFlags::new().write(true));
flush_all.consume(result);
};
flush_all.flush(&mut active_ktable);
flush_all.flush();
let start_frame = Frame::containing_address(PhysicalAddress::new(base));
let vaddr = start_frame.start_address().data() + crate::KERNEL_DEVMAP_OFFSET;

97
src/arch/aarch64/paging/mod.rs
View File

@@ -8,11 +8,17 @@ use spin::Mutex;
use crate::device::cpu::registers::{control_regs, tlb};
use crate::memory::{allocate_frames, Frame};
use self::entry::{EntryFlags, TableDescriptorFlags};
use self::mapper::{Mapper, MapperFlushAll};
use self::mapper::{Mapper, PageFlushAll};
use self::temporary_page::TemporaryPage;
pub use rmm::{PhysicalAddress, TableKind, VirtualAddress};
pub use rmm::{
AArch64Arch as RmmA,
Arch as RmmArch,
PageFlags,
PhysicalAddress,
TableKind,
VirtualAddress,
};
pub mod entry;
pub mod mapper;
@@ -68,7 +74,7 @@ unsafe fn init_mair() {
}
/// Map TSS
unsafe fn map_tss(cpu_id: usize, mapper: &mut Mapper) -> MapperFlushAll {
unsafe fn map_tss(cpu_id: usize, mapper: &mut Mapper) -> PageFlushAll<RmmA> {
extern "C" {
/// The starting byte of the thread data segment
static mut __tdata_start: u8;
@@ -84,17 +90,11 @@ unsafe fn map_tss(cpu_id: usize, mapper: &mut Mapper) -> MapperFlushAll {
let start = crate::KERNEL_PERCPU_OFFSET + crate::KERNEL_PERCPU_SIZE * cpu_id;
let end = start + size;
let mut flush_all = MapperFlushAll::new();
let flush_all = PageFlushAll::new();
let start_page = Page::containing_address(VirtualAddress::new(start));
let end_page = Page::containing_address(VirtualAddress::new(end - 1));
for page in Page::range_inclusive(start_page, end_page) {
let result = mapper.map(
page,
EntryFlags::PRESENT
| EntryFlags::GLOBAL
| EntryFlags::NO_EXECUTE
| EntryFlags::WRITABLE,
);
let result = mapper.map(page, PageFlags::new().write(true));
flush_all.consume(result);
}
flush_all
@@ -171,10 +171,10 @@ pub unsafe fn init(
init_mair();
let mut active_table = ActivePageTable::new_unlocked(PageTableType::Kernel);
let mut active_table = ActivePageTable::new_unlocked(TableKind::Kernel);
let flush_all = map_tss(cpu_id, &mut active_table);
flush_all.flush(&mut active_table);
flush_all.flush();
return (active_table, init_tcb(cpu_id));
}
@@ -185,7 +185,7 @@ pub unsafe fn init_ap(
) -> usize {
init_mair();
let mut active_table = ActivePageTable::new_unlocked(PageTableType::Kernel);
let mut active_table = ActivePageTable::new_unlocked(TableKind::Kernel);
let mut new_table = InactivePageTable::from_address(bsp_table);
@@ -206,6 +206,7 @@ pub unsafe fn init_ap(
init_tcb(cpu_id)
}
#[derive(Debug)]
pub struct ActivePageTable {
mapper: Mapper,
locked: bool,
@@ -226,7 +227,6 @@ impl DerefMut for ActivePageTable {
}
impl ActivePageTable {
//TODO: table_type argument
pub unsafe fn new(table_kind: TableKind) -> ActivePageTable {
page_table_lock();
ActivePageTable {
@@ -235,7 +235,6 @@ impl ActivePageTable {
}
}
//TODO: table_type argument
pub unsafe fn new_unlocked(table_kind: TableKind) -> ActivePageTable {
ActivePageTable {
mapper: Mapper::new(table_kind),
@@ -246,14 +245,14 @@ impl ActivePageTable {
pub fn switch(&mut self, new_table: InactivePageTable) -> InactivePageTable {
let old_table: InactivePageTable;
match self.mapper.mapper_type {
MapperType::User => {
old_table = InactivePageTable { p4_frame: Frame::containing_address(PhysicalAddress::new(unsafe { control_regs::ttbr0_el1() } as usize)) };
unsafe { control_regs::ttbr0_el1_write(new_table.p4_frame.start_address().data() as u64) };
match self.mapper.table_kind {
TableKind::User => {
old_table = InactivePageTable { frame: Frame::containing_address(PhysicalAddress::new(unsafe { control_regs::ttbr0_el1() } as usize)) };
unsafe { control_regs::ttbr0_el1_write(new_table.frame.start_address().data() as u64) };
},
MapperType::Kernel => {
old_table = InactivePageTable { p4_frame: Frame::containing_address(PhysicalAddress::new(unsafe { control_regs::ttbr1_el1() } as usize)) };
unsafe { control_regs::ttbr1_el1_write(new_table.p4_frame.start_address().data() as u64) };
TableKind::Kernel => {
old_table = InactivePageTable { frame: Frame::containing_address(PhysicalAddress::new(unsafe { control_regs::ttbr1_el1() } as usize)) };
unsafe { control_regs::ttbr1_el1_write(new_table.frame.start_address().data() as u64) };
}
}
@@ -277,20 +276,24 @@ impl ActivePageTable {
where F: FnOnce(&mut Mapper)
{
{
let backup: Frame;
let backup = Frame::containing_address(PhysicalAddress::new(unsafe {
match self.mapper.table_kind {
TableKind::User => control_regs::ttbr0_el1() as usize,
TableKind::Kernel => control_regs::ttbr1_el1() as usize,
}
}));
match self.mapper.mapper_type {
MapperType::User => backup = Frame::containing_address(PhysicalAddress::new(unsafe { control_regs::ttbr0_el1() as usize })),
MapperType::Kernel => backup = Frame::containing_address(PhysicalAddress::new(unsafe { control_regs::ttbr1_el1() as usize }))
}
// map temporary_kpage to current p4 table
let p4_table = temporary_page.map_table_frame(backup.clone(), EntryFlags::PRESENT | EntryFlags::WRITABLE | EntryFlags::NO_EXECUTE, self);
// map temporary_page to current p4 table
let p4_table = temporary_page.map_table_frame(
backup.clone(),
PageFlags::new_table().write(true), //TODO: RISC-V will not like this
self,
);
// overwrite recursive mapping
self.p4_mut()[crate::RECURSIVE_PAGE_PML4].page_table_entry_set(
table.p4_frame.clone(),
TableDescriptorFlags::VALID | TableDescriptorFlags::TABLE,
self.p4_mut()[crate::RECURSIVE_PAGE_PML4].set(
table.frame.clone(),
PageFlags::new_table().write(true), //TODO: RISC-V will not like this
);
self.flush_all();
@@ -298,9 +301,9 @@ impl ActivePageTable {
f(self);
// restore recursive mapping to original p4 table
p4_table[crate::RECURSIVE_PAGE_PML4].page_table_entry_set(
p4_table[crate::RECURSIVE_PAGE_PML4].set(
backup,
TableDescriptorFlags::VALID | TableDescriptorFlags::TABLE,
PageFlags::new_table().write(true), //TODO: RISC-V will not like this
);
self.flush_all();
}
@@ -309,9 +312,9 @@ impl ActivePageTable {
}
pub unsafe fn address(&self) -> usize {
match self.mapper.mapper_type {
MapperType::User => control_regs::ttbr0_el1() as usize,
MapperType::Kernel => control_regs::ttbr1_el1() as usize,
match self.mapper.table_kind {
TableKind::User => control_regs::ttbr0_el1() as usize,
TableKind::Kernel => control_regs::ttbr1_el1() as usize,
}
}
}
@@ -326,7 +329,7 @@ impl Drop for ActivePageTable {
}
pub struct InactivePageTable {
p4_frame: Frame,
frame: Frame,
}
impl InactivePageTable {
@@ -338,30 +341,30 @@ impl InactivePageTable {
{
let table = temporary_page.map_table_frame(
frame.clone(),
EntryFlags::PRESENT | EntryFlags::WRITABLE | EntryFlags::NO_EXECUTE,
PageFlags::new_table().write(true), //TODO: RISC-V will not like this
active_table,
);
// now we are able to zero the table
table.zero();
// set up recursive mapping for the table
table[crate::RECURSIVE_PAGE_PML4].page_table_entry_set(
table[crate::RECURSIVE_PAGE_PML4].set(
frame.clone(),
TableDescriptorFlags::VALID | TableDescriptorFlags::TABLE
PageFlags::new_table().write(true), //TODO: RISC-V will not like this
);
}
temporary_page.unmap(active_table);
InactivePageTable { p4_frame: frame }
InactivePageTable { frame: frame }
}
pub unsafe fn from_address(address: usize) -> InactivePageTable {
InactivePageTable {
p4_frame: Frame::containing_address(PhysicalAddress::new(address)),
frame: Frame::containing_address(PhysicalAddress::new(address)),
}
}
pub unsafe fn address(&self) -> usize {
self.p4_frame.start_address().data()
self.frame.start_address().data()
}
}

15
src/arch/aarch64/paging/temporary_page.rs
View File

@@ -3,8 +3,7 @@
use crate::memory::Frame;
use super::{ActivePageTable, Page, VirtualAddress};
use super::entry::EntryFlags;
use super::{ActivePageTable, Page, PageFlags, RmmA, VirtualAddress};
use super::table::{Table, Level1};
pub struct TemporaryPage {
@@ -13,9 +12,7 @@ pub struct TemporaryPage {
impl TemporaryPage {
pub fn new(page: Page) -> TemporaryPage {
TemporaryPage {
page: page,
}
TemporaryPage { page }
}
pub fn start_address (&self) -> VirtualAddress {
@@ -24,22 +21,22 @@ impl TemporaryPage {
/// Maps the temporary page to the given frame in the active table.
/// Returns the start address of the temporary page.
pub fn map(&mut self, frame: Frame, flags: EntryFlags, active_table: &mut ActivePageTable) -> VirtualAddress {
pub fn map(&mut self, frame: Frame, flags: PageFlags<RmmA>, active_table: &mut ActivePageTable) -> VirtualAddress {
assert!(active_table.translate_page(self.page).is_none(), "temporary page is already mapped");
let result = active_table.map_to(self.page, frame, flags);
result.flush(active_table);
result.flush();
self.page.start_address()
}
/// Maps the temporary page to the given page table frame in the active
/// table. Returns a reference to the now mapped table.
pub fn map_table_frame(&mut self, frame: Frame, flags: EntryFlags, active_table: &mut ActivePageTable) -> &mut Table<Level1> {
pub fn map_table_frame(&mut self, frame: Frame, flags: PageFlags<RmmA>, active_table: &mut ActivePageTable) -> &mut Table<Level1> {
unsafe { &mut *(self.map(frame, flags, active_table).data() as *mut Table<Level1>) }
}
/// Unmaps the temporary page in the active table.
pub fn unmap(&mut self, active_table: &mut ActivePageTable) {
let (result, _frame) = active_table.unmap_return(self.page, true);
result.flush(active_table);
result.flush();
}
}

18
src/arch/aarch64/rmm.rs
View File

@@ -1,7 +1,7 @@
use rmm::{
KILOBYTE,
MEGABYTE,
AArch64Arch,
AArch64Arch as RmmA,
Arch,
BuddyAllocator,
BumpAllocator,
@@ -42,13 +42,13 @@ unsafe fn page_flags<A: Arch>(virt: VirtualAddress) -> PageFlags<A> {
if in_section!(text) {
// Remap text read-only, execute
PageFlags::new().write(false).execute(true)
PageFlags::new().execute(true)
} else if in_section!(rodata) {
// Remap rodata read-only, no execute
PageFlags::new().write(false).execute(false)
PageFlags::new()
} else {
// Remap everything else read-write, no execute
PageFlags::new().write(true).execute(false)
PageFlags::new().write(true)
}
}
@@ -181,7 +181,7 @@ unsafe fn inner<A: Arch>(areas: &'static [MemoryArea], kernel_base: usize, kerne
}
pub struct LockedAllocator {
inner: Mutex<Option<BuddyAllocator<AArch64Arch>>>,
inner: Mutex<Option<BuddyAllocator<RmmA>>>,
}
impl LockedAllocator {
@@ -223,21 +223,21 @@ static mut AREAS: [MemoryArea; 512] = [MemoryArea {
pub static mut FRAME_ALLOCATOR: LockedAllocator = LockedAllocator::new();
pub unsafe fn mapper_new(table_addr: PhysicalAddress) -> PageMapper<'static, AArch64Arch, LockedAllocator> {
pub unsafe fn mapper_new(table_addr: PhysicalAddress) -> PageMapper<'static, RmmA, LockedAllocator> {
PageMapper::new(table_addr, &mut FRAME_ALLOCATOR)
}
//TODO: global paging lock?
pub unsafe fn mapper_create() -> Option<PageMapper<'static, AArch64Arch, LockedAllocator>> {
pub unsafe fn mapper_create() -> Option<PageMapper<'static, RmmA, LockedAllocator>> {
PageMapper::create(&mut FRAME_ALLOCATOR)
}
pub unsafe fn mapper_current() -> PageMapper<'static, AArch64Arch, LockedAllocator> {
pub unsafe fn mapper_current() -> PageMapper<'static, RmmA, LockedAllocator> {
PageMapper::current(&mut FRAME_ALLOCATOR)
}
pub unsafe fn init(kernel_base: usize, kernel_size: usize) {
type A = AArch64Arch;
type A = RmmA;
let kernel_size_aligned = ((kernel_size + (A::PAGE_SIZE - 1))/A::PAGE_SIZE) * A::PAGE_SIZE;
let kernel_end = kernel_base + kernel_size_aligned;

5
src/arch/aarch64/start.rs
View File

@@ -7,8 +7,7 @@ use core::slice;
use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use crate::memory::{Frame};
use crate::paging::{ActivePageTable, PageTableType, Page, PAGE_SIZE, PhysicalAddress, VirtualAddress};
use crate::paging::entry::{EntryFlags};
use crate::paging::{ActivePageTable, Page, PAGE_SIZE, PhysicalAddress, VirtualAddress};
use crate::allocator;
use crate::device;
@@ -174,7 +173,7 @@ pub unsafe extern fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! {
}
#[naked]
pub unsafe fn usermode(ip: usize, sp: usize, arg: usize, singlestep: bool) -> ! {
pub unsafe fn usermode(ip: usize, sp: usize, arg: usize, _singlestep: u32) -> ! {
let cpu_id: usize = 0;
let spsr: u32 = 0;

6
src/arch/x86_64/paging/mapper.rs
View File

@@ -91,7 +91,7 @@ impl Mapper {
}
if let Some(p1_frame) = p2[page.p2_index()].pointed_frame() {
//println!("Free p1 {:?}", p1_frame);
//println!("unmap_inner: Free p1 {:?}", p1_frame);
p2.decrement_entry_count();
p2[page.p2_index()].set_unused();
deallocate_frames(p1_frame, 1);
@@ -107,7 +107,7 @@ impl Mapper {
}
if let Some(p2_frame) = p3[page.p3_index()].pointed_frame() {
//println!("Free p2 {:?}", p2_frame);
//println!("unmap_inner: Free p2 {:?}", p2_frame);
p3.decrement_entry_count();
p3[page.p3_index()].set_unused();
deallocate_frames(p2_frame, 1);
@@ -123,7 +123,7 @@ impl Mapper {
}
if let Some(p3_frame) = p4[page.p4_index()].pointed_frame() {
//println!("Free p3 {:?}", p3_frame);
//println!("unmap_inner: Free p3 {:?}", p3_frame);
p4.decrement_entry_count();
p4[page.p4_index()].set_unused();
deallocate_frames(p3_frame, 1);

304
src/context/arch/aarch64.rs
View File

@@ -220,163 +220,163 @@ impl Context {
println!("x9: 0x{:016x}", self.x9);
println!("x8: 0x{:016x}", self.x8);
}
}
#[cold]
#[inline(never)]
#[naked]
pub unsafe fn switch_to(&mut self, next: &mut Context) {
let mut float_regs = &mut *(self.fx_address as *mut FloatRegisters);
#[cold]
#[inline(never)]
#[naked]
pub unsafe fn switch_to(prev: &mut Context, next: &mut Context) {
let mut float_regs = &mut *(prev.fx_address as *mut FloatRegisters);
asm!(
"stp q0, q1, [{0}, #16 * 0]",
"stp q2, q3, [{0}, #16 * 2]",
"stp q4, q5, [{0}, #16 * 4]",
"stp q6, q7, [{0}, #16 * 6]",
"stp q8, q9, [{0}, #16 * 8]",
"stp q10, q11, [{0}, #16 * 10]",
"stp q12, q13, [{0}, #16 * 12]",
"stp q14, q15, [{0}, #16 * 14]",
"stp q16, q17, [{0}, #16 * 16]",
"stp q18, q19, [{0}, #16 * 18]",
"stp q20, q21, [{0}, #16 * 20]",
"stp q22, q23, [{0}, #16 * 22]",
"stp q24, q25, [{0}, #16 * 24]",
"stp q26, q27, [{0}, #16 * 26]",
"stp q28, q29, [{0}, #16 * 28]",
"stp q30, q31, [{0}, #16 * 30]",
"mrs {1}, fpcr",
"mrs {2}, fpsr",
in(reg) &mut float_regs.fp_simd_regs,
out(reg) float_regs.fpcr,
out(reg) float_regs.fpsr
);
prev.fx_loadable = true;
if next.fx_loadable {
let mut float_regs = &mut *(next.fx_address as *mut FloatRegisters);
asm!(
"stp q0, q1, [{0}, #16 * 0]",
"stp q2, q3, [{0}, #16 * 2]",
"stp q4, q5, [{0}, #16 * 4]",
"stp q6, q7, [{0}, #16 * 6]",
"stp q8, q9, [{0}, #16 * 8]",
"stp q10, q11, [{0}, #16 * 10]",
"stp q12, q13, [{0}, #16 * 12]",
"stp q14, q15, [{0}, #16 * 14]",
"stp q16, q17, [{0}, #16 * 16]",
"stp q18, q19, [{0}, #16 * 18]",
"stp q20, q21, [{0}, #16 * 20]",
"stp q22, q23, [{0}, #16 * 22]",
"stp q24, q25, [{0}, #16 * 24]",
"stp q26, q27, [{0}, #16 * 26]",
"stp q28, q29, [{0}, #16 * 28]",
"stp q30, q31, [{0}, #16 * 30]",
"mrs {1}, fpcr",
"mrs {2}, fpsr",
"ldp q0, q1, [{0}, #16 * 0]",
"ldp q2, q3, [{0}, #16 * 2]",
"ldp q4, q5, [{0}, #16 * 4]",
"ldp q6, q7, [{0}, #16 * 6]",
"ldp q8, q9, [{0}, #16 * 8]",
"ldp q10, q11, [{0}, #16 * 10]",
"ldp q12, q13, [{0}, #16 * 12]",
"ldp q14, q15, [{0}, #16 * 14]",
"ldp q16, q17, [{0}, #16 * 16]",
"ldp q18, q19, [{0}, #16 * 18]",
"ldp q20, q21, [{0}, #16 * 20]",
"ldp q22, q23, [{0}, #16 * 22]",
"ldp q24, q25, [{0}, #16 * 24]",
"ldp q26, q27, [{0}, #16 * 26]",
"ldp q28, q29, [{0}, #16 * 28]",
"ldp q30, q31, [{0}, #16 * 30]",
"msr fpcr, {1}",
"msr fpsr, {2}",
in(reg) &mut float_regs.fp_simd_regs,
out(reg) float_regs.fpcr,
out(reg) float_regs.fpsr
in(reg) float_regs.fpcr,
in(reg) float_regs.fpsr
);
self.fx_loadable = true;
if next.fx_loadable {
let mut float_regs = &mut *(next.fx_address as *mut FloatRegisters);
asm!(
"ldp q0, q1, [{0}, #16 * 0]",
"ldp q2, q3, [{0}, #16 * 2]",
"ldp q4, q5, [{0}, #16 * 4]",
"ldp q6, q7, [{0}, #16 * 6]",
"ldp q8, q9, [{0}, #16 * 8]",
"ldp q10, q11, [{0}, #16 * 10]",
"ldp q12, q13, [{0}, #16 * 12]",
"ldp q14, q15, [{0}, #16 * 14]",
"ldp q16, q17, [{0}, #16 * 16]",
"ldp q18, q19, [{0}, #16 * 18]",
"ldp q20, q21, [{0}, #16 * 20]",
"ldp q22, q23, [{0}, #16 * 22]",
"ldp q24, q25, [{0}, #16 * 24]",
"ldp q26, q27, [{0}, #16 * 26]",
"ldp q28, q29, [{0}, #16 * 28]",
"ldp q30, q31, [{0}, #16 * 30]",
"msr fpcr, {1}",
"msr fpsr, {2}",
in(reg) &mut float_regs.fp_simd_regs,
in(reg) float_regs.fpcr,
in(reg) float_regs.fpsr
);
}
self.ttbr0_el1 = control_regs::ttbr0_el1() as usize;
if next.ttbr0_el1 != self.ttbr0_el1 {
control_regs::ttbr0_el1_write(next.ttbr0_el1 as u64);
tlb::flush_all();
}
llvm_asm!("mov $0, x8" : "=r"(self.x8) : : "memory" : "volatile");
llvm_asm!("mov x8, $0" : : "r"(next.x8) :"memory" : "volatile");
llvm_asm!("mov $0, x9" : "=r"(self.x9) : : "memory" : "volatile");
llvm_asm!("mov x9, $0" : : "r"(next.x9) :"memory" : "volatile");
llvm_asm!("mov $0, x10" : "=r"(self.x10) : : "memory" : "volatile");
llvm_asm!("mov x10, $0" : : "r"(next.x10) :"memory" : "volatile");
llvm_asm!("mov $0, x11" : "=r"(self.x11) : : "memory" : "volatile");
llvm_asm!("mov x11, $0" : : "r"(next.x11) :"memory" : "volatile");
llvm_asm!("mov $0, x12" : "=r"(self.x12) : : "memory" : "volatile");
llvm_asm!("mov x12, $0" : : "r"(next.x12) :"memory" : "volatile");
llvm_asm!("mov $0, x13" : "=r"(self.x13) : : "memory" : "volatile");
llvm_asm!("mov x13, $0" : : "r"(next.x13) :"memory" : "volatile");
llvm_asm!("mov $0, x14" : "=r"(self.x14) : : "memory" : "volatile");
llvm_asm!("mov x14, $0" : : "r"(next.x14) :"memory" : "volatile");
llvm_asm!("mov $0, x15" : "=r"(self.x15) : : "memory" : "volatile");
llvm_asm!("mov x15, $0" : : "r"(next.x15) :"memory" : "volatile");
llvm_asm!("mov $0, x16" : "=r"(self.x16) : : "memory" : "volatile");
llvm_asm!("mov x16, $0" : : "r"(next.x16) :"memory" : "volatile");
llvm_asm!("mov $0, x17" : "=r"(self.x17) : : "memory" : "volatile");
llvm_asm!("mov x17, $0" : : "r"(next.x17) :"memory" : "volatile");
llvm_asm!("mov $0, x18" : "=r"(self.x18) : : "memory" : "volatile");
llvm_asm!("mov x18, $0" : : "r"(next.x18) :"memory" : "volatile");
llvm_asm!("mov $0, x19" : "=r"(self.x19) : : "memory" : "volatile");
llvm_asm!("mov x19, $0" : : "r"(next.x19) :"memory" : "volatile");
llvm_asm!("mov $0, x20" : "=r"(self.x20) : : "memory" : "volatile");
llvm_asm!("mov x20, $0" : : "r"(next.x20) :"memory" : "volatile");
llvm_asm!("mov $0, x21" : "=r"(self.x21) : : "memory" : "volatile");
llvm_asm!("mov x21, $0" : : "r"(next.x21) :"memory" : "volatile");
llvm_asm!("mov $0, x22" : "=r"(self.x22) : : "memory" : "volatile");
llvm_asm!("mov x22, $0" : : "r"(next.x22) :"memory" : "volatile");
llvm_asm!("mov $0, x23" : "=r"(self.x23) : : "memory" : "volatile");
llvm_asm!("mov x23, $0" : : "r"(next.x23) :"memory" : "volatile");
llvm_asm!("mov $0, x24" : "=r"(self.x24) : : "memory" : "volatile");
llvm_asm!("mov x24, $0" : : "r"(next.x24) :"memory" : "volatile");
llvm_asm!("mov $0, x25" : "=r"(self.x25) : : "memory" : "volatile");
llvm_asm!("mov x25, $0" : : "r"(next.x25) :"memory" : "volatile");
llvm_asm!("mov $0, x26" : "=r"(self.x26) : : "memory" : "volatile");
llvm_asm!("mov x26, $0" : : "r"(next.x26) :"memory" : "volatile");
llvm_asm!("mov $0, x27" : "=r"(self.x27) : : "memory" : "volatile");
llvm_asm!("mov x27, $0" : : "r"(next.x27) :"memory" : "volatile");
llvm_asm!("mov $0, x28" : "=r"(self.x28) : : "memory" : "volatile");
llvm_asm!("mov x28, $0" : : "r"(next.x28) :"memory" : "volatile");
llvm_asm!("mov $0, x29" : "=r"(self.fp) : : "memory" : "volatile");
llvm_asm!("mov x29, $0" : : "r"(next.fp) :"memory" : "volatile");
llvm_asm!("mov $0, x30" : "=r"(self.lr) : : "memory" : "volatile");
llvm_asm!("mov x30, $0" : : "r"(next.lr) :"memory" : "volatile");
llvm_asm!("mrs $0, elr_el1" : "=r"(self.elr_el1) : : "memory" : "volatile");
llvm_asm!("msr elr_el1, $0" : : "r"(next.elr_el1) : "memory" : "volatile");
llvm_asm!("mrs $0, sp_el0" : "=r"(self.sp_el0) : : "memory" : "volatile");
llvm_asm!("msr sp_el0, $0" : : "r"(next.sp_el0) : "memory" : "volatile");
llvm_asm!("mrs $0, tpidr_el0" : "=r"(self.tpidr_el0) : : "memory" : "volatile");
llvm_asm!("msr tpidr_el0, $0" : : "r"(next.tpidr_el0) : "memory" : "volatile");
llvm_asm!("mrs $0, tpidrro_el0" : "=r"(self.tpidrro_el0) : : "memory" : "volatile");
llvm_asm!("msr tpidrro_el0, $0" : : "r"(next.tpidrro_el0) : "memory" : "volatile");
llvm_asm!("mrs $0, spsr_el1" : "=r"(self.spsr_el1) : : "memory" : "volatile");
llvm_asm!("msr spsr_el1, $0" : : "r"(next.spsr_el1) : "memory" : "volatile");
llvm_asm!("mrs $0, esr_el1" : "=r"(self.esr_el1) : : "memory" : "volatile");
llvm_asm!("msr esr_el1, $0" : : "r"(next.esr_el1) : "memory" : "volatile");
llvm_asm!("mov $0, sp" : "=r"(self.sp) : : "memory" : "volatile");
llvm_asm!("mov sp, $0" : : "r"(next.sp) : "memory" : "volatile");
CONTEXT_SWITCH_LOCK.store(false, Ordering::SeqCst);
}
prev.ttbr0_el1 = control_regs::ttbr0_el1() as usize;
if next.ttbr0_el1 != prev.ttbr0_el1 {
control_regs::ttbr0_el1_write(next.ttbr0_el1 as u64);
tlb::flush_all();
}
llvm_asm!("mov $0, x8" : "=r"(prev.x8) : : "memory" : "volatile");
llvm_asm!("mov x8, $0" : : "r"(next.x8) :"memory" : "volatile");
llvm_asm!("mov $0, x9" : "=r"(prev.x9) : : "memory" : "volatile");
llvm_asm!("mov x9, $0" : : "r"(next.x9) :"memory" : "volatile");
llvm_asm!("mov $0, x10" : "=r"(prev.x10) : : "memory" : "volatile");
llvm_asm!("mov x10, $0" : : "r"(next.x10) :"memory" : "volatile");
llvm_asm!("mov $0, x11" : "=r"(prev.x11) : : "memory" : "volatile");
llvm_asm!("mov x11, $0" : : "r"(next.x11) :"memory" : "volatile");
llvm_asm!("mov $0, x12" : "=r"(prev.x12) : : "memory" : "volatile");
llvm_asm!("mov x12, $0" : : "r"(next.x12) :"memory" : "volatile");
llvm_asm!("mov $0, x13" : "=r"(prev.x13) : : "memory" : "volatile");
llvm_asm!("mov x13, $0" : : "r"(next.x13) :"memory" : "volatile");
llvm_asm!("mov $0, x14" : "=r"(prev.x14) : : "memory" : "volatile");
llvm_asm!("mov x14, $0" : : "r"(next.x14) :"memory" : "volatile");
llvm_asm!("mov $0, x15" : "=r"(prev.x15) : : "memory" : "volatile");
llvm_asm!("mov x15, $0" : : "r"(next.x15) :"memory" : "volatile");
llvm_asm!("mov $0, x16" : "=r"(prev.x16) : : "memory" : "volatile");
llvm_asm!("mov x16, $0" : : "r"(next.x16) :"memory" : "volatile");
llvm_asm!("mov $0, x17" : "=r"(prev.x17) : : "memory" : "volatile");
llvm_asm!("mov x17, $0" : : "r"(next.x17) :"memory" : "volatile");
llvm_asm!("mov $0, x18" : "=r"(prev.x18) : : "memory" : "volatile");
llvm_asm!("mov x18, $0" : : "r"(next.x18) :"memory" : "volatile");
llvm_asm!("mov $0, x19" : "=r"(prev.x19) : : "memory" : "volatile");
llvm_asm!("mov x19, $0" : : "r"(next.x19) :"memory" : "volatile");
llvm_asm!("mov $0, x20" : "=r"(prev.x20) : : "memory" : "volatile");
llvm_asm!("mov x20, $0" : : "r"(next.x20) :"memory" : "volatile");
llvm_asm!("mov $0, x21" : "=r"(prev.x21) : : "memory" : "volatile");
llvm_asm!("mov x21, $0" : : "r"(next.x21) :"memory" : "volatile");
llvm_asm!("mov $0, x22" : "=r"(prev.x22) : : "memory" : "volatile");
llvm_asm!("mov x22, $0" : : "r"(next.x22) :"memory" : "volatile");
llvm_asm!("mov $0, x23" : "=r"(prev.x23) : : "memory" : "volatile");
llvm_asm!("mov x23, $0" : : "r"(next.x23) :"memory" : "volatile");
llvm_asm!("mov $0, x24" : "=r"(prev.x24) : : "memory" : "volatile");
llvm_asm!("mov x24, $0" : : "r"(next.x24) :"memory" : "volatile");
llvm_asm!("mov $0, x25" : "=r"(prev.x25) : : "memory" : "volatile");
llvm_asm!("mov x25, $0" : : "r"(next.x25) :"memory" : "volatile");
llvm_asm!("mov $0, x26" : "=r"(prev.x26) : : "memory" : "volatile");
llvm_asm!("mov x26, $0" : : "r"(next.x26) :"memory" : "volatile");
llvm_asm!("mov $0, x27" : "=r"(prev.x27) : : "memory" : "volatile");
llvm_asm!("mov x27, $0" : : "r"(next.x27) :"memory" : "volatile");
llvm_asm!("mov $0, x28" : "=r"(prev.x28) : : "memory" : "volatile");
llvm_asm!("mov x28, $0" : : "r"(next.x28) :"memory" : "volatile");
llvm_asm!("mov $0, x29" : "=r"(prev.fp) : : "memory" : "volatile");
llvm_asm!("mov x29, $0" : : "r"(next.fp) :"memory" : "volatile");
llvm_asm!("mov $0, x30" : "=r"(prev.lr) : : "memory" : "volatile");
llvm_asm!("mov x30, $0" : : "r"(next.lr) :"memory" : "volatile");
llvm_asm!("mrs $0, elr_el1" : "=r"(prev.elr_el1) : : "memory" : "volatile");
llvm_asm!("msr elr_el1, $0" : : "r"(next.elr_el1) : "memory" : "volatile");
llvm_asm!("mrs $0, sp_el0" : "=r"(prev.sp_el0) : : "memory" : "volatile");
llvm_asm!("msr sp_el0, $0" : : "r"(next.sp_el0) : "memory" : "volatile");
llvm_asm!("mrs $0, tpidr_el0" : "=r"(prev.tpidr_el0) : : "memory" : "volatile");
llvm_asm!("msr tpidr_el0, $0" : : "r"(next.tpidr_el0) : "memory" : "volatile");
llvm_asm!("mrs $0, tpidrro_el0" : "=r"(prev.tpidrro_el0) : : "memory" : "volatile");
llvm_asm!("msr tpidrro_el0, $0" : : "r"(next.tpidrro_el0) : "memory" : "volatile");
llvm_asm!("mrs $0, spsr_el1" : "=r"(prev.spsr_el1) : : "memory" : "volatile");
llvm_asm!("msr spsr_el1, $0" : : "r"(next.spsr_el1) : "memory" : "volatile");
llvm_asm!("mrs $0, esr_el1" : "=r"(prev.esr_el1) : : "memory" : "volatile");
llvm_asm!("msr esr_el1, $0" : : "r"(next.esr_el1) : "memory" : "volatile");
llvm_asm!("mov $0, sp" : "=r"(prev.sp) : : "memory" : "volatile");
llvm_asm!("mov sp, $0" : : "r"(next.sp) : "memory" : "volatile");
CONTEXT_SWITCH_LOCK.store(false, Ordering::SeqCst);
}
#[allow(dead_code)]