diff --git a/aarch64/src/kalloc.rs b/aarch64/src/kalloc.rs index 7bdb886..973ae3b 100644 --- a/aarch64/src/kalloc.rs +++ b/aarch64/src/kalloc.rs @@ -1,6 +1,9 @@ -use crate::vm::{Page4K, PAGE_SIZE_4K}; +use crate::vm::Page4K; use core::ptr; -use port::mcslock::{Lock, LockNode}; +use port::{ + mcslock::{Lock, LockNode}, + mem::PAGE_SIZE_4K, +}; static FREE_LIST: Lock = Lock::new("kmem", FreeList { next: None }); diff --git a/aarch64/src/kmem.rs b/aarch64/src/kmem.rs index 582b8fa..14e8cef 100644 --- a/aarch64/src/kmem.rs +++ b/aarch64/src/kmem.rs @@ -1,13 +1,7 @@ -use port::fdt::RegBlock; +use port::mem::PhysAddr; use crate::{param::KZERO, vm::Page4K}; -use core::{ - fmt, - iter::{Step, StepBy}, - mem, - ops::{self, Range}, - slice, -}; +use core::{mem, slice}; // These map to definitions in kernel.ld extern "C" { @@ -42,119 +36,20 @@ pub fn eearly_pagetables_addr() -> usize { unsafe { eearly_pagetables.as_ptr().addr() } } -#[derive(Clone, Copy, PartialEq, PartialOrd, Eq, Ord)] -#[repr(transparent)] -pub struct PhysAddr(u64); - -impl PhysAddr { - pub const fn new(value: u64) -> Self { - PhysAddr(value) - } - - pub const fn addr(&self) -> u64 { - self.0 - } - - pub const fn as_virt(&self) -> usize { - (self.0 as usize).wrapping_add(KZERO) - } - - pub fn from_virt(a: usize) -> Self { - Self((a - KZERO) as u64) - } - - pub fn from_ptr(a: *const T) -> Self { - Self::from_virt(a.addr()) - } - - pub const fn as_ptr_mut(&self) -> *mut T { - self.as_virt() as *mut T - } - - pub const fn round_up(&self, step: u64) -> PhysAddr { - PhysAddr((self.0 + step - 1) & !(step - 1)) - } - - pub const fn round_down(&self, step: u64) -> PhysAddr { - PhysAddr(self.0 & !(step - 1)) - } +pub const fn physaddr_as_virt(pa: PhysAddr) -> usize { + (pa.addr() as usize).wrapping_add(KZERO) } -impl ops::Add for PhysAddr { - type Output = PhysAddr; - - fn add(self, offset: u64) -> PhysAddr { - PhysAddr(self.0 + offset) - } +pub const fn physaddr_as_ptr_mut(pa: PhysAddr) -> *mut T { + physaddr_as_virt(pa) as *mut T } -/// Note that this implementation will round down the startpa and round up the endpa -impl Step for PhysAddr { - fn steps_between(&startpa: &Self, &endpa: &Self) -> Option { - if startpa.0 <= endpa.0 { - match endpa.0.checked_sub(startpa.0) { - Some(result) => usize::try_from(result).ok(), - None => None, - } - } else { - None - } - } - - fn forward_checked(startpa: Self, count: usize) -> Option { - startpa.0.checked_add(count as u64).map(PhysAddr) - } - - fn backward_checked(startpa: Self, count: usize) -> Option { - startpa.0.checked_sub(count as u64).map(PhysAddr) - } +pub const fn from_virt_to_physaddr(va: usize) -> PhysAddr { + PhysAddr::new((va - KZERO) as u64) } -impl fmt::Debug for PhysAddr { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "PhysAddr({:#016x})", self.0)?; - Ok(()) - } -} - -pub struct PhysRange(pub Range); - -impl PhysRange { - pub fn with_len(start: u64, len: usize) -> Self { - Self(PhysAddr(start)..PhysAddr(start + len as u64)) - } - - #[allow(dead_code)] - pub fn offset_addr(&self, offset: u64) -> Option { - let addr = self.0.start + offset; - if self.0.contains(&addr) { - Some(addr) - } else { - None - } - } - - pub fn start(&self) -> PhysAddr { - self.0.start - } - - pub fn end(&self) -> PhysAddr { - self.0.end - } - - pub fn step_by_rounded(&self, step_size: usize) -> StepBy> { - let startpa = self.start().round_down(step_size as u64); - let endpa = self.end().round_up(step_size as u64); - (startpa..endpa).step_by(step_size) - } -} - -impl From<&RegBlock> for PhysRange { - fn from(r: &RegBlock) -> Self { - let start = PhysAddr(r.addr); - let end = start + r.len.unwrap_or(0); - PhysRange(start..end) - } +pub fn from_ptr_to_physaddr(a: *const T) -> PhysAddr { + from_virt_to_physaddr(a.addr()) } unsafe fn page_slice_mut<'a>(pstart: *mut Page4K, pend: *mut Page4K) -> &'a mut [Page4K] { @@ -174,33 +69,3 @@ pub fn early_pages() -> &'static mut [Page4K] { let early_end = eearly_pagetables_addr() as *mut Page4K; unsafe { page_slice_mut(early_start, early_end) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::vm; - - #[test] - fn physaddr_step() { - let range = PhysRange(PhysAddr::new(4096)..PhysAddr::new(4096 * 3)); - let pas = range.step_by_rounded(vm::PAGE_SIZE_4K).collect::>(); - assert_eq!(pas, [PhysAddr::new(4096), PhysAddr::new(4096 * 2)]); - } - - #[test] - fn physaddr_step_rounds_up_and_down() { - // Start should round down to 8192 - // End should round up to 16384 - let range = PhysRange(PhysAddr::new(9000)..PhysAddr::new(5000 * 3)); - let pas = range.step_by_rounded(vm::PAGE_SIZE_4K).collect::>(); - assert_eq!(pas, [PhysAddr::new(4096 * 2), PhysAddr::new(4096 * 3)]); - } - - #[test] - fn physaddr_step_2m() { - let range = - PhysRange(PhysAddr::new(0x3f000000)..PhysAddr::new(0x3f000000 + 4 * 1024 * 1024)); - let pas = range.step_by_rounded(vm::PAGE_SIZE_2M).collect::>(); - assert_eq!(pas, [PhysAddr::new(0x3f000000), PhysAddr::new(0x3f000000 + 2 * 1024 * 1024)]); - } -} diff --git a/aarch64/src/main.rs b/aarch64/src/main.rs index a62c7e0..4e10056 100644 --- a/aarch64/src/main.rs +++ b/aarch64/src/main.rs @@ -6,7 +6,6 @@ #![feature(asm_const)] #![feature(core_intrinsics)] #![feature(stdsimd)] -#![feature(step_trait)] #![feature(strict_provenance)] #![forbid(unsafe_op_in_unsafe_fn)] @@ -22,11 +21,12 @@ mod uartmini; mod uartpl011; mod vm; -use crate::kmem::{PhysAddr, PhysRange}; +use crate::kmem::from_virt_to_physaddr; use crate::vm::kernel_root; use core::ffi::c_void; use core::ptr; use port::fdt::DeviceTree; +use port::mem::PhysRange; use port::println; use vm::PageTable; @@ -123,7 +123,7 @@ pub extern "C" fn main9(dtb_va: usize) { unsafe { kalloc::free_pages(kmem::early_pages()); - let dtb_range = PhysRange::with_len(PhysAddr::from_virt(dtb_va).addr(), dt.size()); + let dtb_range = PhysRange::with_len(from_virt_to_physaddr(dtb_va).addr(), dt.size()); vm::init(&dt, &mut *ptr::addr_of_mut!(KPGTBL), dtb_range); vm::switch(&*ptr::addr_of!(KPGTBL)); } diff --git a/aarch64/src/registers.rs b/aarch64/src/registers.rs index af6b2ce..65421aa 100644 --- a/aarch64/src/registers.rs +++ b/aarch64/src/registers.rs @@ -1,9 +1,9 @@ #![allow(non_upper_case_globals)] -use crate::{kmem::PhysRange, vm::PAGE_SIZE_2M}; use bitstruct::bitstruct; use core::fmt; use num_enum::TryFromPrimitive; +use port::mem::{PhysRange, PAGE_SIZE_2M}; // GPIO registers pub const GPFSEL1: usize = 0x04; // GPIO function select register 1 diff --git a/aarch64/src/runtime.rs b/aarch64/src/runtime.rs index abf04a3..aa65a36 100644 --- a/aarch64/src/runtime.rs +++ b/aarch64/src/runtime.rs @@ -2,6 +2,7 @@ extern crate alloc; +use crate::kmem::physaddr_as_virt; use crate::registers::rpi_mmio; use crate::uartmini::MiniUart; use alloc::alloc::{GlobalAlloc, Layout}; @@ -16,7 +17,7 @@ use port::mem::VirtRange; // - Add support for raspi4 #[panic_handler] pub fn panic(info: &PanicInfo) -> ! { - let mmio = rpi_mmio().expect("mmio base detect failed").start().as_virt(); + let mmio = physaddr_as_virt(rpi_mmio().expect("mmio base detect failed").start()); let gpio_range = VirtRange::with_len(mmio + 0x200000, 0xb4); let aux_range = VirtRange::with_len(mmio + 0x215000, 0x8); diff --git a/aarch64/src/vm.rs b/aarch64/src/vm.rs index ec9a853..b3e8428 100644 --- a/aarch64/src/vm.rs +++ b/aarch64/src/vm.rs @@ -2,22 +2,24 @@ use crate::{ kalloc, - kmem::{ebss_addr, erodata_addr, etext_addr, text_addr, PhysAddr, PhysRange}, + kmem::{ + ebss_addr, erodata_addr, etext_addr, from_ptr_to_physaddr, from_virt_to_physaddr, + physaddr_as_ptr_mut, physaddr_as_virt, text_addr, + }, registers::rpi_mmio, }; use bitstruct::bitstruct; use core::fmt; use core::ptr::write_volatile; use num_enum::{FromPrimitive, IntoPrimitive}; -use port::fdt::DeviceTree; +use port::{ + fdt::DeviceTree, + mem::{PhysAddr, PhysRange, PAGE_SIZE_1G, PAGE_SIZE_2M, PAGE_SIZE_4K}, +}; #[cfg(not(test))] use port::println; -pub const PAGE_SIZE_4K: usize = 4 << 10; -pub const PAGE_SIZE_2M: usize = 2 << 20; -pub const PAGE_SIZE_1G: usize = 1 << 30; - #[allow(dead_code)] #[derive(Debug, Clone, Copy, PartialEq)] pub enum PageSize { @@ -162,7 +164,7 @@ impl Entry { } fn virt_page_addr(self) -> usize { - self.phys_page_addr().as_virt() + physaddr_as_virt(self.phys_page_addr()) } fn table(self, level: Level) -> bool { @@ -312,7 +314,7 @@ impl Table { // Create a new page table and write the entry into the parent table let table = Self::alloc_pagetable()?; entry = Entry::rw_kernel_data() - .with_phys_addr(PhysAddr::from_ptr(table)) + .with_phys_addr(from_ptr_to_physaddr(table)) .with_page_or_table(true); unsafe { write_volatile(&mut self.entries[index], entry); @@ -358,7 +360,7 @@ impl PageTable { // TODO Only do this if self != kernel_root() let old_recursive_entry = kernel_root().entries[511]; let temp_recursive_entry = Entry::rw_kernel_data() - .with_phys_addr(PhysAddr::from_ptr(self)) + .with_phys_addr(from_ptr_to_physaddr(self)) .with_page_or_table(true); unsafe { @@ -413,7 +415,7 @@ impl PageTable { let mut startva = None; let mut endva = 0; for pa in range.step_by_rounded(page_size.size()) { - let va = pa.as_virt(); + let va = physaddr_as_virt(pa); self.map_to(entry.with_phys_addr(pa), va, page_size)?; startva.get_or_insert(va); endva = va + page_size.size(); @@ -479,7 +481,7 @@ pub unsafe fn init(_dt: &DeviceTree, kpage_table: &mut PageTable, dtb_range: Phy // Write the recursive entry unsafe { let entry = Entry::rw_kernel_data() - .with_phys_addr(PhysAddr::from_ptr(kpage_table)) + .with_phys_addr(from_ptr_to_physaddr(kpage_table)) .with_page_or_table(true); write_volatile(&mut kpage_table.entries[511], entry); } @@ -490,13 +492,13 @@ pub unsafe fn init(_dt: &DeviceTree, kpage_table: &mut PageTable, dtb_range: Phy // in depth! let custom_map = { let text_range = - PhysRange(PhysAddr::from_virt(text_addr())..PhysAddr::from_virt(etext_addr())); + PhysRange(from_virt_to_physaddr(text_addr())..from_virt_to_physaddr(etext_addr())); let data_range = PhysRange::with_len( - PhysAddr::from_virt(etext_addr()).addr(), + from_virt_to_physaddr(etext_addr()).addr(), erodata_addr() - etext_addr(), ); let bss_range = PhysRange::with_len( - PhysAddr::from_virt(erodata_addr()).addr(), + from_virt_to_physaddr(erodata_addr()).addr(), ebss_addr() - erodata_addr(), ); @@ -549,7 +551,7 @@ fn ttbr1_el1() -> u64 { pub unsafe fn switch(kpage_table: &PageTable) { #[cfg(not(test))] unsafe { - let pt_phys = PhysAddr::from_ptr(kpage_table).addr(); + let pt_phys = from_ptr_to_physaddr(kpage_table).addr(); // https://forum.osdev.org/viewtopic.php?t=36412&p=303237 core::arch::asm!( "msr ttbr1_el1, {pt_phys}", @@ -577,7 +579,7 @@ pub unsafe fn invalidate_all_tlb_entries() { /// Return the root kernel page table pub fn kernel_root() -> &'static mut PageTable { - unsafe { &mut *PhysAddr::new(ttbr1_el1()).as_ptr_mut::() } + unsafe { &mut *physaddr_as_ptr_mut::(PhysAddr::new(ttbr1_el1())) } } #[cfg(test)] diff --git a/port/src/lib.rs b/port/src/lib.rs index e152358..aca277a 100644 --- a/port/src/lib.rs +++ b/port/src/lib.rs @@ -1,6 +1,7 @@ #![allow(clippy::upper_case_acronyms)] #![cfg_attr(not(any(test)), no_std)] #![feature(maybe_uninit_slice)] +#![feature(step_trait)] #![forbid(unsafe_op_in_unsafe_fn)] pub mod dat; diff --git a/port/src/mem.rs b/port/src/mem.rs index 5cac028..d232c0c 100644 --- a/port/src/mem.rs +++ b/port/src/mem.rs @@ -1,5 +1,13 @@ use crate::fdt::RegBlock; -use core::ops::Range; +use core::{ + fmt, + iter::{Step, StepBy}, + ops::{self, Range}, +}; + +pub const PAGE_SIZE_4K: usize = 4 << 10; +pub const PAGE_SIZE_2M: usize = 2 << 20; +pub const PAGE_SIZE_1G: usize = 1 << 30; pub struct VirtRange(pub Range); @@ -33,3 +41,131 @@ impl From<&RegBlock> for VirtRange { VirtRange(start..end) } } + +#[derive(Clone, Copy, PartialEq, PartialOrd, Eq, Ord)] +#[repr(transparent)] +pub struct PhysAddr(u64); + +impl PhysAddr { + pub const fn new(value: u64) -> Self { + PhysAddr(value) + } + + pub const fn addr(&self) -> u64 { + self.0 + } + + pub const fn round_up(&self, step: u64) -> PhysAddr { + PhysAddr((self.0 + step - 1) & !(step - 1)) + } + + pub const fn round_down(&self, step: u64) -> PhysAddr { + PhysAddr(self.0 & !(step - 1)) + } +} + +impl ops::Add for PhysAddr { + type Output = PhysAddr; + + fn add(self, offset: u64) -> PhysAddr { + PhysAddr(self.0 + offset) + } +} + +/// Note that this implementation will round down the startpa and round up the endpa +impl Step for PhysAddr { + fn steps_between(&startpa: &Self, &endpa: &Self) -> Option { + if startpa.0 <= endpa.0 { + match endpa.0.checked_sub(startpa.0) { + Some(result) => usize::try_from(result).ok(), + None => None, + } + } else { + None + } + } + + fn forward_checked(startpa: Self, count: usize) -> Option { + startpa.0.checked_add(count as u64).map(PhysAddr) + } + + fn backward_checked(startpa: Self, count: usize) -> Option { + startpa.0.checked_sub(count as u64).map(PhysAddr) + } +} + +impl fmt::Debug for PhysAddr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "PhysAddr({:#016x})", self.0)?; + Ok(()) + } +} + +pub struct PhysRange(pub Range); + +impl PhysRange { + pub fn with_len(start: u64, len: usize) -> Self { + Self(PhysAddr(start)..PhysAddr(start + len as u64)) + } + + #[allow(dead_code)] + pub fn offset_addr(&self, offset: u64) -> Option { + let addr = self.0.start + offset; + if self.0.contains(&addr) { + Some(addr) + } else { + None + } + } + + pub fn start(&self) -> PhysAddr { + self.0.start + } + + pub fn end(&self) -> PhysAddr { + self.0.end + } + + pub fn step_by_rounded(&self, step_size: usize) -> StepBy> { + let startpa = self.start().round_down(step_size as u64); + let endpa = self.end().round_up(step_size as u64); + (startpa..endpa).step_by(step_size) + } +} + +impl From<&RegBlock> for PhysRange { + fn from(r: &RegBlock) -> Self { + let start = PhysAddr(r.addr); + let end = start + r.len.unwrap_or(0); + PhysRange(start..end) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn physaddr_step() { + let range = PhysRange(PhysAddr::new(4096)..PhysAddr::new(4096 * 3)); + let pas = range.step_by_rounded(PAGE_SIZE_4K).collect::>(); + assert_eq!(pas, [PhysAddr::new(4096), PhysAddr::new(4096 * 2)]); + } + + #[test] + fn physaddr_step_rounds_up_and_down() { + // Start should round down to 8192 + // End should round up to 16384 + let range = PhysRange(PhysAddr::new(9000)..PhysAddr::new(5000 * 3)); + let pas = range.step_by_rounded(PAGE_SIZE_4K).collect::>(); + assert_eq!(pas, [PhysAddr::new(4096 * 2), PhysAddr::new(4096 * 3)]); + } + + #[test] + fn physaddr_step_2m() { + let range = + PhysRange(PhysAddr::new(0x3f000000)..PhysAddr::new(0x3f000000 + 4 * 1024 * 1024)); + let pas = range.step_by_rounded(PAGE_SIZE_2M).collect::>(); + assert_eq!(pas, [PhysAddr::new(0x3f000000), PhysAddr::new(0x3f000000 + 2 * 1024 * 1024)]); + } +}