Simple pagetable setup in rust

aarch64/Cargo.toml

@@ -6,4 +6,4 @@ edition = "2021"
 [dependencies]
 bitstruct = "0.1"
 port = { path = "../port" }
-num_enum = { version = "0.6.1", default-features = false }
+num_enum = { version = "0.7.0", default-features = false }

aarch64/lib/kernel.ld

@@ -49,8 +49,27 @@ SECTIONS {
 		*(COMMON)
 		. = ALIGN(2097152);
 	}
-	end = .;
+	ebss = .;
+
+	/* Reserve section for kernel heap.  Align to 2MiB to allow us to map as
+	   a 2MiB page. */
+	. = ALIGN(2 * 1024 * 1024);
+	heap = .;
+	. += 64 * 1024 * 1024;
+	eheap = .;
 
+	/* Reserve section for early pagetables.  Align to 2MiB to allow us to map
+	   as a 2MiB page.Note that this won't be needed once we transition to
+	   recursive pagetables.
+	   Note this can go when we use recursive pagetables */
+	. = ALIGN(2 * 1024 * 1024);
+	early_pagetables = .;
+	. += 2 * 1024 * 1024;
+	eearly_pagetables = .;
+
+	end = .;
+	PROVIDE(end = .);
+
 	/DISCARD/ : {
 		*(.eh_frame .note.GNU-stack)
 	}

aarch64/src/kalloc.rs

@@ -0,0 +1,54 @@
+use crate::vm::{Page4K, PAGE_SIZE_4K};
+use core::ptr;
+use port::mcslock::{Lock, LockNode};
+
+static FREE_LIST: Lock<FreeList> = Lock::new("kmem", FreeList { next: None });
+
+#[repr(align(4096))]
+struct FreeList {
+    next: Option<ptr::NonNull<FreeList>>,
+}
+unsafe impl Send for FreeList {}
+
+#[derive(Debug)]
+pub enum Error {
+    NoFreeBlocks,
+}
+
+impl FreeList {
+    pub fn put(&mut self, page: &mut Page4K) {
+        let ptr = (page as *mut Page4K).addr();
+        assert_eq!(ptr % PAGE_SIZE_4K, 0, "freeing unaligned page");
+        page.scribble();
+        let f = page as *mut Page4K as *mut FreeList;
+        unsafe {
+            ptr::write(f, FreeList { next: self.next });
+        }
+        self.next = ptr::NonNull::new(f);
+    }
+
+    pub fn get(&mut self) -> Result<&'static mut Page4K, Error> {
+        let mut next = self.next.ok_or(Error::NoFreeBlocks)?;
+        let next = unsafe { next.as_mut() };
+        self.next = next.next;
+        let pg = unsafe { &mut *(next as *mut FreeList as *mut Page4K) };
+        pg.clear();
+        Ok(pg)
+    }
+}
+
+pub unsafe fn free_pages(pages: &mut [Page4K]) {
+    static mut NODE: LockNode = LockNode::new();
+    let mut lock = FREE_LIST.lock(unsafe { &NODE });
+    let fl = &mut *lock;
+    for page in pages.iter_mut() {
+        fl.put(page);
+    }
+}
+
+pub fn alloc() -> Result<&'static mut Page4K, Error> {
+    static mut NODE: LockNode = LockNode::new();
+    let mut lock = FREE_LIST.lock(unsafe { &NODE });
+    let fl = &mut *lock;
+    fl.get()
+}

aarch64/src/kmem.rs

@@ -0,0 +1,187 @@
+use crate::{param::KZERO, vm::Page4K};
+use core::{
+    fmt,
+    iter::{Step, StepBy},
+    mem,
+    ops::{self, Range},
+    slice,
+};
+
+// These map to definitions in kernel.ld
+extern "C" {
+    static etext: [u64; 0];
+    static erodata: [u64; 0];
+    static ebss: [u64; 0];
+    static early_pagetables: [u64; 0];
+    static eearly_pagetables: [u64; 0];
+    static heap: [u64; 0];
+    static eheap: [u64; 0];
+}
+
+pub fn text_addr() -> usize {
+    0xFFFF_8000_0000_0000
+}
+
+pub fn etext_addr() -> usize {
+    unsafe { etext.as_ptr().addr() }
+}
+
+pub fn erodata_addr() -> usize {
+    unsafe { erodata.as_ptr().addr() }
+}
+
+pub fn ebss_addr() -> usize {
+    unsafe { ebss.as_ptr().addr() }
+}
+
+pub fn heap_addr() -> usize {
+    unsafe { heap.as_ptr().addr() }
+}
+
+pub fn eheap_addr() -> usize {
+    unsafe { eheap.as_ptr().addr() }
+}
+
+pub fn early_pagetables_addr() -> usize {
+    unsafe { early_pagetables.as_ptr().addr() }
+}
+
+pub fn eearly_pagetables_addr() -> usize {
+    unsafe { eearly_pagetables.as_ptr().addr() }
+}
+
+#[derive(Clone, Copy, PartialEq, PartialOrd)]
+#[repr(transparent)]
+pub struct PhysAddr(u64);
+
+impl PhysAddr {
+    pub const fn new(value: u64) -> Self {
+        PhysAddr(value)
+    }
+
+    pub const fn addr(&self) -> u64 {
+        self.0
+    }
+
+    pub const fn to_virt(&self) -> usize {
+        (self.0 as usize).wrapping_add(KZERO)
+    }
+
+    pub fn from_virt(a: usize) -> Self {
+        Self((a - KZERO) as u64)
+    }
+
+    pub fn from_ptr<T>(a: *const T) -> Self {
+        Self::from_virt(a.addr())
+    }
+
+    pub const fn to_ptr_mut<T>(&self) -> *mut T {
+        self.to_virt() as *mut T
+    }
+
+    pub const fn round_up(&self, step: u64) -> PhysAddr {
+        PhysAddr((self.0 + step - 1) & !(step - 1))
+    }
+
+    pub const fn round_down(&self, step: u64) -> PhysAddr {
+        PhysAddr(self.0 & !(step - 1))
+    }
+
+    pub fn step_by_rounded(
+        startpa: PhysAddr,
+        endpa: PhysAddr,
+        step_size: usize,
+    ) -> StepBy<Range<Self>> {
+        let startpa = startpa.round_down(step_size as u64);
+        let endpa = endpa.round_up(step_size as u64);
+        (startpa..endpa).step_by(step_size)
+    }
+}
+
+impl ops::Add<u64> for PhysAddr {
+    type Output = PhysAddr;
+
+    fn add(self, offset: u64) -> PhysAddr {
+        PhysAddr(self.0 + offset)
+    }
+}
+
+/// Note that this implementation will round down the startpa and round up the endpa
+impl Step for PhysAddr {
+    fn steps_between(&startpa: &Self, &endpa: &Self) -> Option<usize> {
+        if startpa.0 <= endpa.0 {
+            match endpa.0.checked_sub(startpa.0) {
+                Some(result) => usize::try_from(result).ok(),
+                None => None,
+            }
+        } else {
+            None
+        }
+    }
+
+    fn forward_checked(startpa: Self, count: usize) -> Option<Self> {
+        startpa.0.checked_add(count as u64).map(|x| PhysAddr(x))
+    }
+
+    fn backward_checked(startpa: Self, count: usize) -> Option<Self> {
+        startpa.0.checked_sub(count as u64).map(|x| PhysAddr(x))
+    }
+}
+
+impl fmt::Debug for PhysAddr {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "PhysAddr({:#016x})", self.0)?;
+        Ok(())
+    }
+}
+
+unsafe fn page_slice_mut<'a>(pstart: *mut Page4K, pend: *mut Page4K) -> &'a mut [Page4K] {
+    let ustart = pstart.addr();
+    let uend = pend.addr();
+    const PAGE_SIZE: usize = mem::size_of::<Page4K>();
+    assert_eq!(ustart % PAGE_SIZE, 0, "page_slice_mut: unaligned start page");
+    assert_eq!(uend % PAGE_SIZE, 0, "page_slice_mut: unaligned end page");
+    assert!(ustart < uend, "page_slice_mut: bad range");
+
+    let len = (uend - ustart) / PAGE_SIZE;
+    unsafe { slice::from_raw_parts_mut(ustart as *mut Page4K, len) }
+}
+
+pub fn early_pages() -> &'static mut [Page4K] {
+    let early_start = early_pagetables_addr() as *mut Page4K;
+    let early_end = eearly_pagetables_addr() as *mut Page4K;
+    unsafe { page_slice_mut(early_start, early_end) }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::vm;
+
+    #[test]
+    fn physaddr_step() {
+        let startpa = PhysAddr::new(4096);
+        let endpa = PhysAddr::new(4096 * 3);
+        let pas =
+            PhysAddr::step_by_rounded(startpa, endpa, vm::PAGE_SIZE_4K).collect::<Vec<PhysAddr>>();
+        assert_eq!(pas, [PhysAddr::new(4096), PhysAddr::new(4096 * 2)]);
+    }
+
+    #[test]
+    fn physaddr_step_rounds_up_and_down() {
+        let startpa = PhysAddr::new(9000); // Should round down to 8192
+        let endpa = PhysAddr::new(5000 * 3); // Should round up to 16384
+        let pas =
+            PhysAddr::step_by_rounded(startpa, endpa, vm::PAGE_SIZE_4K).collect::<Vec<PhysAddr>>();
+        assert_eq!(pas, [PhysAddr::new(4096 * 2), PhysAddr::new(4096 * 3)]);
+    }
+
+    #[test]
+    fn physaddr_step_2m() {
+        let startpa = PhysAddr::new(0x3f000000);
+        let endpa = PhysAddr::new(0x3f000000 + 4 * 1024 * 1024);
+        let pas =
+            PhysAddr::step_by_rounded(startpa, endpa, vm::PAGE_SIZE_2M).collect::<Vec<PhysAddr>>();
+        assert_eq!(pas, [PhysAddr::new(0x3f000000), PhysAddr::new(0x3f000000 + 2 * 1024 * 1024)]);
+    }
+}

aarch64/src/l.S

@@ -45,11 +45,11 @@ SCTLR_EL1			= (SCTLR_EL1_I|SCTLR_EL1_C|SCTLR_EL1_M)
 
 // Preset memory attributes.  This register stores 8 8-bit presets that are
 // referenced by index in the page table entries:
-//  [0] 0x00 - Device (Non-gathering, non-reordering, no early write acknowledgement (most restrictive))
-//  [1] 0xff - Normal
-MAIR_EL1			= 0xff00
-PT_MAIR_DEVICE			= (0<<2)		// Use device memory attributes
-PT_MAIR_NORMAL			= (1<<2)		// Use normal memory attributes
+//  [0] 0xff - Normal
+//  [1] 0x00 - Device (Non-gathering, non-reordering, no early write acknowledgement (most restrictive))
+MAIR_EL1			= 0x00ff
+PT_MAIR_NORMAL			= (0<<2)		// Use normal memory attributes
+PT_MAIR_DEVICE			= (1<<2)		// Use device memory attributes
 
 PT_PAGE				= 3			// 4KiB granule
 PT_BLOCK			= 1			// 2MiB granule

aarch64/src/main.rs

@@ -3,26 +3,36 @@
 #![cfg_attr(not(test), no_main)]
 #![feature(alloc_error_handler)]
 #![feature(asm_const)]
+#![feature(core_intrinsics)]
 #![feature(stdsimd)]
+#![feature(step_trait)]
+#![feature(strict_provenance)]
 #![forbid(unsafe_op_in_unsafe_fn)]
 
 mod devcons;
 mod io;
+mod kalloc;
+mod kmem;
 mod mailbox;
-mod mem;
 mod param;
 mod registers;
 mod trap;
 mod uartmini;
 mod uartpl011;
+mod vm;
 
+use crate::kmem::PhysAddr;
+use crate::vm::kernel_root;
 use core::ffi::c_void;
 use port::fdt::DeviceTree;
 use port::println;
+use vm::PageTable;
 
 #[cfg(not(test))]
 core::arch::global_asm!(include_str!("l.S"));
 
+static mut KPGTBL: PageTable = PageTable::empty();
+
 unsafe fn print_memory_range(name: &str, start: &*const c_void, end: &*const c_void) {
     let start = start as *const _ as u64;
     let end = end as *const _ as u64;
@@ -93,7 +103,18 @@ fn print_board_info() {
 pub extern "C" fn main9(dtb_ptr: u64) {
     trap::init();
 
+    // Parse the DTB before we set up memory so we can correctly map it
     let dt = unsafe { DeviceTree::from_u64(dtb_ptr).unwrap() };
+
+    unsafe {
+        kalloc::free_pages(kmem::early_pages());
+
+        let dtb_phys = PhysAddr::new(dtb_ptr);
+        let edtb_phys = dtb_phys + dt.size() as u64;
+        vm::init(&mut KPGTBL, dtb_phys, edtb_phys);
+        vm::switch(&KPGTBL);
+    }
+
     mailbox::init(&dt);
     devcons::init(&dt);
 
@@ -106,10 +127,12 @@ pub extern "C" fn main9(dtb_ptr: u64) {
     print_physical_memory_map();
     print_board_info();
 
+    // Dump out pagetables
+    kernel_root().print_tables();
+
     println!("looping now");
 
     #[allow(clippy::empty_loop)]
     loop {}
 }
-
 mod runtime;