Initial vmem code (wip)

Signed-off-by: Graham MacDonald <[email protected]>

aarch64/lib/kernel.ld

@@ -51,6 +51,12 @@ SECTIONS {
 	}
 	ebss = .;
 
+	heap = .;
+	.heap : ALIGN(4096) {
+		. = ALIGN(8*2097152);
+	}
+	eheap = .;
+
 	/* Reserve section for early pagetables.  Align to 2MiB to allow us to map
 	   as a 2MiB page.Note that this won't be needed once we transition to
 	   recursive pagetables.

aarch64/src/kmem.rs

@@ -1,36 +1,86 @@
 use crate::param::KZERO;
-use port::mem::{PhysAddr, PhysRange};
+use port::mem::{PhysAddr, PhysRange, VirtRange};
 
 // These map to definitions in kernel.ld
 extern "C" {
+    static boottext: [u64; 0];
+    static eboottext: [u64; 0];
+    static text: [u64; 0];
     static etext: [u64; 0];
+    static rodata: [u64; 0];
     static erodata: [u64; 0];
+    static data: [u64; 0];
+    static edata: [u64; 0];
+    static bss: [u64; 0];
     static ebss: [u64; 0];
     static early_pagetables: [u64; 0];
     static eearly_pagetables: [u64; 0];
+    static heap: [u64; 0];
+    static eheap: [u64; 0];
+    static end: [u64; 0];
 }
 
-pub fn text_addr() -> usize {
-    0xffff_8000_0000_0000
+fn start_addr() -> usize {
+    unsafe { boottext.as_ptr().addr() }
 }
 
-pub fn etext_addr() -> usize {
+fn end_addr() -> usize {
+    unsafe { end.as_ptr().addr() }
+}
+
+fn boottext_addr() -> usize {
+    unsafe { boottext.as_ptr().addr() }
+}
+
+fn eboottext_addr() -> usize {
+    unsafe { eboottext.as_ptr().addr() }
+}
+
+fn text_addr() -> usize {
+    unsafe { text.as_ptr().addr() }
+}
+
+fn etext_addr() -> usize {
     unsafe { etext.as_ptr().addr() }
 }
 
-pub fn erodata_addr() -> usize {
+fn rodata_addr() -> usize {
+    unsafe { rodata.as_ptr().addr() }
+}
+
+fn erodata_addr() -> usize {
     unsafe { erodata.as_ptr().addr() }
 }
 
-pub fn ebss_addr() -> usize {
+fn data_addr() -> usize {
+    unsafe { data.as_ptr().addr() }
+}
+
+fn edata_addr() -> usize {
+    unsafe { edata.as_ptr().addr() }
+}
+
+fn bss_addr() -> usize {
+    unsafe { bss.as_ptr().addr() }
+}
+
+fn ebss_addr() -> usize {
     unsafe { ebss.as_ptr().addr() }
 }
 
-pub fn early_pagetables_addr() -> usize {
+fn heap_addr() -> usize {
+    unsafe { heap.as_ptr().addr() }
+}
+
+fn eheap_addr() -> usize {
+    unsafe { eheap.as_ptr().addr() }
+}
+
+fn early_pagetables_addr() -> usize {
     unsafe { early_pagetables.as_ptr().addr() }
 }
 
-pub fn eearly_pagetables_addr() -> usize {
+fn eearly_pagetables_addr() -> usize {
     unsafe { eearly_pagetables.as_ptr().addr() }
 }
 
@@ -50,7 +100,51 @@ pub fn from_ptr_to_physaddr<T>(a: *const T) -> PhysAddr {
     from_virt_to_physaddr(a.addr())
 }
 
-pub fn early_pages_range() -> PhysRange {
+pub fn kernel_text_physrange() -> PhysRange {
+    PhysRange(from_virt_to_physaddr(text_addr())..from_virt_to_physaddr(etext_addr()))
+}
+
+pub fn kernel_data_physrange() -> PhysRange {
+    PhysRange::with_len(from_virt_to_physaddr(etext_addr()).addr(), erodata_addr() - etext_addr())
+}
+
+pub fn kernel_bss_physrange() -> PhysRange {
+    PhysRange::with_len(from_virt_to_physaddr(erodata_addr()).addr(), ebss_addr() - erodata_addr())
+}
+
+pub fn kernel_heap_physrange() -> PhysRange {
+    PhysRange::with_len(from_virt_to_physaddr(heap_addr()).addr(), eheap_addr() - heap_addr())
+}
+
+pub fn total_virtrange() -> VirtRange {
+    VirtRange(start_addr()..end_addr())
+}
+
+pub fn boottext_virtrange() -> VirtRange {
+    VirtRange(boottext_addr()..eboottext_addr())
+}
+
+pub fn text_virtrange() -> VirtRange {
+    VirtRange(text_addr()..etext_addr())
+}
+
+pub fn rodata_virtrange() -> VirtRange {
+    VirtRange(rodata_addr()..erodata_addr())
+}
+
+pub fn data_virtrange() -> VirtRange {
+    VirtRange(data_addr()..edata_addr())
+}
+
+pub fn bss_virtrange() -> VirtRange {
+    VirtRange(bss_addr()..ebss_addr())
+}
+
+pub fn heap_virtrange() -> VirtRange {
+    VirtRange::with_len(heap_addr(), eheap_addr() - heap_addr())
+}
+
+pub fn early_pages_physrange() -> PhysRange {
     PhysRange::new(
         from_virt_to_physaddr(early_pagetables_addr()),
         from_virt_to_physaddr(eearly_pagetables_addr()),

aarch64/src/main.rs

@@ -19,13 +19,13 @@ mod trap;
 mod uartmini;
 mod uartpl011;
 mod vm;
+mod vmalloc;
 
 use crate::kmem::from_virt_to_physaddr;
 use crate::vm::kernel_root;
-use core::ffi::c_void;
 use core::ptr;
 use port::fdt::DeviceTree;
-use port::mem::PhysRange;
+use port::mem::{PhysRange, VirtRange};
 use port::println;
 use vm::PageTable;
 
@@ -34,35 +34,23 @@ core::arch::global_asm!(include_str!("l.S"));
 
 static mut KPGTBL: PageTable = PageTable::empty();
 
-unsafe fn print_memory_range(name: &str, start: &*const c_void, end: &*const c_void) {
-    let start = start as *const _ as u64;
-    let end = end as *const _ as u64;
-    let size = end - start;
+unsafe fn print_memory_range(name: &str, range: VirtRange) {
+    let start = range.start();
+    let end = range.end();
+    let size = range.size();
     println!("  {name}{start:#x}..{end:#x} ({size:#x})");
 }
 
 fn print_binary_sections() {
-    extern "C" {
-        static boottext: *const c_void;
-        static eboottext: *const c_void;
-        static text: *const c_void;
-        static etext: *const c_void;
-        static rodata: *const c_void;
-        static erodata: *const c_void;
-        static data: *const c_void;
-        static edata: *const c_void;
-        static bss: *const c_void;
-        static end: *const c_void;
-    }
-
     println!("Binary sections:");
     unsafe {
-        print_memory_range("boottext:\t", &boottext, &eboottext);
-        print_memory_range("text:\t\t", &text, &etext);
-        print_memory_range("rodata:\t", &rodata, &erodata);
-        print_memory_range("data:\t\t", &data, &edata);
-        print_memory_range("bss:\t\t", &bss, &end);
-        print_memory_range("total:\t", &boottext, &end);
+        print_memory_range("boottext:\t", kmem::boottext_virtrange());
+        print_memory_range("text:\t\t", kmem::text_virtrange());
+        print_memory_range("rodata:\t", kmem::rodata_virtrange());
+        print_memory_range("data:\t\t", kmem::data_virtrange());
+        print_memory_range("bss:\t\t", kmem::bss_virtrange());
+        print_memory_range("heap:\t\t", kmem::heap_virtrange());
+        print_memory_range("total:\t", kmem::total_virtrange());
     }
 }
 
@@ -73,7 +61,7 @@ fn print_memory_info() {
     let vc_mem = mailbox::get_vc_memory();
     println!("  Video:\t{vc_mem} ({:#x})", vc_mem.size());
 
-    println!("Memory usage::");
+    println!("Memory usage:");
     let (used, total) = pagealloc::usage_bytes();
     println!("  Used:\t\t{used:#016x}");
     println!("  Total:\t{total:#016x}");
@@ -140,6 +128,13 @@ pub extern "C" fn main9(dtb_va: usize) {
 
     println!("looping now");
 
+    {
+        let test = vmalloc::alloc(1024);
+        println!("test alloc: {:p}", test);
+        let test2 = vmalloc::alloc(1024);
+        println!("test alloc: {:p}", test2);
+    }
+
     #[allow(clippy::empty_loop)]
     loop {}
 }

aarch64/src/pagealloc.rs

@@ -33,7 +33,7 @@ pub fn init_page_allocator() {
     let mut lock = PAGE_ALLOC.lock(&node);
     let page_alloc = &mut *lock;
 
-    let early_pages_range = kmem::early_pages_range();
+    let early_pages_range = kmem::early_pages_physrange();
     if let Err(err) = page_alloc.mark_free(&early_pages_range) {
         panic!("Couldn't mark early pages free: range: {} err: {:?}", early_pages_range, err);
     }

aarch64/src/runtime.rs

@@ -40,9 +40,9 @@ fn oom(_layout: Layout) -> ! {
     panic!("oom");
 }
 
-struct FakeAlloc;
+struct VmemAllocator;
 
-unsafe impl GlobalAlloc for FakeAlloc {
+unsafe impl GlobalAlloc for VmemAllocator {
     unsafe fn alloc(&self, _layout: Layout) -> *mut u8 {
         panic!("fake alloc");
     }
@@ -52,4 +52,4 @@ unsafe impl GlobalAlloc for FakeAlloc {
 }
 
 #[global_allocator]
-static FAKE_ALLOCATOR: FakeAlloc = FakeAlloc {};
+static VMEM_ALLOCATOR: VmemAllocator = VmemAllocator {};

aarch64/src/vm.rs

@@ -2,11 +2,12 @@
 
 use crate::{
     kmem::{
-        ebss_addr, erodata_addr, etext_addr, from_ptr_to_physaddr, from_virt_to_physaddr,
-        physaddr_as_ptr_mut, physaddr_as_virt, text_addr,
+        from_ptr_to_physaddr, heap_virtrange, kernel_bss_physrange, kernel_data_physrange,
+        kernel_heap_physrange, kernel_text_physrange, physaddr_as_ptr_mut, physaddr_as_virt,
     },
     pagealloc,
     registers::rpi_mmio,
+    vmalloc,
 };
 use bitstruct::bitstruct;
 use core::fmt;
@@ -468,6 +469,7 @@ fn print_pte(indent: usize, i: usize, level: Level, pte: Entry) {
 
 pub unsafe fn init(kpage_table: &mut PageTable, dtb_range: PhysRange, available_mem: PhysRange) {
     pagealloc::init_page_allocator();
+    vmalloc::init(heap_virtrange());
 
     // We use recursive page tables, but we have to be careful in the init call,
     // since the kpage_table is not currently pointed to by ttbr1_el1.  Any
@@ -485,24 +487,14 @@ pub unsafe fn init(kpage_table: &mut PageTable, dtb_range: PhysRange, available_
 
     // TODO leave the first page unmapped to catch null pointer dereferences in unsafe code
     let custom_map = {
-        let text_range =
-            PhysRange(from_virt_to_physaddr(text_addr())..from_virt_to_physaddr(etext_addr()));
-        let data_range = PhysRange::with_len(
-            from_virt_to_physaddr(etext_addr()).addr(),
-            erodata_addr() - etext_addr(),
-        );
-        let bss_range = PhysRange::with_len(
-            from_virt_to_physaddr(erodata_addr()).addr(),
-            ebss_addr() - erodata_addr(),
-        );
-
         let mmio_range = rpi_mmio().expect("mmio base detect failed");
 
         let mut map = [
             ("DTB", dtb_range, Entry::ro_kernel_data(), PageSize::Page4K),
-            ("Kernel Text", text_range, Entry::ro_kernel_text(), PageSize::Page2M),
-            ("Kernel Data", data_range, Entry::ro_kernel_data(), PageSize::Page2M),
-            ("Kernel BSS", bss_range, Entry::rw_kernel_data(), PageSize::Page2M),
+            ("Kernel Text", kernel_text_physrange(), Entry::ro_kernel_text(), PageSize::Page2M),
+            ("Kernel Data", kernel_data_physrange(), Entry::ro_kernel_data(), PageSize::Page2M),
+            ("Kernel BSS", kernel_bss_physrange(), Entry::rw_kernel_data(), PageSize::Page2M),
+            ("Kernel Heap", kernel_heap_physrange(), Entry::rw_kernel_data(), PageSize::Page2M),
             ("MMIO", mmio_range, Entry::ro_kernel_device(), PageSize::Page2M),
         ];
         map.sort_by_key(|a| a.1.start());

aarch64/src/vmalloc.rs

@@ -0,0 +1,54 @@
+use core::{mem::MaybeUninit, ptr::addr_of};
+
+use port::{
+    mcslock::{Lock, LockNode},
+    mem::VirtRange,
+    vmem::{Arena, Boundary},
+};
+
+static VMALLOC: Lock<Option<&'static mut VmAlloc>> = Lock::new("vmalloc", None);
+
+static mut EARLY_TAGS_PAGE: [u8; 4096] = [0; 4096];
+
+struct VmAlloc {
+    heap_arena: Arena,
+}
+
+impl VmAlloc {
+    fn new(heap_range: VirtRange) -> Self {
+        let quantum = 4096;
+
+        let early_tags_ptr = unsafe { addr_of!(EARLY_TAGS_PAGE) as usize };
+        let early_tags_size = unsafe { EARLY_TAGS_PAGE.len() };
+        let early_tags_range = VirtRange::with_len(early_tags_ptr, early_tags_size);
+
+        Self {
+            heap_arena: Arena::new_with_static_range(
+                "heap",
+                Some(Boundary::from(heap_range)),
+                quantum,
+                None,
+                early_tags_range,
+            ),
+        }
+    }
+}
+
+pub fn init(heap_range: VirtRange) {
+    let node = LockNode::new();
+    let mut vmalloc = VMALLOC.lock(&node);
+    *vmalloc = Some({
+        static mut MAYBE_VMALLOC: MaybeUninit<VmAlloc> = MaybeUninit::uninit();
+        unsafe {
+            MAYBE_VMALLOC.write(VmAlloc::new(heap_range));
+            MAYBE_VMALLOC.assume_init_mut()
+        }
+    });
+}
+
+pub fn alloc(size: usize) -> *mut u8 {
+    let node = LockNode::new();
+    let mut lock = VMALLOC.lock(&node);
+    let vmalloc = lock.as_deref_mut().unwrap();
+    vmalloc.heap_arena.alloc(size)
+}

port/src/lib.rs

@@ -10,3 +10,4 @@ pub mod devcons;
 pub mod fdt;
 pub mod mcslock;
 pub mod mem;
+pub mod vmem;