From b9a26cd274813545187d0b13bb03ebca15c2be48 Mon Sep 17 00:00:00 2001 From: Graham MacDonald Date: Sat, 20 Jul 2024 17:23:24 +0100 Subject: [PATCH] wip Signed-off-by: Graham MacDonald --- aarch64/src/vmalloc.rs | 6 +- port/src/vmem.rs | 254 ++++++++++++++++++++++++++--------------- rust-toolchain.toml | 2 +- 3 files changed, 165 insertions(+), 97 deletions(-) diff --git a/aarch64/src/vmalloc.rs b/aarch64/src/vmalloc.rs index 2e89d47..f978f12 100644 --- a/aarch64/src/vmalloc.rs +++ b/aarch64/src/vmalloc.rs @@ -3,7 +3,7 @@ use core::{mem::MaybeUninit, ptr::addr_of}; use port::{ mcslock::{Lock, LockNode}, mem::VirtRange, - vmem::Arena, + vmem::{Arena, Boundary}, }; static VMALLOC: Lock> = Lock::new("vmalloc", None); @@ -25,9 +25,9 @@ impl VmAlloc { Self { heap_arena: Arena::new_with_static_range( "heap", - heap_range.start(), - heap_range.size(), + Some(Boundary::from(heap_range)), quantum, + None, early_tags_range, ), } diff --git a/port/src/vmem.rs b/port/src/vmem.rs index addbd84..f7b2bd5 100644 --- a/port/src/vmem.rs +++ b/port/src/vmem.rs @@ -1,11 +1,14 @@ -use core::{ptr::null_mut, slice}; +use core::{ops::Range, ptr::null_mut, slice}; use crate::mem::VirtRange; #[cfg(not(test))] use crate::println; -// TODO reserve recursive area in vmem +// TODO reserve recursive area in vmem(?) +// TODO Add hashtable for allocated tags - makes it faster when freeing, given only an address. +// TODO Add support for quantum caches once we have slab allocators implemented. +// TODO Add power-of-two freelists for freed allocations. #[derive(Debug, PartialEq)] pub enum BoundaryError { @@ -22,7 +25,7 @@ pub enum AllocError { type BoundaryResult = core::result::Result; #[derive(Clone, Copy, Debug, PartialEq)] -struct Boundary { +pub struct Boundary { start: usize, size: usize, } @@ -56,6 +59,18 @@ impl Boundary { } } +impl From for Boundary { + fn from(r: VirtRange) -> Self { + Boundary::new_unchecked(r.start(), r.size()) + } +} + +impl From> for Boundary { + fn from(r: Range) -> Self { + Boundary::new_unchecked(r.start, r.end - r.start) + } +} + #[derive(Clone, Copy, Debug, PartialEq)] enum TagType { Allocated, @@ -70,22 +85,21 @@ struct Tag { } impl Tag { - #[allow(dead_code)] fn new(tag_type: TagType, boundary: Boundary) -> Self { Self { tag_type, boundary } } #[cfg(test)] fn new_allocated(boundary: Boundary) -> Self { - Self { tag_type: TagType::Allocated, boundary } + Tag::new(TagType::Allocated, boundary) } fn new_free(boundary: Boundary) -> Self { - Self { tag_type: TagType::Free, boundary } + Tag::new(TagType::Free, boundary) } fn new_span(boundary: Boundary) -> Self { - Self { tag_type: TagType::Span, boundary } + Tag::new(TagType::Span, boundary) } } @@ -115,34 +129,22 @@ impl TagItem { fn new_allocated(boundary: Boundary) -> Self { Self { tag: Tag::new_allocated(boundary), next: null_mut(), prev: null_mut() } } - - // #[cfg(test)] - // fn new_free(boundary: Boundary) -> Self { - // Self { tag: Tag::new_free(boundary), next: null_mut(), prev: null_mut() } - // } - - // #[cfg(test)] - // fn new_span(boundary: Boundary) -> Self { - // Self { tag: Tag::new_span(boundary), next: null_mut(), prev: null_mut() } - // } - - fn clear_links(&mut self) { - self.next = null_mut(); - self.prev = null_mut(); - } } -/// Stack of tags, useful for freelist -struct TagStack { +/// Pool of boundary tags. Vmem uses external boundary tags. We allocate a page +/// of tags at a time, making them available via this pool. This allows us to +/// set up the pool initially with a static page, before we have any kind of +/// allocator. The pool can later be populated dynamically. +struct TagPool { tags: *mut TagItem, } -impl TagStack { +impl TagPool { fn new() -> Self { Self { tags: null_mut() } } - fn push(&mut self, tag: &mut TagItem) { + fn add(&mut self, tag: &mut TagItem) { if self.tags.is_null() { self.tags = tag; } else { @@ -152,14 +154,16 @@ impl TagStack { } } - fn pop(&mut self) -> *mut TagItem { - if let Some(tag) = unsafe { self.tags.as_mut() } { - self.tags = tag.next; + fn take(&mut self, tag: Tag) -> *mut TagItem { + if let Some(tag_item) = unsafe { self.tags.as_mut() } { + self.tags = tag_item.next; if let Some(next_tag) = unsafe { self.tags.as_mut() } { next_tag.prev = null_mut(); } - tag.clear_links(); - tag as *mut TagItem + tag_item.next = null_mut(); + tag_item.prev = null_mut(); + tag_item.tag = tag; + tag_item as *mut TagItem } else { null_mut() } @@ -178,6 +182,7 @@ impl TagStack { } /// Ordered list of tags (by Tag::start) +/// This is a simple linked list that assumes no overlaps. struct TagList { tags: *mut TagItem, } @@ -187,7 +192,6 @@ impl TagList { Self { tags: null_mut() } } - // ATM this is a simple linked list that assumes no overlaps. fn push(&mut self, new_tag: &mut TagItem) { if self.tags.is_null() { self.tags = new_tag; @@ -283,18 +287,18 @@ impl TagList { pub struct Arena { _name: &'static str, quantum: usize, - used_tags: TagList, - free_tags: TagStack, - // TODO Add hashtable for allocated tags - makes it faster when freeing, given only an address + + tag_pool: TagPool, // Pool of available tags + segment_list: TagList, // List of all segments in address order } impl Arena { /// Only to be used for creation of initial heap pub fn new_with_static_range( name: &'static str, - base: usize, - size: usize, + initial_span: Option, quantum: usize, + parent: Option, static_range: VirtRange, ) -> Self { let tags_addr = unsafe { &mut *(static_range.start() as *mut TagItem) }; @@ -303,61 +307,60 @@ impl Arena { }; println!( - "Arena::new_with_static_range name:{} base:{:x} size:{:x} quantum:{:x}", - name, base, size, quantum + "Arena::new_with_static_range name:{} initial_span:{:?} quantum:{:x}", + name, initial_span, quantum ); - Self::new_with_tags(name, base, size, quantum, tags) + Self::new_with_tags(name, initial_span, quantum, parent, tags) } /// Create a new arena, assuming there is no dynamic allocation available, /// and all free tags come from the free_tags provided. fn new_with_tags( name: &'static str, - base: usize, - size: usize, + initial_span: Option, quantum: usize, + _parent: Option, free_tags: &mut [TagItem], ) -> Self { - assert_eq!(base % quantum, 0); - assert_eq!(size % quantum, 0); - assert!(base.checked_add(size).is_some()); - let mut arena = Self { _name: name, quantum: quantum, - used_tags: TagList::new(), - free_tags: TagStack::new(), + segment_list: TagList::new(), + tag_pool: TagPool::new(), }; - arena.add_free_tags(free_tags); - arena.add_span(base, size); + arena.add_tags_to_pool(free_tags); + + if let Some(span) = initial_span { + assert_eq!(span.start % quantum, 0); + assert_eq!(span.size % quantum, 0); + assert!(span.start.checked_add(span.size).is_some()); + arena.add_free_span(span); + } + arena } - pub fn add_span(&mut self, base: usize, size: usize) { - self.used_tags.push({ - let item = unsafe { self.free_tags.pop().as_mut().expect("no free tags") }; - item.tag = Tag::new_span(Boundary::new_unchecked(base, size)); - item + fn add_free_span(&mut self, boundary: Boundary) { + self.segment_list.push(unsafe { + self.tag_pool.take(Tag::new_span(boundary)).as_mut().expect("no free tags") }); - self.used_tags.push({ - let item = unsafe { self.free_tags.pop().as_mut().expect("no free tags") }; - item.tag = Tag::new_free(Boundary::new_unchecked(base, size)); - item + self.segment_list.push(unsafe { + self.tag_pool.take(Tag::new_free(boundary)).as_mut().expect("no free tags") }); } - fn add_free_tags(&mut self, tags: &mut [TagItem]) { + fn add_tags_to_pool(&mut self, tags: &mut [TagItem]) { for tag in tags { - tag.clear_links(); - self.free_tags.push(tag); + tag.next = null_mut(); + tag.prev = null_mut(); + self.tag_pool.add(tag); } } pub fn alloc(&mut self, size: usize) -> *mut u8 { let boundary = self.alloc_segment(size); if boundary.is_ok() { - // TODO Register in allocation hashtable boundary.unwrap().start as *mut u8 } else { null_mut() @@ -365,7 +368,6 @@ impl Arena { } pub fn free(&mut self, addr: *mut u8) { - // TODO Look up in allocation hashtable let _ = self.free_segment(addr as usize); } @@ -384,7 +386,7 @@ impl Arena { }; // Find the first free tag that's large enough - let mut curr_item = self.used_tags.tags; + let mut curr_item = self.segment_list.tags; while let Some(item) = unsafe { curr_item.as_mut() } { if item.tag.tag_type == TagType::Free && item.tag.boundary.size >= size { // Mark this tag as allocated, and if there's any left over space, @@ -396,11 +398,12 @@ impl Arena { let remainder = item.tag.boundary.size - size; item.tag.boundary.size = size; - let new_item = unsafe { self.free_tags.pop().as_mut().expect("no free tags") }; - new_item.tag = Tag::new_free(Boundary::new_unchecked( + let new_tag = Tag::new_free(Boundary::new_unchecked( item.tag.boundary.start + size, remainder, )); + let new_item = + unsafe { self.tag_pool.take(new_tag).as_mut().expect("no free tags") }; // Insert new_item after item new_item.next = item.next; @@ -420,10 +423,9 @@ impl Arena { // Free addr. We don't need to know size because we don't merge allocations. // (We only merge freed segments) // TODO Error on precondition fail - // TODO Use hashmap if available fn free_segment(&mut self, addr: usize) -> Result<(), AllocError> { // Need to manually scan the used tags - let mut curr_item = self.used_tags.tags; + let mut curr_item = self.segment_list.tags; while let Some(item) = unsafe { curr_item.as_mut() } { if item.tag.boundary.start == addr && item.tag.tag_type == TagType::Allocated { break; @@ -460,7 +462,7 @@ impl Arena { next.tag.boundary.start = curr_tag.tag.boundary.start; next.tag.boundary.size += curr_tag.tag.boundary.size; TagList::unlink(&mut curr_tag); - self.free_tags.push(&mut curr_tag); + self.tag_pool.add(&mut curr_tag); } (Some(TagType::Free), None) | (Some(TagType::Free), Some(TagType::Span)) @@ -470,7 +472,7 @@ impl Arena { let prev = unsafe { curr_tag.prev.as_mut() }.unwrap(); prev.tag.boundary.size += curr_tag.tag.boundary.size; TagList::unlink(&mut curr_tag); - self.free_tags.push(&mut curr_tag); + self.tag_pool.add(&mut curr_tag); } (Some(TagType::Free), Some(TagType::Free)) => { // Prev and next both free @@ -480,8 +482,8 @@ impl Arena { prev.tag.boundary.size += curr_tag.tag.boundary.size + next.tag.boundary.size; TagList::unlink(&mut curr_tag); TagList::unlink(&mut next); - self.free_tags.push(&mut curr_tag); - self.free_tags.push(&mut next); + self.tag_pool.add(&mut curr_tag); + self.tag_pool.add(&mut next); } (None, None) | (None, Some(TagType::Span)) @@ -496,13 +498,13 @@ impl Arena { } fn tags_iter(&self) -> impl Iterator + '_ { - self.used_tags.tags_iter() + self.segment_list.tags_iter() } /// Checks that all invariants are correct. fn assert_tags_are_consistent(&self) { // There must be at least 2 tags - debug_assert!(self.used_tags.len() >= 2); + debug_assert!(self.segment_list.len() >= 2); // Tags must be in order, without gaps let mut last_tag: Option = None; @@ -611,11 +613,11 @@ mod tests { let mut page = Page4K([0; 4096]); const NUM_TAGS: usize = size_of::() / size_of::(); let tags = unsafe { &mut *(&mut page as *mut Page4K as *mut [TagItem; NUM_TAGS]) }; - let mut tag_stack = TagStack::new(); + let mut tag_stack = TagPool::new(); assert_eq!(tag_stack.len(), 0); for tag in tags { - tag_stack.push(tag); + tag_stack.add(tag); } assert_eq!(tag_stack.len(), NUM_TAGS); } @@ -676,14 +678,14 @@ mod tests { fn create_arena_with_static_tags( name: &'static str, - base: usize, - size: usize, + initial_span: Option, quantum: usize, + _parent_arena: Option<&mut Arena>, ) -> Arena { let mut page = Page4K([0; 4096]); const NUM_TAGS: usize = size_of::() / size_of::(); let tags = unsafe { &mut *(&mut page as *mut Page4K as *mut [TagItem; NUM_TAGS]) }; - Arena::new_with_tags(name, base, size, quantum, tags) + Arena::new_with_tags(name, initial_span, quantum, None, tags) } fn assert_tags_eq(arena: &Arena, expected: &[Tag]) { @@ -694,8 +696,13 @@ mod tests { #[test] fn test_arena_create() { - let arena = create_arena_with_static_tags("test", 4096, 4096 * 20, 4096); - assert_eq!(arena.free_tags.len(), 100); + let arena = create_arena_with_static_tags( + "test", + Some(Boundary::new_unchecked(4096, 4096 * 20)), + 4096, + None, + ); + assert_eq!(arena.tag_pool.len(), 100); assert_tags_eq( &arena, @@ -708,7 +715,12 @@ mod tests { #[test] fn test_arena_alloc() { - let mut arena = create_arena_with_static_tags("test", 4096, 4096 * 20, 4096); + let mut arena = create_arena_with_static_tags( + "test", + Some(Boundary::new_unchecked(4096, 4096 * 20)), + 4096, + None, + ); arena.alloc(4096 * 2); @@ -724,15 +736,25 @@ mod tests { #[test] fn test_arena_alloc_rounds_if_wrong_granule() { - let mut arena = create_arena_with_static_tags("test", 4096, 4096 * 20, 4096); + let mut arena = create_arena_with_static_tags( + "test", + Some(Boundary::new_unchecked(4096, 4096 * 20)), + 4096, + None, + ); let a = arena.alloc_segment(1024); assert_eq!(a.unwrap().size, 4096); } #[test] fn test_arena_free() { - let mut arena = create_arena_with_static_tags("test", 4096, 4096 * 20, 4096); - assert_eq!(arena.free_tags.len(), 100); + let mut arena = create_arena_with_static_tags( + "test", + Some(Boundary::new_unchecked(4096, 4096 * 20)), + 4096, + None, + ); + assert_eq!(arena.tag_pool.len(), 100); // We need to test each case where we're freeing by scanning the tags linearly. // To do this we run through each case (comments from the `free` function) @@ -740,7 +762,7 @@ mod tests { // Prev and next both non-free let a1 = arena.alloc(4096); let a2 = arena.alloc(4096); - assert_eq!(arena.free_tags.len(), 98); + assert_eq!(arena.tag_pool.len(), 98); assert_tags_eq( &arena, &[ @@ -751,7 +773,7 @@ mod tests { ], ); arena.free(a1); - assert_eq!(arena.free_tags.len(), 98); + assert_eq!(arena.tag_pool.len(), 98); assert_tags_eq( &arena, &[ @@ -764,7 +786,7 @@ mod tests { // Prev and next both free arena.free(a2); - assert_eq!(arena.free_tags.len(), 100); + assert_eq!(arena.tag_pool.len(), 100); assert_tags_eq( &arena, &[ @@ -778,7 +800,7 @@ mod tests { let a2 = arena.alloc(4096); let a3 = arena.alloc(4096); arena.free(a1); - assert_eq!(arena.free_tags.len(), 97); + assert_eq!(arena.tag_pool.len(), 97); assert_tags_eq( &arena, &[ @@ -790,7 +812,7 @@ mod tests { ], ); arena.free(a2); - assert_eq!(arena.free_tags.len(), 98); + assert_eq!(arena.tag_pool.len(), 98); assert_tags_eq( &arena, &[ @@ -804,7 +826,7 @@ mod tests { // Prev non-free, next free arena.free(a3); let a1 = arena.alloc(4096); - assert_eq!(arena.free_tags.len(), 99); + assert_eq!(arena.tag_pool.len(), 99); assert_tags_eq( &arena, &[ @@ -814,7 +836,7 @@ mod tests { ], ); arena.free(a1); - assert_eq!(arena.free_tags.len(), 100); + assert_eq!(arena.tag_pool.len(), 100); assert_tags_eq( &arena, &[ @@ -823,4 +845,50 @@ mod tests { ], ); } + + #[test] + fn test_arena_nesting() { + // Create a page of tags we can share amongst the first arenas + let mut page = Page4K([0; 4096]); + const NUM_TAGS: usize = size_of::() / size_of::(); + let all_tags = unsafe { &mut *(&mut page as *mut Page4K as *mut [TagItem; NUM_TAGS]) }; + + const NUM_ARENAS: usize = 4; + const NUM_TAGS_PER_ARENA: usize = NUM_TAGS / NUM_ARENAS; + let (arena1_tags, all_tags) = all_tags.split_at_mut(NUM_TAGS_PER_ARENA); + let (arena2_tags, all_tags) = all_tags.split_at_mut(NUM_TAGS_PER_ARENA); + let (arena3a_tags, all_tags) = all_tags.split_at_mut(NUM_TAGS_PER_ARENA); + let (arena3b_tags, _) = all_tags.split_at_mut(NUM_TAGS_PER_ARENA); + + let mut arena1 = Arena::new_with_tags( + "arena1", + Some(Boundary::new_unchecked(4096, 4096 * 20)), + 4096, + None, + arena1_tags, + ); + + // Import all + let mut arena2 = Arena::new_with_tags("arena2", None, 4096, None, arena2_tags); + + // Import first half + let mut arena3a = Arena::new_with_tags( + "arena3a", + Some(Boundary::from(4096..4096 * 10)), + 4096, + None, + arena3a_tags, + ); + + // Import second half + let mut arena3b = Arena::new_with_tags( + "arena3b", + Some(Boundary::from(4096 * 10..4096 * 21)), + 4096, + None, + arena3b_tags, + ); + + // Let's do some allocations + } } diff --git a/rust-toolchain.toml b/rust-toolchain.toml index edc3dd8..cdd3635 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,5 +1,5 @@ [toolchain] -channel = "nightly-2024-06-08" +channel = "nightly-2024-07-01" components = [ "rustfmt", "rust-src", "clippy", "llvm-tools" ] targets = [ "aarch64-unknown-none",