diff --git a/examples/msaa.rs b/examples/msaa.rs index 14bf40d..1564344 100644 --- a/examples/msaa.rs +++ b/examples/msaa.rs @@ -24,7 +24,7 @@ fn main() -> anyhow::Result<()> { let mesh_msaa_pipeline = create_mesh_pipeline(&event_loop.device, sample_count)?; let mesh_noaa_pipeline = create_mesh_pipeline(&event_loop.device, SampleCount::X1)?; let cube_mesh = load_cube_mesh(&event_loop.device)?; - let mut pool = HashPool::new(&event_loop.device); + let mut pool = FifoPool::new(&event_loop.device); let mut angle = 0f32; diff --git a/examples/vsm_omni.rs b/examples/vsm_omni.rs index 835d2aa..d271e40 100644 --- a/examples/vsm_omni.rs +++ b/examples/vsm_omni.rs @@ -89,7 +89,7 @@ fn main() -> anyhow::Result<()> { }?; // A pool will be used for per-frame resources - let mut pool = LazyPool::new(&event_loop.device); + let mut pool = FifoPool::new(&event_loop.device); let mut elapsed = 0.0; event_loop.run(|frame| { diff --git a/src/lib.rs b/src/lib.rs index b62b287..476480f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -385,7 +385,10 @@ pub mod prelude { input::{ update_input, update_keyboard, update_mouse, KeyBuf, KeyMap, MouseBuf, MouseButton, }, - pool::{hash::HashPool, lazy::LazyPool, Lease, Pool}, + pool::{ + fifo::FifoPool, hash::HashPool, lazy::LazyPool, Lease, Pool, PoolInfo, + PoolInfoBuilder, + }, }, ash::vk, log::{debug, error, info, logger, trace, warn}, // Everyone wants a log diff --git a/src/pool/fifo.rs b/src/pool/fifo.rs new file mode 100644 index 0000000..ae2efdc --- /dev/null +++ b/src/pool/fifo.rs @@ -0,0 +1,289 @@ +//! Pool which leases from a single bucket per resource type. + +use { + super::{can_lease_command_buffer, Cache, Lease, Pool, PoolInfo}, + crate::driver::{ + accel_struct::{AccelerationStructure, AccelerationStructureInfo}, + buffer::{Buffer, BufferInfo}, + device::Device, + image::{Image, ImageInfo}, + CommandBuffer, CommandBufferInfo, DescriptorPool, DescriptorPoolInfo, DriverError, + RenderPass, RenderPassInfo, + }, + log::debug, + std::{collections::HashMap, sync::Arc}, +}; + +/// A memory-efficient resource allocator. +/// +/// The information for each lease request is compared against the stored resources for +/// compatibility. If no acceptable resources are stored for the information provided a new resource +/// is created and returned. +/// +/// # Details +/// +/// * Acceleration structures may be larger than requested +/// * Buffers may be larger than requested or have additional usage flags +/// * Images may have additional usage flags +/// +/// # Bucket Strategy +/// +/// All resources are stored in a single bucket per resource type, regardless of their individual +/// attributes. +/// +/// In practice this means that for a [`PoolInfo::image_capacity`] of `4`, a maximum of `4` images +/// will be stored. Requests to lease an image or other resource will first look for a compatible +/// resource in the bucket and create a new resource as needed. +/// +/// # Memory Management +/// +/// The single-bucket strategy means that there will always be a reasonable and predictable number +/// of stored resources, however you may call [`FifoPool::clear`] or the other memory management +/// functions at any time to discard stored resources. +#[derive(Debug)] +pub struct FifoPool { + accel_struct_cache: Cache, + buffer_cache: Cache, + command_buffer_cache: HashMap>, + descriptor_pool_cache: Cache, + device: Arc, + image_cache: Cache, + info: PoolInfo, + render_pass_cache: HashMap>, +} + +impl FifoPool { + /// Constructs a new `FifoPool`. + pub fn new(device: &Arc) -> Self { + Self::with_capacity(device, PoolInfo::default()) + } + + /// Constructs a new `FifoPool` with the given capacity information. + pub fn with_capacity(device: &Arc, info: impl Into) -> Self { + let info: PoolInfo = info.into(); + let device = Arc::clone(device); + + Self { + accel_struct_cache: PoolInfo::explicit_cache(info.accel_struct_capacity), + buffer_cache: PoolInfo::explicit_cache(info.buffer_capacity), + command_buffer_cache: Default::default(), + descriptor_pool_cache: PoolInfo::default_cache(), + device, + image_cache: PoolInfo::explicit_cache(info.image_capacity), + info, + render_pass_cache: Default::default(), + } + } + + /// Clears the pool, removing all resources. + pub fn clear(&mut self) { + self.clear_accel_structs(); + self.clear_buffers(); + self.clear_images(); + } + + /// Clears the pool of acceleration structure resources. + pub fn clear_accel_structs(&mut self) { + self.accel_struct_cache = PoolInfo::explicit_cache(self.info.accel_struct_capacity); + } + + /// Clears the pool of buffer resources. + pub fn clear_buffers(&mut self) { + self.buffer_cache = PoolInfo::explicit_cache(self.info.buffer_capacity); + } + + /// Clears the pool of image resources. + pub fn clear_images(&mut self) { + self.image_cache = PoolInfo::explicit_cache(self.info.image_capacity); + } +} + +impl Pool for FifoPool { + #[profiling::function] + fn lease( + &mut self, + info: AccelerationStructureInfo, + ) -> Result, DriverError> { + let cache_ref = Arc::downgrade(&self.accel_struct_cache); + let mut cache = self.accel_struct_cache.lock(); + + { + profiling::scope!("Check cache"); + + // Look for a compatible acceleration structure (big enough and same type) + for idx in 0..cache.len() { + let item = &cache[idx]; + if item.info.size >= info.size && item.info.ty == info.ty { + let item = cache.remove(idx).unwrap(); + + return Ok(Lease::new(cache_ref, item)); + } + } + } + + debug!("Creating new {}", stringify!(AccelerationStructure)); + + let item = AccelerationStructure::create(&self.device, info)?; + + Ok(Lease::new(cache_ref, item)) + } +} + +impl Pool for FifoPool { + #[profiling::function] + fn lease(&mut self, info: BufferInfo) -> Result, DriverError> { + let cache_ref = Arc::downgrade(&self.buffer_cache); + let mut cache = self.buffer_cache.lock(); + + { + profiling::scope!("Check cache"); + + // Look for a compatible buffer (compatible alignment, same mapping mode, big enough and + // superset of usage flags) + for idx in 0..cache.len() { + let item = &cache[idx]; + if item.info.alignment >= info.alignment + && item.info.can_map == info.can_map + && item.info.size >= info.size + && item.info.usage.contains(info.usage) + { + let item = cache.remove(idx).unwrap(); + + return Ok(Lease::new(cache_ref, item)); + } + } + } + + debug!("Creating new {}", stringify!(Buffer)); + + let item = Buffer::create(&self.device, info)?; + + Ok(Lease::new(cache_ref, item)) + } +} + +impl Pool for FifoPool { + #[profiling::function] + fn lease(&mut self, info: CommandBufferInfo) -> Result, DriverError> { + let cache = self + .command_buffer_cache + .entry(info.queue_family_index) + .or_insert_with(PoolInfo::default_cache); + let mut item = cache + .lock() + .pop_front() + .filter(can_lease_command_buffer) + .map(Ok) + .unwrap_or_else(|| { + debug!("Creating new {}", stringify!(CommandBuffer)); + + CommandBuffer::create(&self.device, info) + })?; + + // Drop anything we were holding from the last submission + CommandBuffer::drop_fenced(&mut item); + + Ok(Lease::new(Arc::downgrade(cache), item)) + } +} + +impl Pool for FifoPool { + #[profiling::function] + fn lease(&mut self, info: DescriptorPoolInfo) -> Result, DriverError> { + let cache_ref = Arc::downgrade(&self.descriptor_pool_cache); + let mut cache = self.descriptor_pool_cache.lock(); + + { + profiling::scope!("Check cache"); + + // Look for a compatible descriptor pool (has enough sets and descriptors) + for idx in 0..cache.len() { + let item = &cache[idx]; + if item.info.max_sets >= info.max_sets + && item.info.acceleration_structure_count >= info.acceleration_structure_count + && item.info.combined_image_sampler_count >= info.combined_image_sampler_count + && item.info.input_attachment_count >= info.input_attachment_count + && item.info.sampled_image_count >= info.sampled_image_count + && item.info.storage_buffer_count >= info.storage_buffer_count + && item.info.storage_buffer_dynamic_count >= info.storage_buffer_dynamic_count + && item.info.storage_image_count >= info.storage_image_count + && item.info.storage_texel_buffer_count >= info.storage_texel_buffer_count + && item.info.uniform_buffer_count >= info.uniform_buffer_count + && item.info.uniform_buffer_dynamic_count >= info.uniform_buffer_dynamic_count + && item.info.uniform_texel_buffer_count >= info.uniform_texel_buffer_count + { + let item = cache.remove(idx).unwrap(); + + return Ok(Lease::new(cache_ref, item)); + } + } + } + + debug!("Creating new {}", stringify!(DescriptorPool)); + + let item = DescriptorPool::create(&self.device, info)?; + + Ok(Lease::new(cache_ref, item)) + } +} + +impl Pool for FifoPool { + #[profiling::function] + fn lease(&mut self, info: ImageInfo) -> Result, DriverError> { + let cache_ref = Arc::downgrade(&self.image_cache); + let mut cache = self.image_cache.lock(); + + { + profiling::scope!("Check cache"); + + // Look for a compatible image (same properties, superset of creation flags and usage + // flags) + for idx in 0..cache.len() { + let item = &cache[idx]; + if item.info.array_elements == info.array_elements + && item.info.depth == info.depth + && item.info.fmt == info.fmt + && item.info.height == info.height + && item.info.linear_tiling == info.linear_tiling + && item.info.mip_level_count == info.mip_level_count + && item.info.sample_count == info.sample_count + && item.info.ty == info.ty + && item.info.width == info.width + && item.info.flags.contains(info.flags) + && item.info.usage.contains(info.usage) + { + let item = cache.remove(idx).unwrap(); + + return Ok(Lease::new(cache_ref, item)); + } + } + } + + debug!("Creating new {}", stringify!(Image)); + + let item = Image::create(&self.device, info)?; + + Ok(Lease::new(cache_ref, item)) + } +} + +impl Pool for FifoPool { + #[profiling::function] + fn lease(&mut self, info: RenderPassInfo) -> Result, DriverError> { + let cache = if let Some(cache) = self.render_pass_cache.get(&info) { + cache + } else { + // We tried to get the cache first in order to avoid this clone + self.render_pass_cache + .entry(info.clone()) + .or_insert_with(PoolInfo::default_cache) + }; + let item = cache.lock().pop_front().map(Ok).unwrap_or_else(|| { + debug!("Creating new {}", stringify!(RenderPass)); + + RenderPass::create(&self.device, info) + })?; + + Ok(Lease::new(Arc::downgrade(cache), item)) + } +} diff --git a/src/pool/hash.rs b/src/pool/hash.rs index 942d3d9..ed17434 100644 --- a/src/pool/hash.rs +++ b/src/pool/hash.rs @@ -1,29 +1,39 @@ //! Pool which leases by exactly matching the information before creating new resources. -//! -//! The information for each lease request is placed into a `HashMap`. If no resources exist for -//! the exact information provided then a new resource is created and returned. use { - super::{Cache, Lease, Pool}, + super::{can_lease_command_buffer, Cache, Lease, Pool, PoolInfo}, crate::driver::{ - accel_struct::{ - AccelerationStructure, AccelerationStructureInfo, AccelerationStructureInfoBuilder, - }, - buffer::{Buffer, BufferInfo, BufferInfoBuilder}, + accel_struct::{AccelerationStructure, AccelerationStructureInfo}, + buffer::{Buffer, BufferInfo}, device::Device, - image::{Image, ImageInfo, ImageInfoBuilder}, + image::{Image, ImageInfo}, CommandBuffer, CommandBufferInfo, DescriptorPool, DescriptorPoolInfo, DriverError, RenderPass, RenderPassInfo, }, + log::debug, parking_lot::Mutex, + paste::paste, std::{ collections::{HashMap, VecDeque}, - fmt::Debug, sync::Arc, }, }; /// A high-performance resource allocator. +/// +/// # Bucket Strategy +/// +/// The information for each lease request is the key for a `HashMap` of buckets. If no bucket +/// exists with the exact information provided a new bucket is created. +/// +/// In practice this means that for a [`PoolInfo::image_capacity`] of `4`, requests for a 1024x1024 +/// image with certain attributes will store a maximum of `4` such images. Requests for any image +/// having a different size or attributes will store an additional maximum of `4` images. +/// +/// # Memory Management +/// +/// If requests for varying resources is common [`HashPool::clear_images_by_info`] and other memory +/// management functions are nessecery in order to avoid using all available device memory. #[derive(Debug)] pub struct HashPool { acceleration_structure_cache: HashMap>, @@ -32,13 +42,19 @@ pub struct HashPool { descriptor_pool_cache: HashMap>, device: Arc, image_cache: HashMap>, + info: PoolInfo, render_pass_cache: HashMap>, } -// TODO: Add some sort of manager features (like, I dunno, "Clear Some Memory For me") impl HashPool { /// Constructs a new `HashPool`. pub fn new(device: &Arc) -> Self { + Self::with_capacity(device, PoolInfo::default()) + } + + /// Constructs a new `HashPool` with the given capacity information. + pub fn with_capacity(device: &Arc, info: impl Into) -> Self { + let info: PoolInfo = info.into(); let device = Arc::clone(device); Self { @@ -48,110 +64,153 @@ impl HashPool { descriptor_pool_cache: Default::default(), device, image_cache: Default::default(), + info, render_pass_cache: Default::default(), } } + + /// Clears the pool, removing all resources. + pub fn clear(&mut self) { + self.clear_accel_structs(); + self.clear_buffers(); + self.clear_images(); + } } -impl HashPool { - fn can_lease_command_buffer(cmd_buf: &mut CommandBuffer) -> bool { - let can_lease = unsafe { - // Don't lease this command buffer if it is unsignalled; we'll create a new one - // and wait for this, and those behind it, to signal. - cmd_buf - .device - .get_fence_status(cmd_buf.fence) - .unwrap_or_default() - }; +macro_rules! resource_mgmt_fns { + ($fn_plural:literal, $doc_singular:literal, $ty:ty, $field:ident) => { + paste! { + impl HashPool { + #[doc = "Clears the pool of " $doc_singular " resources."] + pub fn [](&mut self) { + self.$field.clear(); + } - if can_lease { - // Drop anything we were holding from the last submission - CommandBuffer::drop_fenced(cmd_buf); - } + #[doc = "Clears the pool of all " $doc_singular " resources matching the given +information."] + pub fn []( + &mut self, + info: impl Into<$ty>, + ) { + self.$field.remove(&info.into()); + } - can_lease - } + #[doc = "Retains only the " $doc_singular " resources specified by the predicate.\n +\nIn other words, remove all " $doc_singular " resources for which `f(" $ty ")` returns `false`.\n +\n"] + /// The elements are visited in unsorted (and unspecified) order. + /// + /// # Performance + /// + /// Provides the same performance guarantees as + /// [`HashMap::retain`](HashMap::retain). + pub fn [](&mut self, mut f: F) + where + F: FnMut($ty) -> bool, + { + self.$field.retain(|&info, _| f(info)) + } + } + } + }; } +resource_mgmt_fns!( + "accel_structs", + "acceleration structure", + AccelerationStructureInfo, + acceleration_structure_cache +); +resource_mgmt_fns!("buffers", "buffer", BufferInfo, buffer_cache); +resource_mgmt_fns!("images", "image", ImageInfo, image_cache); + impl Pool for HashPool { #[profiling::function] fn lease(&mut self, info: CommandBufferInfo) -> Result, DriverError> { - let command_buffer_cache = self + let cache = self .command_buffer_cache .entry(info.queue_family_index) - .or_default(); - let cache_ref = Arc::downgrade(command_buffer_cache); - let mut cache = command_buffer_cache.lock(); + .or_insert_with(PoolInfo::default_cache); + let mut item = cache + .lock() + .pop_front() + .filter(can_lease_command_buffer) + .map(Ok) + .unwrap_or_else(|| { + debug!("Creating new {}", stringify!(CommandBuffer)); + + CommandBuffer::create(&self.device, info) + })?; + + // Drop anything we were holding from the last submission + CommandBuffer::drop_fenced(&mut item); + + Ok(Lease::new(Arc::downgrade(cache), item)) + } +} + +impl Pool for HashPool { + #[profiling::function] + fn lease(&mut self, info: DescriptorPoolInfo) -> Result, DriverError> { + let cache = self + .descriptor_pool_cache + .entry(info.clone()) + .or_insert_with(PoolInfo::default_cache); + let item = cache.lock().pop_front().map(Ok).unwrap_or_else(|| { + debug!("Creating new {}", stringify!(DescriptorPool)); + + DescriptorPool::create(&self.device, info) + })?; + + Ok(Lease::new(Arc::downgrade(cache), item)) + } +} - if cache.is_empty() || !Self::can_lease_command_buffer(cache.front_mut().unwrap()) { - let item = CommandBuffer::create(&self.device, info)?; +impl Pool for HashPool { + #[profiling::function] + fn lease(&mut self, info: RenderPassInfo) -> Result, DriverError> { + let cache = if let Some(cache) = self.render_pass_cache.get(&info) { + cache + } else { + // We tried to get the cache first in order to avoid this clone + self.render_pass_cache + .entry(info.clone()) + .or_insert_with(PoolInfo::default_cache) + }; + let item = cache.lock().pop_front().map(Ok).unwrap_or_else(|| { + debug!("Creating new {}", stringify!(RenderPass)); - return Ok(Lease { - cache: Some(cache_ref), - item: Some(item), - }); - } + RenderPass::create(&self.device, info) + })?; - Ok(Lease { - cache: Some(cache_ref), - item: cache.pop_front(), - }) + Ok(Lease::new(Arc::downgrade(cache), item)) } } // Enable leasing items using their basic info macro_rules! lease { - ($info:ident => $item:ident) => { + ($info:ident => $item:ident, $capacity:ident) => { paste::paste! { impl Pool<$info, $item> for HashPool { #[profiling::function] fn lease(&mut self, info: $info) -> Result, DriverError> { let cache = self.[<$item:snake _cache>].entry(info.clone()) .or_insert_with(|| { - Arc::new(Mutex::new(VecDeque::new())) - }); - let cache_ref = Arc::downgrade(cache); - let mut cache = cache.lock(); - - if cache.is_empty() { - let item = $item::create(&self.device, info)?; - - return Ok(Lease { - cache: Some(cache_ref), - item: Some(item), + Cache::new(Mutex::new(VecDeque::with_capacity(self.info.$capacity))) }); - } + let item = cache.lock().pop_front().map(Ok).unwrap_or_else(|| { + debug!("Creating new {}", stringify!($item)); - Ok(Lease { - cache: Some(cache_ref), - item: cache.pop_front(), - }) - } - } - } - }; -} - -lease!(RenderPassInfo => RenderPass); -lease!(DescriptorPoolInfo => DescriptorPool); - -// Enable leasing items as above, but also using their info builder type for convenience -macro_rules! lease_builder { - ($info:ident => $item:ident) => { - lease!($info => $item); - - paste::paste! { - impl Pool<[<$info Builder>], $item> for HashPool { - fn lease(&mut self, builder: [<$info Builder>]) -> Result, DriverError> { - let info = builder.build(); + $item::create(&self.device, info) + })?; - self.lease(info) + Ok(Lease::new(Arc::downgrade(cache), item)) } } } }; } -lease_builder!(AccelerationStructureInfo => AccelerationStructure); -lease_builder!(BufferInfo => Buffer); -lease_builder!(ImageInfo => Image); +lease!(AccelerationStructureInfo => AccelerationStructure, accel_struct_capacity); +lease!(BufferInfo => Buffer, buffer_capacity); +lease!(ImageInfo => Image, image_capacity); diff --git a/src/pool/lazy.rs b/src/pool/lazy.rs index 9b113c9..6dfccc3 100644 --- a/src/pool/lazy.rs +++ b/src/pool/lazy.rs @@ -1,32 +1,18 @@ //! Pool which leases by looking for compatibile information before creating new resources. -//! -//! The information for each lease request is loosely bucketed by compatibility. If no acceptable -//! resources exist for the information provided then a new resource is created and returned. -//! -//! # Details -//! * Acceleration structures may be larger than requested -//! * Buffers may be larger than request or have additional usage flags -//! * Images may have additional usage flags use { - super::{Cache, Lease, Pool}, + super::{can_lease_command_buffer, Cache, Lease, Pool, PoolInfo}, crate::driver::{ - accel_struct::{ - AccelerationStructure, AccelerationStructureInfo, AccelerationStructureInfoBuilder, - }, - buffer::{Buffer, BufferInfo, BufferInfoBuilder}, + accel_struct::{AccelerationStructure, AccelerationStructureInfo}, + buffer::{Buffer, BufferInfo}, device::Device, - image::{Image, ImageInfo, ImageInfoBuilder, ImageType, SampleCount}, + image::{Image, ImageInfo, ImageType, SampleCount}, CommandBuffer, CommandBufferInfo, DescriptorPool, DescriptorPoolInfo, DriverError, RenderPass, RenderPassInfo, }, ash::vk, - parking_lot::Mutex, - std::{ - collections::{HashMap, VecDeque}, - fmt::Debug, - sync::Arc, - }, + log::debug, + std::{collections::HashMap, sync::Arc}, }; #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)] @@ -42,53 +28,131 @@ struct ImageKey { width: u32, } -/// A high-efficiency resource allocator. +impl From for ImageKey { + fn from(info: ImageInfo) -> Self { + Self { + array_elements: info.array_elements, + depth: info.depth, + fmt: info.fmt, + height: info.height, + linear_tiling: info.linear_tiling, + mip_level_count: info.mip_level_count, + sample_count: info.sample_count, + ty: info.ty, + width: info.width, + } + } +} + +/// A balanced resource allocator. +/// +/// The information for each lease request is compared against the stored resources for +/// compatibility. If no acceptable resources are stored for the information provided a new resource +/// is created and returned. +/// +/// # Details +/// +/// * Acceleration structures may be larger than requested +/// * Buffers may be larger than requested or have additional usage flags +/// * Images may have additional usage flags +/// +/// # Bucket Strategy +/// +/// The information for each lease request is the key for a `HashMap` of buckets. If no bucket +/// exists with compatible information a new bucket is created. +/// +/// In practice this means that for a [`PoolInfo::image_capacity`] of `4`, requests for a 1024x1024 +/// image with certain attributes will store a maximum of `4` such images. Requests for any image +/// having a different size or incompatible attributes will store an additional maximum of `4` +/// images. +/// +/// # Memory Management +/// +/// If requests for varying resources is common [`LazyPool::clear_images_by_info`] and other memory +/// management functions are nessecery in order to avoid using all available device memory. #[derive(Debug)] pub struct LazyPool { - acceleration_structure_cache: - HashMap>, - buffer_cache: HashMap>, + accel_struct_cache: HashMap>, + buffer_cache: HashMap<(bool, vk::DeviceSize), Cache>, command_buffer_cache: HashMap>, descriptor_pool_cache: Cache, device: Arc, image_cache: HashMap>, + info: PoolInfo, render_pass_cache: HashMap>, } -// TODO: Add some sort of manager features (like, I dunno, "Clear Some Memory For me") impl LazyPool { /// Constructs a new `LazyPool`. pub fn new(device: &Arc) -> Self { + Self::with_capacity(device, PoolInfo::default()) + } + + /// Constructs a new `LazyPool` with the given capacity information. + pub fn with_capacity(device: &Arc, info: impl Into) -> Self { + let info: PoolInfo = info.into(); let device = Arc::clone(device); Self { - acceleration_structure_cache: Default::default(), + accel_struct_cache: Default::default(), buffer_cache: Default::default(), command_buffer_cache: Default::default(), - descriptor_pool_cache: Default::default(), + descriptor_pool_cache: PoolInfo::default_cache(), device, image_cache: Default::default(), + info, render_pass_cache: Default::default(), } } - #[profiling::function] - fn can_lease_command_buffer(cmd_buf: &mut CommandBuffer) -> bool { - let can_lease = unsafe { - // Don't lease this command buffer if it is unsignalled; we'll create a new one - // and wait for this, and those behind it, to signal. - cmd_buf - .device - .get_fence_status(cmd_buf.fence) - .unwrap_or_default() - }; + /// Clears the pool, removing all resources. + pub fn clear(&mut self) { + self.clear_accel_structs(); + self.clear_buffers(); + self.clear_images(); + } - if can_lease { - // Drop anything we were holding from the last submission - CommandBuffer::drop_fenced(cmd_buf); - } + /// Clears the pool of acceleration structure resources. + pub fn clear_accel_structs(&mut self) { + self.accel_struct_cache.clear(); + } + + /// Clears the pool of all acceleration structure resources matching the given type. + pub fn clear_accel_structs_by_ty(&mut self, ty: vk::AccelerationStructureTypeKHR) { + self.accel_struct_cache.remove(&ty); + } + + /// Clears the pool of buffer resources. + pub fn clear_buffers(&mut self) { + self.buffer_cache.clear(); + } + + /// Clears the pool of image resources. + pub fn clear_images(&mut self) { + self.image_cache.clear(); + } + + /// Clears the pool of image resources matching the given information. + pub fn clear_images_by_info(&mut self, info: impl Into) { + self.image_cache.remove(&info.into().into()); + } - can_lease + /// Retains only the acceleration structure resources specified by the predicate. + /// + /// In other words, remove all resources for which `f(vk::AccelerationStructureTypeKHR)` returns + /// `false`. + /// + /// The elements are visited in unsorted (and unspecified) order. + /// + /// # Performance + /// + /// Provides the same performance guarantees as + /// [`HashMap::retain`](HashMap::retain). + pub fn retain_accel_structs(&mut self, mut f: F) + where + F: FnMut(vk::AccelerationStructureTypeKHR) -> bool, + { + self.accel_struct_cache.retain(|&ty, _| f(ty)) } } @@ -98,21 +162,12 @@ impl Pool for LazyPool { &mut self, info: AccelerationStructureInfo, ) -> Result, DriverError> { - let acceleration_structure_cache = self - .acceleration_structure_cache + let cache = self + .accel_struct_cache .entry(info.ty) - .or_default(); - let cache_ref = Arc::downgrade(acceleration_structure_cache); - let mut cache = acceleration_structure_cache.lock(); - - if cache.is_empty() { - let item = AccelerationStructure::create(&self.device, info)?; - - return Ok(Lease { - cache: Some(cache_ref), - item: Some(item), - }); - } + .or_insert_with(|| PoolInfo::explicit_cache(self.info.accel_struct_capacity)); + let cache_ref = Arc::downgrade(cache); + let mut cache = cache.lock(); { profiling::scope!("Check cache"); @@ -123,80 +178,73 @@ impl Pool for LazyPool { if item.info.size >= info.size { let item = cache.remove(idx).unwrap(); - return Ok(Lease { - cache: Some(cache_ref), - item: Some(item), - }); + return Ok(Lease::new(cache_ref, item)); } } } - let item = AccelerationStructure::create(&self.device, info)?; + debug!("Creating new {}", stringify!(AccelerationStructure)); - Ok(Lease { - cache: Some(cache_ref), - item: Some(item), - }) - } -} + let item = AccelerationStructure::create(&self.device, info)?; -impl Pool for LazyPool { - fn lease( - &mut self, - info: AccelerationStructureInfoBuilder, - ) -> Result, DriverError> { - self.lease(info.build()) + Ok(Lease::new(cache_ref, item)) } } impl Pool for LazyPool { #[profiling::function] fn lease(&mut self, info: BufferInfo) -> Result, DriverError> { - let buffer_cache = self.buffer_cache.entry(info.can_map).or_default(); - let cache_ref = Arc::downgrade(buffer_cache); - let mut cache = buffer_cache.lock(); - - if cache.is_empty() { - let item = Buffer::create(&self.device, info)?; - - return Ok(Lease { - cache: Some(cache_ref), - item: Some(item), - }); - } + let cache = self + .buffer_cache + .entry((info.can_map, info.alignment)) + .or_insert_with(|| PoolInfo::explicit_cache(self.info.buffer_capacity)); + let cache_ref = Arc::downgrade(cache); + let mut cache = cache.lock(); { profiling::scope!("Check cache"); - // Look for a compatible buffer (same mapping mode, big enough, superset of usage flags) + // Look for a compatible buffer (big enough and superset of usage flags) for idx in 0..cache.len() { let item = &cache[idx]; - if item.info.can_map == info.can_map - && item.info.size >= info.size - && item.info.usage.contains(info.usage) - { + if item.info.size >= info.size && item.info.usage.contains(info.usage) { let item = cache.remove(idx).unwrap(); - return Ok(Lease { - cache: Some(cache_ref), - item: Some(item), - }); + return Ok(Lease::new(cache_ref, item)); } } } + debug!("Creating new {}", stringify!(Buffer)); + let item = Buffer::create(&self.device, info)?; - Ok(Lease { - cache: Some(cache_ref), - item: Some(item), - }) + Ok(Lease::new(cache_ref, item)) } } -impl Pool for LazyPool { - fn lease(&mut self, info: BufferInfoBuilder) -> Result, DriverError> { - self.lease(info.build()) +impl Pool for LazyPool { + #[profiling::function] + fn lease(&mut self, info: CommandBufferInfo) -> Result, DriverError> { + let cache = self + .command_buffer_cache + .entry(info.queue_family_index) + .or_insert_with(PoolInfo::default_cache); + let mut item = cache + .lock() + .pop_front() + .filter(can_lease_command_buffer) + .map(Ok) + .unwrap_or_else(|| { + debug!("Creating new {}", stringify!(CommandBuffer)); + + CommandBuffer::create(&self.device, info) + })?; + + // Drop anything we were holding from the last submission + CommandBuffer::drop_fenced(&mut item); + + Ok(Lease::new(Arc::downgrade(cache), item)) } } @@ -206,15 +254,6 @@ impl Pool for LazyPool { let cache_ref = Arc::downgrade(&self.descriptor_pool_cache); let mut cache = self.descriptor_pool_cache.lock(); - if cache.is_empty() { - let item = DescriptorPool::create(&self.device, info)?; - - return Ok(Lease { - cache: Some(cache_ref), - item: Some(item), - }); - } - { profiling::scope!("Check cache"); @@ -236,51 +275,28 @@ impl Pool for LazyPool { { let item = cache.remove(idx).unwrap(); - return Ok(Lease { - cache: Some(cache_ref), - item: Some(item), - }); + return Ok(Lease::new(cache_ref, item)); } } } + debug!("Creating new {}", stringify!(DescriptorPool)); + let item = DescriptorPool::create(&self.device, info)?; - Ok(Lease { - cache: Some(cache_ref), - item: Some(item), - }) + Ok(Lease::new(cache_ref, item)) } } impl Pool for LazyPool { #[profiling::function] fn lease(&mut self, info: ImageInfo) -> Result, DriverError> { - let image_cache = self + let cache = self .image_cache - .entry(ImageKey { - array_elements: info.array_elements, - depth: info.depth, - fmt: info.fmt, - height: info.height, - linear_tiling: info.linear_tiling, - mip_level_count: info.mip_level_count, - sample_count: info.sample_count, - ty: info.ty, - width: info.width, - }) - .or_default(); - let cache_ref = Arc::downgrade(image_cache); - let mut cache = image_cache.lock(); - - if cache.is_empty() { - let item = Image::create(&self.device, info)?; - - return Ok(Lease { - cache: Some(cache_ref), - item: Some(item), - }); - } + .entry(info.into()) + .or_insert_with(|| PoolInfo::explicit_cache(self.info.image_capacity)); + let cache_ref = Arc::downgrade(cache); + let mut cache = cache.lock(); { profiling::scope!("Check cache"); @@ -291,80 +307,36 @@ impl Pool for LazyPool { if item.info.flags.contains(info.flags) && item.info.usage.contains(info.usage) { let item = cache.remove(idx).unwrap(); - return Ok(Lease { - cache: Some(cache_ref), - item: Some(item), - }); + return Ok(Lease::new(cache_ref, item)); } } } - let item = Image::create(&self.device, info)?; + debug!("Creating new {}", stringify!(Image)); - Ok(Lease { - cache: Some(cache_ref), - item: Some(item), - }) - } -} + let item = Image::create(&self.device, info)?; -impl Pool for LazyPool { - fn lease(&mut self, info: ImageInfoBuilder) -> Result, DriverError> { - self.lease(info.build()) + Ok(Lease::new(cache_ref, item)) } } impl Pool for LazyPool { #[profiling::function] fn lease(&mut self, info: RenderPassInfo) -> Result, DriverError> { - if let Some(cache) = self.render_pass_cache.get(&info) { - let item = if let Some(item) = cache.lock().pop_front() { - item - } else { - RenderPass::create(&self.device, info)? - }; - - Ok(Lease { - cache: Some(Arc::downgrade(cache)), - item: Some(item), - }) + let cache = if let Some(cache) = self.render_pass_cache.get(&info) { + cache } else { - let cache = Arc::new(Mutex::new(VecDeque::new())); - let cache_ref = Arc::downgrade(&cache); - self.render_pass_cache.insert(info.clone(), cache); - - let item = RenderPass::create(&self.device, info)?; - - Ok(Lease { - cache: Some(cache_ref), - item: Some(item), - }) - } - } -} - -impl Pool for LazyPool { - #[profiling::function] - fn lease(&mut self, info: CommandBufferInfo) -> Result, DriverError> { - let command_buffer_cache = self - .command_buffer_cache - .entry(info.queue_family_index) - .or_default(); - let cache_ref = Arc::downgrade(command_buffer_cache); - let mut cache = command_buffer_cache.lock(); - - if cache.is_empty() || !Self::can_lease_command_buffer(cache.front_mut().unwrap()) { - let item = CommandBuffer::create(&self.device, info)?; + // We tried to get the cache first in order to avoid this clone + self.render_pass_cache + .entry(info.clone()) + .or_insert_with(PoolInfo::default_cache) + }; + let item = cache.lock().pop_front().map(Ok).unwrap_or_else(|| { + debug!("Creating new {}", stringify!(RenderPass)); - return Ok(Lease { - cache: Some(cache_ref), - item: Some(item), - }); - } + RenderPass::create(&self.device, info) + })?; - Ok(Lease { - cache: Some(cache_ref), - item: cache.pop_front(), - }) + Ok(Lease::new(Arc::downgrade(cache), item)) } } diff --git a/src/pool/mod.rs b/src/pool/mod.rs index 8e2e23f..51ca095 100644 --- a/src/pool/mod.rs +++ b/src/pool/mod.rs @@ -1,12 +1,23 @@ //! Resource leasing and pooling types. //! -//! _Screen 13_ provides caching for acceleration structure, buffer, and image resources which may -//! be leased from configurable pools using their corresponding information structure. Most programs -//! will do fine with a single `LazyPool`. +//! _Screen 13_ provides caching for acceleration structure, buffer and image resources which may be +//! leased from configurable pools using their corresponding information structure. Most programs +//! will do fine with a single [`FifoPool`](self::fifo::FifoPool). //! //! Leased resources may be bound directly to a render graph and used in the same manner as regular //! resources. After rendering has finished, the leased resources will return to the pool for reuse. //! +//! # Buckets +//! +//! The provided [`Pool`] implementations store resources in buckets, with each implementation +//! offering a different strategy which balances performance (_more buckets_) with memory efficiency +//! (_fewer buckets_). +//! +//! _Screen 13_'s pools can be grouped into two major categories: +//! +//! * Single-bucket: [`FifoPool`](self::fifo::FifoPool) +//! * Multi-bucket: [`LazyPool`](self::lazy::LazyPool), [`HashPool`](self::hash::HashPool) +//! //! # Examples //! //! Leasing an image: @@ -33,26 +44,38 @@ //! # When Should You Use Which Pool? //! //! These are fairly high-level break-downs of when each pool should be considered. You may need -//! to investigate each type of pool individually or write your own implementation to provide the -//! absolute best fit for your purpose. +//! to investigate each type of pool individually to provide the absolute best fit for your purpose. //! -//! ### Use a `LazyPool` when: -//! * Memory usage is most important +//! ### Use a [`FifoPool`](self::fifo::FifoPool) when: +//! * Low memory usage is most important +//! * Automatic bucket management is desired +//! +//! ### Use a [`LazyPool`](self::lazy::LazyPool) when: //! * Resources have different attributes each frame //! -//! ### Use a `HashPool` when: -//! * Processor usage is most important +//! ### Use a [`HashPool`](self::hash::HashPool) when: +//! * High performance is most important //! * Resources have consistent attributes each frame +pub mod fifo; pub mod hash; pub mod lazy; use { - crate::driver::DriverError, + crate::driver::{ + accel_struct::{ + AccelerationStructure, AccelerationStructureInfo, AccelerationStructureInfoBuilder, + }, + buffer::{Buffer, BufferInfo, BufferInfoBuilder}, + image::{Image, ImageInfo, ImageInfoBuilder}, + CommandBuffer, DriverError, + }, + derive_builder::{Builder, UninitializedFieldError}, parking_lot::Mutex, std::{ collections::VecDeque, fmt::Debug, + mem::ManuallyDrop, ops::{Deref, DerefMut}, sync::{Arc, Weak}, thread::panicking, @@ -62,6 +85,17 @@ use { type Cache = Arc>>; type CacheRef = Weak>>; +fn can_lease_command_buffer(cmd_buf: &CommandBuffer) -> bool { + unsafe { + // Don't lease this command buffer if it is unsignalled; we'll create a new one + // and wait for this, and those behind it, to signal. + cmd_buf + .device + .get_fence_status(cmd_buf.fence) + .unwrap_or_default() + } +} + /// Holds a leased resource and implements `Drop` in order to return the resource. /// /// This simple wrapper type implements only the `AsRef`, `AsMut`, `Deref` and `DerefMut` traits @@ -69,19 +103,28 @@ type CacheRef = Weak>>; /// owners and may be mutably accessed. #[derive(Debug)] pub struct Lease { - cache: Option>, - item: Option, + cache_ref: CacheRef, + item: ManuallyDrop, +} + +impl Lease { + fn new(cache_ref: CacheRef, item: T) -> Self { + Self { + cache_ref, + item: ManuallyDrop::new(item), + } + } } impl AsRef for Lease { fn as_ref(&self) -> &T { - self.item.as_ref().unwrap() + &self.item } } impl AsMut for Lease { fn as_mut(&mut self) -> &mut T { - self.item.as_mut().unwrap() + &mut self.item } } @@ -89,13 +132,13 @@ impl Deref for Lease { type Target = T; fn deref(&self) -> &Self::Target { - self.item.as_ref().unwrap() + &self.item } } impl DerefMut for Lease { fn deref_mut(&mut self) -> &mut Self::Target { - self.item.as_mut().unwrap() + &mut self.item } } @@ -105,9 +148,19 @@ impl Drop for Lease { return; } - if let Some(cache) = self.cache.as_ref() { - if let Some(cache) = cache.upgrade() { - cache.lock().push_back(self.item.take().unwrap()); + // If the pool cache has been dropped we must manually drop the item, otherwise it goes back + // into the pool. + if let Some(cache) = self.cache_ref.upgrade() { + let mut cache = cache.lock(); + + if cache.len() >= cache.capacity() { + cache.pop_front(); + } + + cache.push_back(unsafe { ManuallyDrop::take(&mut self.item) }); + } else { + unsafe { + ManuallyDrop::drop(&mut self.item); } } } @@ -118,3 +171,130 @@ pub trait Pool { /// Lease a resource. fn lease(&mut self, info: I) -> Result, DriverError>; } + +// Enable leasing items using their info builder type for convenience +macro_rules! lease_builder { + ($info:ident => $item:ident) => { + paste::paste! { + impl Pool<[<$info Builder>], $item> for T where T: Pool<$info, $item> { + fn lease(&mut self, builder: [<$info Builder>]) -> Result, DriverError> { + let info = builder.build(); + + self.lease(info) + } + } + } + }; +} + +lease_builder!(AccelerationStructureInfo => AccelerationStructure); +lease_builder!(BufferInfo => Buffer); +lease_builder!(ImageInfo => Image); + +/// Information used to create a [`FifoPool`](self::fifo::FifoPool), +/// [`HashPool`](self::hash::HashPool) or [`LazyPool`](self::lazy::LazyPool) instance. +#[derive(Builder, Clone, Copy, Debug)] +#[builder( + build_fn(private, name = "fallible_build", error = "PoolInfoBuilderError"), + derive(Debug), + pattern = "owned" +)] +pub struct PoolInfo { + /// The maximum size of a single bucket of acceleration structure resource instances. The + /// default value is [`PoolInfo::DEFAULT_RESOURCE_CAPACITY`]. + /// + /// # Note + /// + /// Individual [`Pool`] implementations store varying numbers of buckets. Read the documentation + /// of each implementation to understand how this affects total number of stored acceleration + /// structure instances. + #[builder(default = "PoolInfo::DEFAULT_RESOURCE_CAPACITY", setter(strip_option))] + pub accel_struct_capacity: usize, + + /// The maximum size of a single bucket of buffer resource instances. The default value is + /// [`PoolInfo::DEFAULT_RESOURCE_CAPACITY`]. + /// + /// # Note + /// + /// Individual [`Pool`] implementations store varying numbers of buckets. Read the documentation + /// of each implementation to understand how this affects total number of stored buffer + /// instances. + #[builder(default = "PoolInfo::DEFAULT_RESOURCE_CAPACITY", setter(strip_option))] + pub buffer_capacity: usize, + + /// The maximum size of a single bucket of image resource instances. The default value is + /// [`PoolInfo::DEFAULT_RESOURCE_CAPACITY`]. + /// + /// # Note + /// + /// Individual [`Pool`] implementations store varying numbers of buckets. Read the documentation + /// of each implementation to understand how this affects total number of stored image + /// instances. + #[builder(default = "PoolInfo::DEFAULT_RESOURCE_CAPACITY", setter(strip_option))] + pub image_capacity: usize, +} + +impl PoolInfo { + /// The maximum size of a single bucket of resource instances. + pub const DEFAULT_RESOURCE_CAPACITY: usize = 4; + + /// Constructs a new `PoolInfo` with the given acceleration structure, buffer and image resource + /// capacity for any single bucket. + pub const fn with_capacity(resource_capacity: usize) -> Self { + Self { + accel_struct_capacity: resource_capacity, + buffer_capacity: resource_capacity, + image_capacity: resource_capacity, + } + } + + fn default_cache() -> Cache { + Cache::new(Mutex::new(VecDeque::with_capacity( + Self::DEFAULT_RESOURCE_CAPACITY, + ))) + } + + fn explicit_cache(capacity: usize) -> Cache { + Cache::new(Mutex::new(VecDeque::with_capacity(capacity))) + } +} + +impl Default for PoolInfo { + fn default() -> Self { + PoolInfoBuilder::default().into() + } +} + +impl From for PoolInfo { + fn from(info: PoolInfoBuilder) -> Self { + info.build() + } +} + +impl From for PoolInfo { + fn from(value: usize) -> Self { + Self { + accel_struct_capacity: value, + buffer_capacity: value, + image_capacity: value, + } + } +} + +// HACK: https://github.com/colin-kiegel/rust-derive-builder/issues/56 +impl PoolInfoBuilder { + /// Builds a new `PoolInfo`. + pub fn build(self) -> PoolInfo { + self.fallible_build() + .expect("All required fields set at initialization") + } +} + +#[derive(Debug)] +struct PoolInfoBuilderError; + +impl From for PoolInfoBuilderError { + fn from(_: UninitializedFieldError) -> Self { + Self + } +}