Source code
Revision control
Copy as Markdown
Other Tools
use {
alloc::{collections::VecDeque, vec::Vec},
core::{
convert::TryFrom as _,
fmt::{self, Debug, Display},
},
gpu_descriptor_types::{
CreatePoolError, DescriptorDevice, DescriptorPoolCreateFlags, DescriptorTotalCount,
DeviceAllocationError,
},
hashbrown::HashMap,
};
bitflags::bitflags! {
/// Flags to augment descriptor set allocation.
#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)]
pub struct DescriptorSetLayoutCreateFlags: u32 {
/// Specified that descriptor set must be allocated from\
/// pool with `DescriptorPoolCreateFlags::UPDATE_AFTER_BIND`.
///
/// This flag must be specified when and only when layout was created with matching backend-specific flag,
/// that allows layout to have UpdateAfterBind bindings.
const UPDATE_AFTER_BIND = 0x2;
}
}
/// Descriptor set from allocator.
#[derive(Debug)]
pub struct DescriptorSet<S> {
raw: S,
pool_id: u64,
size: DescriptorTotalCount,
update_after_bind: bool,
}
impl<S> DescriptorSet<S> {
/// Returns reference to raw descriptor set.
pub fn raw(&self) -> &S {
&self.raw
}
/// Returns mutable reference to raw descriptor set.
///
/// # Safety
///
/// Object must not be replaced.
pub unsafe fn raw_mut(&mut self) -> &mut S {
&mut self.raw
}
}
/// AllocationError that may occur during descriptor sets allocation.
#[derive(Debug)]
pub enum AllocationError {
/// Backend reported that device memory has been exhausted.\
/// Deallocating device memory or other resources may increase chance
/// that another allocation would succeed.
OutOfDeviceMemory,
/// Backend reported that host memory has been exhausted.\
/// Deallocating host memory may increase chance that another allocation would succeed.
OutOfHostMemory,
/// The total number of descriptors across all pools created\
/// with flag `CREATE_UPDATE_AFTER_BIND_BIT` set exceeds `max_update_after_bind_descriptors_in_all_pools`
/// Or fragmentation of the underlying hardware resources occurs.
Fragmentation,
}
impl Display for AllocationError {
fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
AllocationError::OutOfDeviceMemory => fmt.write_str("Device memory exhausted"),
AllocationError::OutOfHostMemory => fmt.write_str("Host memory exhausted"),
AllocationError::Fragmentation => fmt.write_str("Fragmentation"),
}
}
}
#[cfg(feature = "std")]
impl std::error::Error for AllocationError {}
impl From<CreatePoolError> for AllocationError {
fn from(err: CreatePoolError) -> Self {
match err {
CreatePoolError::OutOfDeviceMemory => AllocationError::OutOfDeviceMemory,
CreatePoolError::OutOfHostMemory => AllocationError::OutOfHostMemory,
CreatePoolError::Fragmentation => AllocationError::Fragmentation,
}
}
}
const MIN_SETS: u32 = 64;
const MAX_SETS: u32 = 512;
#[derive(Debug)]
struct DescriptorPool<P> {
raw: P,
/// Number of sets allocated from pool.
allocated: u32,
/// Expected number of sets available.
available: u32,
}
#[derive(Debug)]
struct DescriptorBucket<P> {
offset: u64,
pools: VecDeque<DescriptorPool<P>>,
total: u32,
update_after_bind: bool,
size: DescriptorTotalCount,
}
impl<P> Drop for DescriptorBucket<P> {
#[cfg(feature = "tracing")]
fn drop(&mut self) {
#[cfg(feature = "std")]
{
if std::thread::panicking() {
return;
}
}
if self.total > 0 {
tracing::error!("Descriptor sets were not deallocated");
}
}
#[cfg(all(not(feature = "tracing"), feature = "std"))]
fn drop(&mut self) {
if std::thread::panicking() {
return;
}
if self.total > 0 {
eprintln!("Descriptor sets were not deallocated")
}
}
#[cfg(all(not(feature = "tracing"), not(feature = "std")))]
fn drop(&mut self) {
if self.total > 0 {
panic!("Descriptor sets were not deallocated")
}
}
}
impl<P> DescriptorBucket<P> {
fn new(update_after_bind: bool, size: DescriptorTotalCount) -> Self {
DescriptorBucket {
offset: 0,
pools: VecDeque::new(),
total: 0,
update_after_bind,
size,
}
}
fn new_pool_size(&self, minimal_set_count: u32) -> (DescriptorTotalCount, u32) {
let mut max_sets = MIN_SETS // at least MIN_SETS
.max(minimal_set_count) // at least enough for allocation
.max(self.total.min(MAX_SETS)) // at least as much as was allocated so far capped to MAX_SETS
.checked_next_power_of_two() // rounded up to nearest 2^N
.unwrap_or(i32::MAX as u32);
max_sets = (u32::MAX / self.size.sampler.max(1)).min(max_sets);
max_sets = (u32::MAX / self.size.combined_image_sampler.max(1)).min(max_sets);
max_sets = (u32::MAX / self.size.sampled_image.max(1)).min(max_sets);
max_sets = (u32::MAX / self.size.storage_image.max(1)).min(max_sets);
max_sets = (u32::MAX / self.size.uniform_texel_buffer.max(1)).min(max_sets);
max_sets = (u32::MAX / self.size.storage_texel_buffer.max(1)).min(max_sets);
max_sets = (u32::MAX / self.size.uniform_buffer.max(1)).min(max_sets);
max_sets = (u32::MAX / self.size.storage_buffer.max(1)).min(max_sets);
max_sets = (u32::MAX / self.size.uniform_buffer_dynamic.max(1)).min(max_sets);
max_sets = (u32::MAX / self.size.storage_buffer_dynamic.max(1)).min(max_sets);
max_sets = (u32::MAX / self.size.input_attachment.max(1)).min(max_sets);
max_sets = (u32::MAX / self.size.acceleration_structure.max(1)).min(max_sets);
max_sets = (u32::MAX / self.size.inline_uniform_block_bytes.max(1)).min(max_sets);
max_sets = (u32::MAX / self.size.inline_uniform_block_bindings.max(1)).min(max_sets);
let mut pool_size = DescriptorTotalCount {
sampler: self.size.sampler * max_sets,
combined_image_sampler: self.size.combined_image_sampler * max_sets,
sampled_image: self.size.sampled_image * max_sets,
storage_image: self.size.storage_image * max_sets,
uniform_texel_buffer: self.size.uniform_texel_buffer * max_sets,
storage_texel_buffer: self.size.storage_texel_buffer * max_sets,
uniform_buffer: self.size.uniform_buffer * max_sets,
storage_buffer: self.size.storage_buffer * max_sets,
uniform_buffer_dynamic: self.size.uniform_buffer_dynamic * max_sets,
storage_buffer_dynamic: self.size.storage_buffer_dynamic * max_sets,
input_attachment: self.size.input_attachment * max_sets,
acceleration_structure: self.size.acceleration_structure * max_sets,
inline_uniform_block_bytes: self.size.inline_uniform_block_bytes * max_sets,
inline_uniform_block_bindings: self.size.inline_uniform_block_bindings * max_sets,
};
if pool_size == Default::default() {
pool_size.sampler = 1;
}
(pool_size, max_sets)
}
unsafe fn allocate<L, S>(
&mut self,
device: &impl DescriptorDevice<L, P, S>,
layout: &L,
mut count: u32,
allocated_sets: &mut Vec<DescriptorSet<S>>,
) -> Result<(), AllocationError> {
debug_assert!(usize::try_from(count).is_ok(), "Must be ensured by caller");
if count == 0 {
return Ok(());
}
for (index, pool) in self.pools.iter_mut().enumerate().rev() {
if pool.available == 0 {
continue;
}
let allocate = pool.available.min(count);
#[cfg(feature = "tracing")]
tracing::trace!("Allocate `{}` sets from exising pool", allocate);
let result = device.alloc_descriptor_sets(
&mut pool.raw,
(0..allocate).map(|_| layout),
&mut Allocation {
size: self.size,
update_after_bind: self.update_after_bind,
pool_id: index as u64 + self.offset,
sets: allocated_sets,
},
);
match result {
Ok(()) => {}
Err(DeviceAllocationError::OutOfDeviceMemory) => {
return Err(AllocationError::OutOfDeviceMemory)
}
Err(DeviceAllocationError::OutOfHostMemory) => {
return Err(AllocationError::OutOfHostMemory)
}
Err(DeviceAllocationError::FragmentedPool) => {
// Should not happen, but better this than panicing.
#[cfg(feature = "tracing")]
tracing::error!("Unexpectedly failed to allocated descriptor sets due to pool fragmentation");
pool.available = 0;
continue;
}
Err(DeviceAllocationError::OutOfPoolMemory) => {
pool.available = 0;
continue;
}
}
count -= allocate;
pool.available -= allocate;
pool.allocated += allocate;
self.total += allocate;
if count == 0 {
return Ok(());
}
}
while count > 0 {
let (pool_size, max_sets) = self.new_pool_size(count);
#[cfg(feature = "tracing")]
tracing::trace!(
"Create new pool with {} sets and {:?} descriptors",
max_sets,
pool_size,
);
let mut raw = device.create_descriptor_pool(
&pool_size,
max_sets,
if self.update_after_bind {
DescriptorPoolCreateFlags::FREE_DESCRIPTOR_SET
| DescriptorPoolCreateFlags::UPDATE_AFTER_BIND
} else {
DescriptorPoolCreateFlags::FREE_DESCRIPTOR_SET
},
)?;
let pool_id = self.pools.len() as u64 + self.offset;
let allocate = max_sets.min(count);
let result = device.alloc_descriptor_sets(
&mut raw,
(0..allocate).map(|_| layout),
&mut Allocation {
pool_id,
size: self.size,
update_after_bind: self.update_after_bind,
sets: allocated_sets,
},
);
match result {
Ok(()) => {}
Err(err) => {
device.destroy_descriptor_pool(raw);
match err {
DeviceAllocationError::OutOfDeviceMemory => {
return Err(AllocationError::OutOfDeviceMemory)
}
DeviceAllocationError::OutOfHostMemory => {
return Err(AllocationError::OutOfHostMemory)
}
DeviceAllocationError::FragmentedPool => {
// Should not happen, but better this than panicing.
#[cfg(feature = "trace")]
trace::error!("Unexpectedly failed to allocated descriptor sets due to pool fragmentation");
}
DeviceAllocationError::OutOfPoolMemory => {}
}
panic!("Failed to allocate descriptor sets from fresh pool");
}
}
count -= allocate;
self.pools.push_back(DescriptorPool {
raw,
allocated: allocate,
available: max_sets - allocate,
});
self.total += allocate;
}
Ok(())
}
unsafe fn free<L, S>(
&mut self,
device: &impl DescriptorDevice<L, P, S>,
raw_sets: impl IntoIterator<Item = S>,
pool_id: u64,
) {
let pool = usize::try_from(pool_id - self.offset)
.ok()
.and_then(|index| self.pools.get_mut(index))
.expect("Invalid pool id");
let mut raw_sets = raw_sets.into_iter();
let mut count = 0;
device.dealloc_descriptor_sets(&mut pool.raw, raw_sets.by_ref().inspect(|_| count += 1));
debug_assert!(
raw_sets.next().is_none(),
"Device must deallocated all sets from iterator"
);
pool.available += count;
pool.allocated -= count;
self.total -= count;
#[cfg(feature = "tracing")]
tracing::trace!("Freed {} from descriptor bucket", count);
while let Some(pool) = self.pools.pop_front() {
if self.pools.is_empty() || pool.allocated != 0 {
self.pools.push_front(pool);
break;
}
#[cfg(feature = "tracing")]
tracing::trace!("Destroying old descriptor pool");
device.destroy_descriptor_pool(pool.raw);
self.offset += 1;
}
}
unsafe fn cleanup<L, S>(&mut self, device: &impl DescriptorDevice<L, P, S>) {
while let Some(pool) = self.pools.pop_front() {
if pool.allocated != 0 {
self.pools.push_front(pool);
break;
}
#[cfg(feature = "tracing")]
tracing::trace!("Destroying old descriptor pool");
device.destroy_descriptor_pool(pool.raw);
self.offset += 1;
}
}
}
/// Descriptor allocator.
/// Can be used to allocate descriptor sets for any layout.
#[derive(Debug)]
pub struct DescriptorAllocator<P, S> {
buckets: HashMap<(DescriptorTotalCount, bool), DescriptorBucket<P>>,
sets_cache: Vec<DescriptorSet<S>>,
raw_sets_cache: Vec<S>,
max_update_after_bind_descriptors_in_all_pools: u32,
current_update_after_bind_descriptors_in_all_pools: u32,
total: u32,
}
impl<P, S> Drop for DescriptorAllocator<P, S> {
fn drop(&mut self) {
if self.buckets.drain().any(|(_, bucket)| bucket.total != 0) {
#[cfg(feature = "tracing")]
tracing::error!(
"`DescriptorAllocator` is dropped while some descriptor sets were not deallocated"
);
}
}
}
impl<P, S> DescriptorAllocator<P, S> {
/// Create new allocator instance.
pub fn new(max_update_after_bind_descriptors_in_all_pools: u32) -> Self {
DescriptorAllocator {
buckets: HashMap::default(),
total: 0,
sets_cache: Vec::new(),
raw_sets_cache: Vec::new(),
max_update_after_bind_descriptors_in_all_pools,
current_update_after_bind_descriptors_in_all_pools: 0,
}
}
/// Allocate descriptor set with specified layout.
///
/// # Safety
///
/// * Same `device` instance must be passed to all method calls of
/// one `DescriptorAllocator` instance.
/// * `flags` must match flags that were used to create the layout.
/// * `layout_descriptor_count` must match descriptor numbers in the layout.
pub unsafe fn allocate<L, D>(
&mut self,
device: &D,
layout: &L,
flags: DescriptorSetLayoutCreateFlags,
layout_descriptor_count: &DescriptorTotalCount,
count: u32,
) -> Result<Vec<DescriptorSet<S>>, AllocationError>
where
S: Debug,
L: Debug,
D: DescriptorDevice<L, P, S>,
{
if count == 0 {
return Ok(Vec::new());
}
let descriptor_count = count * layout_descriptor_count.total();
let update_after_bind = flags.contains(DescriptorSetLayoutCreateFlags::UPDATE_AFTER_BIND);
if update_after_bind
&& self.max_update_after_bind_descriptors_in_all_pools
- self.current_update_after_bind_descriptors_in_all_pools
< descriptor_count
{
return Err(AllocationError::Fragmentation);
}
#[cfg(feature = "tracing")]
tracing::trace!(
"Allocating {} sets with layout {:?} @ {:?}",
count,
layout,
layout_descriptor_count
);
let bucket = self
.buckets
.entry((*layout_descriptor_count, update_after_bind))
.or_insert_with(|| DescriptorBucket::new(update_after_bind, *layout_descriptor_count));
match bucket.allocate(device, layout, count, &mut self.sets_cache) {
Ok(()) => {
self.total += descriptor_count;
if update_after_bind {
self.current_update_after_bind_descriptors_in_all_pools += descriptor_count;
}
Ok(core::mem::take(&mut self.sets_cache))
}
Err(err) => {
debug_assert!(self.raw_sets_cache.is_empty());
// Free sets allocated so far.
let mut last = None;
for set in self.sets_cache.drain(..) {
if Some(set.pool_id) != last {
if let Some(last_id) = last {
// Free contiguous range of sets from one pool in one go.
bucket.free(device, self.raw_sets_cache.drain(..), last_id);
}
}
last = Some(set.pool_id);
self.raw_sets_cache.push(set.raw);
}
if let Some(last_id) = last {
bucket.free(device, self.raw_sets_cache.drain(..), last_id);
}
Err(err)
}
}
}
/// Free descriptor sets.
///
/// # Safety
///
/// * Same `device` instance must be passed to all method calls of
/// one `DescriptorAllocator` instance.
/// * None of descriptor sets can be referenced in any pending command buffers.
/// * All command buffers where at least one of descriptor sets referenced
/// move to invalid state.
pub unsafe fn free<L, D, I>(&mut self, device: &D, sets: I)
where
D: DescriptorDevice<L, P, S>,
I: IntoIterator<Item = DescriptorSet<S>>,
{
debug_assert!(self.raw_sets_cache.is_empty());
let mut last_key = (EMPTY_COUNT, false);
let mut last_pool_id = None;
for set in sets {
if last_key != (set.size, set.update_after_bind) || last_pool_id != Some(set.pool_id) {
if let Some(pool_id) = last_pool_id {
let bucket = self
.buckets
.get_mut(&last_key)
.expect("Set must be allocated from this allocator");
debug_assert!(u32::try_from(self.raw_sets_cache.len())
.ok()
.map_or(false, |count| count <= bucket.total));
bucket.free(device, self.raw_sets_cache.drain(..), pool_id);
}
last_key = (set.size, set.update_after_bind);
last_pool_id = Some(set.pool_id);
}
self.raw_sets_cache.push(set.raw);
}
if let Some(pool_id) = last_pool_id {
let bucket = self
.buckets
.get_mut(&last_key)
.expect("Set must be allocated from this allocator");
debug_assert!(u32::try_from(self.raw_sets_cache.len())
.ok()
.map_or(false, |count| count <= bucket.total));
bucket.free(device, self.raw_sets_cache.drain(..), pool_id);
}
}
/// Perform cleanup to allow resources reuse.
///
/// # Safety
///
/// * Same `device` instance must be passed to all method calls of
/// one `DescriptorAllocator` instance.
pub unsafe fn cleanup<L>(&mut self, device: &impl DescriptorDevice<L, P, S>) {
for bucket in self.buckets.values_mut() {
bucket.cleanup(device)
}
self.buckets.retain(|_, bucket| !bucket.pools.is_empty());
}
}
/// Empty descriptor per_type.
const EMPTY_COUNT: DescriptorTotalCount = DescriptorTotalCount {
sampler: 0,
combined_image_sampler: 0,
sampled_image: 0,
storage_image: 0,
uniform_texel_buffer: 0,
storage_texel_buffer: 0,
uniform_buffer: 0,
storage_buffer: 0,
uniform_buffer_dynamic: 0,
storage_buffer_dynamic: 0,
input_attachment: 0,
acceleration_structure: 0,
inline_uniform_block_bytes: 0,
inline_uniform_block_bindings: 0,
};
struct Allocation<'a, S> {
update_after_bind: bool,
size: DescriptorTotalCount,
pool_id: u64,
sets: &'a mut Vec<DescriptorSet<S>>,
}
impl<S> Extend<S> for Allocation<'_, S> {
fn extend<T: IntoIterator<Item = S>>(&mut self, iter: T) {
let update_after_bind = self.update_after_bind;
let size = self.size;
let pool_id = self.pool_id;
self.sets.extend(iter.into_iter().map(|raw| DescriptorSet {
raw,
pool_id,
update_after_bind,
size,
}))
}
}