hal/vulkan: Replace gpu-alloc by gpu-allocator (#8158)

This commit is contained in:
David Duarte authored and GitHub committed 2025-12-12 19:10:42 +00:00
1 parent 2a190372f7
commit ea361fc114
9 files changed
+302 -390

No files matched your search

+8
View File
@@ -43,6 +43,14 @@ Bottom level categories:
### Major Changes
#### Switch from `gpu-alloc` to `gpu-allocator` in the `vulkan` backend
`gpu-allocator` is the allocator used in the `dx12` backend, allowing to configure
the allocator the same way in those two backends converging their behavior.
This also brings the `Device::generate_allocator_report` feature to
the vulkan backend.
#### `wgpu::Instance::enumerate_adapters` is now `async` & available on WebGPU
BREAKING CHANGE: `enumerate_adapters` is now `async`:
Generated
+1 -20
View File
@@ -1741,31 +1741,13 @@ dependencies = [
"gl_generator",
]
[[package]]
name = "gpu-alloc"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fbcd2dba93594b227a1f57ee09b8b9da8892c34d55aa332e034a228d0fe6a171"
dependencies = [
"bitflags 2.10.0",
"gpu-alloc-types",
]
[[package]]
name = "gpu-alloc-types"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "98ff03b468aa837d70984d55f5d3f846f6ec31fe34bbb97c4f85219caeee1ca4"
dependencies = [
"bitflags 2.10.0",
]
[[package]]
name = "gpu-allocator"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51255ea7cfaadb6c5f1528d43e92a82acb2b96c43365989a28b2d44ee38f8795"
dependencies = [
"ash",
"hashbrown 0.16.1",
"log",
"presser",
@@ -4865,7 +4847,6 @@ dependencies = [
"glutin",
"glutin-winit",
"glutin_wgl_sys 0.6.1",
"gpu-alloc",
"gpu-allocator",
"gpu-descriptor",
"hashbrown 0.16.1",
+5 -4
View File
@@ -208,17 +208,18 @@ objc = "0.2.5"
# Vulkan dependencies
android_system_properties = "0.1.1"
ash = "0.38"
gpu-alloc = "0.6"
gpu-descriptor = "0.3.2"
# DX12 dependencies
gpu-allocator = { version = "0.28", default-features = false, features = [
"hashbrown",
] }
range-alloc = "0.1"
mach-dxcompiler-rs = { version = "0.1.4", default-features = false } # remember to increase max_shader_model if applicable
windows-core = { version = "0.62", default-features = false }
# DX12 and Vulkan dependencies
gpu-allocator = { version = "0.28", default-features = false, features = [
"hashbrown",
] }
# Gles dependencies
khronos-egl = "6"
glow = "0.16"
+3 -3
View File
@@ -92,7 +92,6 @@ vulkan = [
"dep:arrayvec",
"dep:ash",
"dep:bytemuck",
"dep:gpu-alloc",
"dep:gpu-descriptor",
"dep:hashbrown",
"dep:libc",
@@ -103,6 +102,7 @@ vulkan = [
"dep:profiling",
"dep:smallvec",
"dep:windows",
"gpu-allocator/vulkan",
"windows/Win32",
]
gles = [
@@ -224,9 +224,10 @@ glow = { workspace = true, optional = true }
########################
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
# Backend: Vulkan and Dx12
gpu-allocator = { workspace = true, optional = true }
# Backend: Vulkan
ash = { workspace = true, optional = true }
gpu-alloc = { workspace = true, optional = true }
gpu-descriptor = { workspace = true, optional = true }
smallvec = { workspace = true, optional = true, features = ["union"] }
# Backend: GLES
@@ -253,7 +254,6 @@ windows-core = { workspace = true, optional = true }
# Backend: Dx12
bit-set = { workspace = true, optional = true }
range-alloc = { workspace = true, optional = true }
gpu-allocator = { workspace = true, optional = true, features = ["d3d12"] }
once_cell = { workspace = true, optional = true }
# backend: GLES
glutin_wgl_sys = { workspace = true, optional = true }
+5 -60
View File
@@ -7,6 +7,7 @@ use windows::Win32::Graphics::{Direct3D12, Dxgi};
use crate::{
auxil::dxgi::{name::ObjectExt as _, result::HResult as _},
dx12::conv,
AllocationSizes,
};
#[derive(Debug)]
@@ -74,36 +75,14 @@ impl Allocator {
memory_hints: &wgt::MemoryHints,
memory_budget_thresholds: wgt::MemoryBudgetThresholds,
) -> Result<Self, crate::DeviceError> {
// TODO: the allocator's configuration should take hardware capability into
// account.
const MB: u64 = 1024 * 1024;
let (device_memblock_size, host_memblock_size) = match memory_hints {
wgt::MemoryHints::Performance => (256 * MB, 64 * MB),
wgt::MemoryHints::MemoryUsage => (8 * MB, 4 * MB),
wgt::MemoryHints::Manual {
suballocated_device_memory_block_size,
} => {
// TODO: Would it be useful to expose the host size in memory hints
// instead of always using half of the device size?
let device_size = suballocated_device_memory_block_size.start;
let host_size = device_size / 2;
(device_size, host_size)
}
};
// gpu_allocator clamps the sizes between 4MiB and 256MiB, but we clamp them ourselves since we use
// the sizes when detecting high memory pressure and there is no way to query the values otherwise.
let device_memblock_size = device_memblock_size.clamp(4 * MB, 256 * MB);
let host_memblock_size = host_memblock_size.clamp(4 * MB, 256 * MB);
let allocation_sizes =
gpu_allocator::AllocationSizes::new(device_memblock_size, host_memblock_size);
let allocation_sizes = AllocationSizes::from_memory_hints(memory_hints);
let device_memblock_size = allocation_sizes.min_device_memblock_size;
let host_memblock_size = allocation_sizes.min_host_memblock_size;
let allocator_desc = gpu_allocator::d3d12::AllocatorCreateDesc {
device: gpu_allocator::d3d12::ID3D12DeviceVersion::Device(raw.clone()),
debug_settings: Default::default(),
allocation_sizes,
allocation_sizes: allocation_sizes.into(),
};
let allocator = gpu_allocator::d3d12::Allocator::new(&allocator_desc).inspect_err(|e| {
@@ -622,37 +601,3 @@ impl<'a> DeviceAllocationContext<'a> {
Ok(allocation_info)
}
}
impl From<gpu_allocator::AllocationError> for crate::DeviceError {
fn from(result: gpu_allocator::AllocationError) -> Self {
match result {
gpu_allocator::AllocationError::OutOfMemory => Self::OutOfMemory,
gpu_allocator::AllocationError::FailedToMap(e) => {
log::error!("DX12 gpu-allocator: Failed to map: {e}");
Self::Lost
}
gpu_allocator::AllocationError::NoCompatibleMemoryTypeFound => {
log::error!("DX12 gpu-allocator: No Compatible Memory Type Found");
Self::Lost
}
gpu_allocator::AllocationError::InvalidAllocationCreateDesc => {
log::error!("DX12 gpu-allocator: Invalid Allocation Creation Description");
Self::Lost
}
gpu_allocator::AllocationError::InvalidAllocatorCreateDesc(e) => {
log::error!("DX12 gpu-allocator: Invalid Allocator Creation Description: {e}");
Self::Lost
}
gpu_allocator::AllocationError::Internal(e) => {
log::error!("DX12 gpu-allocator: Internal Error: {e}");
Self::Lost
}
gpu_allocator::AllocationError::BarrierLayoutNeedsDevice10
| gpu_allocator::AllocationError::CastableFormatsRequiresEnhancedBarriers
| gpu_allocator::AllocationError::CastableFormatsRequiresAtLeastDevice12 => {
unreachable!()
}
}
}
}
+101
View File
@@ -381,6 +381,107 @@ pub enum DeviceError {
Unexpected,
}
#[cfg(any(dx12, vulkan))]
impl From<gpu_allocator::AllocationError> for DeviceError {
fn from(result: gpu_allocator::AllocationError) -> Self {
match result {
gpu_allocator::AllocationError::OutOfMemory => Self::OutOfMemory,
gpu_allocator::AllocationError::FailedToMap(e) => {
log::error!("gpu-allocator: Failed to map: {e}");
Self::Lost
}
gpu_allocator::AllocationError::NoCompatibleMemoryTypeFound => {
log::error!("gpu-allocator: No Compatible Memory Type Found");
Self::Lost
}
gpu_allocator::AllocationError::InvalidAllocationCreateDesc => {
log::error!("gpu-allocator: Invalid Allocation Creation Description");
Self::Lost
}
gpu_allocator::AllocationError::InvalidAllocatorCreateDesc(e) => {
log::error!("gpu-allocator: Invalid Allocator Creation Description: {e}");
Self::Lost
}
gpu_allocator::AllocationError::Internal(e) => {
log::error!("gpu-allocator: Internal Error: {e}");
Self::Lost
}
gpu_allocator::AllocationError::BarrierLayoutNeedsDevice10
| gpu_allocator::AllocationError::CastableFormatsRequiresEnhancedBarriers
| gpu_allocator::AllocationError::CastableFormatsRequiresAtLeastDevice12 => {
unreachable!()
}
}
}
}
// A copy of gpu_allocator::AllocationSizes, allowing to read the configured value for
// the dx12 backend, we should instead add getters to gpu_allocator::AllocationSizes
// and remove this type.
// https://github.com/Traverse-Research/gpu-allocator/issues/295
#[cfg_attr(not(any(dx12, vulkan)), expect(dead_code))]
pub(crate) struct AllocationSizes {
pub(crate) min_device_memblock_size: u64,
pub(crate) max_device_memblock_size: u64,
pub(crate) min_host_memblock_size: u64,
pub(crate) max_host_memblock_size: u64,
}
impl AllocationSizes {
#[allow(dead_code)] // may be unused on some platforms
pub(crate) fn from_memory_hints(memory_hints: &wgt::MemoryHints) -> Self {
// TODO: the allocator's configuration should take hardware capability into
// account.
const MB: u64 = 1024 * 1024;
match memory_hints {
wgt::MemoryHints::Performance => Self {
min_device_memblock_size: 128 * MB,
max_device_memblock_size: 256 * MB,
min_host_memblock_size: 64 * MB,
max_host_memblock_size: 128 * MB,
},
wgt::MemoryHints::MemoryUsage => Self {
min_device_memblock_size: 8 * MB,
max_device_memblock_size: 64 * MB,
min_host_memblock_size: 4 * MB,
max_host_memblock_size: 32 * MB,
},
wgt::MemoryHints::Manual {
suballocated_device_memory_block_size,
} => {
// TODO: https://github.com/gfx-rs/wgpu/issues/8625
// Would it be useful to expose the host size in memory hints
// instead of always using half of the device size?
let device_size = suballocated_device_memory_block_size;
let host_size = device_size.start / 2..device_size.end / 2;
// gpu_allocator clamps the sizes between 4MiB and 256MiB, but we clamp them ourselves since we use
// the sizes when detecting high memory pressure and there is no way to query the values otherwise.
Self {
min_device_memblock_size: device_size.start.clamp(4 * MB, 256 * MB),
max_device_memblock_size: device_size.end.clamp(4 * MB, 256 * MB),
min_host_memblock_size: host_size.start.clamp(4 * MB, 256 * MB),
max_host_memblock_size: host_size.end.clamp(4 * MB, 256 * MB),
}
}
}
}
}
#[cfg(any(dx12, vulkan))]
impl From<AllocationSizes> for gpu_allocator::AllocationSizes {
fn from(value: AllocationSizes) -> gpu_allocator::AllocationSizes {
gpu_allocator::AllocationSizes::new(
value.min_device_memblock_size,
value.min_host_memblock_size,
)
.with_max_device_memblock_size(value.max_device_memblock_size)
.with_max_host_memblock_size(value.max_host_memblock_size)
}
}
#[allow(dead_code)] // may be unused on some platforms
#[cold]
fn hal_usage_error<T: fmt::Display>(txt: T) -> ! {
+13 -80
View File
@@ -4,7 +4,7 @@ use core::{ffi::CStr, marker::PhantomData};
use ash::{ext, google, khr, vk};
use parking_lot::Mutex;
use crate::vulkan::semaphore_list::SemaphoreList;
use crate::{vulkan::semaphore_list::SemaphoreList, AllocationSizes};
use super::semaphore_list::SemaphoreListMode;
@@ -2390,87 +2390,20 @@ impl super::Adapter {
signal_semaphores: Mutex::new(SemaphoreList::new(SemaphoreListMode::Signal)),
};
let mem_allocator = {
let limits = self.phd_capabilities.properties.limits;
let allocation_sizes = AllocationSizes::from_memory_hints(memory_hints).into();
// Note: the parameters here are not set in stone nor where they picked with
// strong confidence.
// `final_free_list_chunk` should be bigger than starting_free_list_chunk if
// we want the behavior of starting with smaller block sizes and using larger
// ones only after we observe that the small ones aren't enough, which I think
// is a good "I don't know what the workload is going to be like" approach.
//
// For reference, `VMA`, and `gpu_allocator` both start with 256 MB blocks
// (then VMA doubles the block size each time it needs a new block).
// At some point it would be good to experiment with real workloads
//
// TODO(#5925): The plan is to switch the Vulkan backend from `gpu_alloc` to
// `gpu_allocator` which has a different (simpler) set of configuration options.
//
// TODO: These parameters should take hardware capabilities into account.
let mb = 1024 * 1024;
let perf_cfg = gpu_alloc::Config {
starting_free_list_chunk: 128 * mb,
final_free_list_chunk: 512 * mb,
minimal_buddy_size: 1,
initial_buddy_dedicated_size: 8 * mb,
dedicated_threshold: 32 * mb,
preferred_dedicated_threshold: mb,
transient_dedicated_threshold: 128 * mb,
};
let mem_usage_cfg = gpu_alloc::Config {
starting_free_list_chunk: 8 * mb,
final_free_list_chunk: 64 * mb,
minimal_buddy_size: 1,
initial_buddy_dedicated_size: 8 * mb,
dedicated_threshold: 8 * mb,
preferred_dedicated_threshold: mb,
transient_dedicated_threshold: 16 * mb,
};
let config = match memory_hints {
wgt::MemoryHints::Performance => perf_cfg,
wgt::MemoryHints::MemoryUsage => mem_usage_cfg,
wgt::MemoryHints::Manual {
suballocated_device_memory_block_size,
} => gpu_alloc::Config {
starting_free_list_chunk: suballocated_device_memory_block_size.start,
final_free_list_chunk: suballocated_device_memory_block_size.end,
initial_buddy_dedicated_size: suballocated_device_memory_block_size.start,
..perf_cfg
},
};
let buffer_device_address = enabled_extensions.contains(&khr::buffer_device_address::NAME);
let mem_allocator =
gpu_allocator::vulkan::Allocator::new(&gpu_allocator::vulkan::AllocatorCreateDesc {
instance: self.instance.raw.clone(),
device: shared.raw.clone(),
physical_device: self.raw,
debug_settings: Default::default(),
buffer_device_address,
allocation_sizes,
})?;
let max_memory_allocation_size =
if let Some(maintenance_3) = self.phd_capabilities.maintenance_3 {
maintenance_3.max_memory_allocation_size
} else {
u64::MAX
};
let properties = gpu_alloc::DeviceProperties {
max_memory_allocation_count: limits.max_memory_allocation_count,
max_memory_allocation_size,
non_coherent_atom_size: limits.non_coherent_atom_size,
memory_types: memory_types
.iter()
.map(|memory_type| gpu_alloc::MemoryType {
props: gpu_alloc::MemoryPropertyFlags::from_bits_truncate(
memory_type.property_flags.as_raw() as u8,
),
heap: memory_type.heap_index,
})
.collect(),
memory_heaps: mem_properties
.memory_heaps_as_slice()
.iter()
.map(|&memory_heap| gpu_alloc::MemoryHeap {
size: memory_heap.size,
})
.collect(),
buffer_device_address: enabled_extensions
.contains(&khr::buffer_device_address::NAME),
};
gpu_alloc::GpuAllocator::new(config, properties)
};
let desc_allocator = gpu_descriptor::DescriptorAllocator::new(
if let Some(di) = self.phd_capabilities.descriptor_indexing {
di.max_update_after_bind_descriptors_in_all_pools
+153 -210
View File
@@ -235,105 +235,17 @@ impl super::DeviceShared {
buffer: &'a super::Buffer,
ranges: I,
) -> Option<impl 'a + Iterator<Item = vk::MappedMemoryRange<'a>>> {
let block = buffer.block.as_ref()?.lock();
let allocation = buffer.allocation.as_ref()?.lock();
let mask = self.private_caps.non_coherent_map_mask;
Some(ranges.map(move |range| {
vk::MappedMemoryRange::default()
.memory(*block.memory())
.offset((block.offset() + range.start) & !mask)
.memory(allocation.memory())
.offset((allocation.offset() + range.start) & !mask)
.size((range.end - range.start + mask) & !mask)
}))
}
}
impl gpu_alloc::MemoryDevice<vk::DeviceMemory> for super::DeviceShared {
unsafe fn allocate_memory(
&self,
size: u64,
memory_type: u32,
flags: gpu_alloc::AllocationFlags,
) -> Result<vk::DeviceMemory, gpu_alloc::OutOfMemory> {
let mut info = vk::MemoryAllocateInfo::default()
.allocation_size(size)
.memory_type_index(memory_type);
let mut info_flags;
if flags.contains(gpu_alloc::AllocationFlags::DEVICE_ADDRESS) {
info_flags = vk::MemoryAllocateFlagsInfo::default()
.flags(vk::MemoryAllocateFlags::DEVICE_ADDRESS);
info = info.push_next(&mut info_flags);
}
match unsafe { self.raw.allocate_memory(&info, None) } {
Ok(memory) => {
self.memory_allocations_counter.add(1);
Ok(memory)
}
Err(vk::Result::ERROR_OUT_OF_DEVICE_MEMORY) => {
Err(gpu_alloc::OutOfMemory::OutOfDeviceMemory)
}
Err(vk::Result::ERROR_OUT_OF_HOST_MEMORY) => {
Err(gpu_alloc::OutOfMemory::OutOfHostMemory)
}
// We don't use VK_KHR_external_memory
// VK_ERROR_INVALID_EXTERNAL_HANDLE
// We don't use VK_KHR_buffer_device_address
// VK_ERROR_INVALID_OPAQUE_CAPTURE_ADDRESS_KHR
Err(err) => handle_unexpected(err),
}
}
unsafe fn deallocate_memory(&self, memory: vk::DeviceMemory) {
self.memory_allocations_counter.sub(1);
unsafe { self.raw.free_memory(memory, None) };
}
unsafe fn map_memory(
&self,
memory: &mut vk::DeviceMemory,
offset: u64,
size: u64,
) -> Result<ptr::NonNull<u8>, gpu_alloc::DeviceMapError> {
match unsafe {
self.raw
.map_memory(*memory, offset, size, vk::MemoryMapFlags::empty())
} {
Ok(ptr) => Ok(ptr::NonNull::new(ptr.cast::<u8>())
.expect("Pointer to memory mapping must not be null")),
Err(vk::Result::ERROR_OUT_OF_DEVICE_MEMORY) => {
Err(gpu_alloc::DeviceMapError::OutOfDeviceMemory)
}
Err(vk::Result::ERROR_OUT_OF_HOST_MEMORY) => {
Err(gpu_alloc::DeviceMapError::OutOfHostMemory)
}
Err(vk::Result::ERROR_MEMORY_MAP_FAILED) => Err(gpu_alloc::DeviceMapError::MapFailed),
Err(err) => handle_unexpected(err),
}
}
unsafe fn unmap_memory(&self, memory: &mut vk::DeviceMemory) {
unsafe { self.raw.unmap_memory(*memory) };
}
unsafe fn invalidate_memory_ranges(
&self,
_ranges: &[gpu_alloc::MappedMemoryRange<'_, vk::DeviceMemory>],
) -> Result<(), gpu_alloc::OutOfMemory> {
// should never be called
unimplemented!()
}
unsafe fn flush_memory_ranges(
&self,
_ranges: &[gpu_alloc::MappedMemoryRange<'_, vk::DeviceMemory>],
) -> Result<(), gpu_alloc::OutOfMemory> {
// should never be called
unimplemented!()
}
}
impl
gpu_descriptor::DescriptorDevice<vk::DescriptorSetLayout, vk::DescriptorPool, vk::DescriptorSet>
for super::DeviceShared
@@ -966,59 +878,49 @@ impl crate::Device for super::Device {
.create_buffer(&vk_info, None)
.map_err(super::map_host_device_oom_and_ioca_err)?
};
let req = unsafe { self.shared.raw.get_buffer_memory_requirements(raw) };
let mut alloc_usage = if desc
.usage
.intersects(wgt::BufferUses::MAP_READ | wgt::BufferUses::MAP_WRITE)
{
let mut flags = gpu_alloc::UsageFlags::HOST_ACCESS;
//TODO: find a way to use `crate::MemoryFlags::PREFER_COHERENT`
flags.set(
gpu_alloc::UsageFlags::DOWNLOAD,
desc.usage.contains(wgt::BufferUses::MAP_READ),
);
flags.set(
gpu_alloc::UsageFlags::UPLOAD,
desc.usage.contains(wgt::BufferUses::MAP_WRITE),
);
flags
} else {
gpu_alloc::UsageFlags::FAST_DEVICE_ACCESS
let requirements = unsafe { self.shared.raw.get_buffer_memory_requirements(raw) };
let is_cpu_read = desc.usage.contains(wgt::BufferUses::MAP_READ);
let is_cpu_write = desc.usage.contains(wgt::BufferUses::MAP_WRITE);
let location = match (is_cpu_read, is_cpu_write) {
(true, true) => gpu_allocator::MemoryLocation::CpuToGpu,
(true, false) => gpu_allocator::MemoryLocation::GpuToCpu,
(false, true) => gpu_allocator::MemoryLocation::CpuToGpu,
(false, false) => gpu_allocator::MemoryLocation::GpuOnly,
};
alloc_usage.set(
gpu_alloc::UsageFlags::TRANSIENT,
desc.memory_flags.contains(crate::MemoryFlags::TRANSIENT),
);
let needs_host_access = alloc_usage.contains(gpu_alloc::UsageFlags::HOST_ACCESS);
let needs_host_access = is_cpu_read || is_cpu_write;
self.error_if_would_oom_on_resource_allocation(needs_host_access, req.size)
self.error_if_would_oom_on_resource_allocation(needs_host_access, requirements.size)
.inspect_err(|_| {
unsafe { self.shared.raw.destroy_buffer(raw, None) };
})?;
let alignment_mask = req.alignment - 1;
let name = desc.label.unwrap_or("Unlabeled buffer");
let block = unsafe {
self.mem_allocator.lock().alloc(
&*self.shared,
gpu_alloc::Request {
size: req.size,
align_mask: alignment_mask,
usage: alloc_usage,
memory_types: req.memory_type_bits & self.valid_ash_memory_types,
let allocation = self
.mem_allocator
.lock()
.allocate(&gpu_allocator::vulkan::AllocationCreateDesc {
name,
requirements: vk::MemoryRequirements {
memory_type_bits: requirements.memory_type_bits & self.valid_ash_memory_types,
..requirements
},
)
}
.inspect_err(|_| {
unsafe { self.shared.raw.destroy_buffer(raw, None) };
})?;
location,
linear: true, // Buffers are always linear
allocation_scheme: gpu_allocator::vulkan::AllocationScheme::GpuAllocatorManaged,
})
.inspect_err(|_| {
unsafe { self.shared.raw.destroy_buffer(raw, None) };
})?;
unsafe {
self.shared
.raw
.bind_buffer_memory(raw, *block.memory(), block.offset())
.bind_buffer_memory(raw, allocation.memory(), allocation.offset())
}
.map_err(super::map_host_device_oom_and_ioca_err)
.inspect_err(|_| {
@@ -1029,23 +931,26 @@ impl crate::Device for super::Device {
unsafe { self.shared.set_object_name(raw, label) };
}
self.counters.buffer_memory.add(block.size() as isize);
self.counters.buffer_memory.add(allocation.size() as isize);
self.counters.buffers.add(1);
Ok(super::Buffer {
raw,
block: Some(Mutex::new(super::BufferMemoryBacking::Managed(block))),
allocation: Some(Mutex::new(super::BufferMemoryBacking::Managed(allocation))),
})
}
unsafe fn destroy_buffer(&self, buffer: super::Buffer) {
unsafe { self.shared.raw.destroy_buffer(buffer.raw, None) };
if let Some(block) = buffer.block {
let block = block.into_inner();
self.counters.buffer_memory.sub(block.size() as isize);
match block {
super::BufferMemoryBacking::Managed(block) => unsafe {
self.mem_allocator.lock().dealloc(&*self.shared, block)
},
if let Some(allocation) = buffer.allocation {
let allocation = allocation.into_inner();
self.counters.buffer_memory.sub(allocation.size() as isize);
match allocation {
super::BufferMemoryBacking::Managed(allocation) => {
let result = self.mem_allocator.lock().free(allocation);
if let Err(err) = result {
log::warn!("Failed to free buffer allocation: {err}");
}
}
super::BufferMemoryBacking::VulkanMemory { memory, .. } => unsafe {
self.shared.raw.free_memory(memory, None);
},
@@ -1064,15 +969,22 @@ impl crate::Device for super::Device {
buffer: &super::Buffer,
range: crate::MemoryRange,
) -> Result<crate::BufferMapping, crate::DeviceError> {
if let Some(ref block) = buffer.block {
let size = range.end - range.start;
let mut block = block.lock();
if let super::BufferMemoryBacking::Managed(ref mut block) = *block {
let ptr = unsafe { block.map(&*self.shared, range.start, size as usize)? };
let is_coherent = block
.props()
.contains(gpu_alloc::MemoryPropertyFlags::HOST_COHERENT);
Ok(crate::BufferMapping { ptr, is_coherent })
if let Some(ref allocation) = buffer.allocation {
let mut allocation = allocation.lock();
if let super::BufferMemoryBacking::Managed(ref mut allocation) = *allocation {
let is_coherent = allocation
.memory_properties()
.contains(vk::MemoryPropertyFlags::HOST_COHERENT);
Ok(crate::BufferMapping {
ptr: unsafe {
allocation
.mapped_ptr()
.unwrap()
.cast()
.offset(range.start as isize)
},
is_coherent,
})
} else {
crate::hal_usage_error("tried to map externally created buffer")
}
@@ -1080,14 +992,10 @@ impl crate::Device for super::Device {
crate::hal_usage_error("tried to map external buffer")
}
}
unsafe fn unmap_buffer(&self, buffer: &super::Buffer) {
if let Some(ref block) = buffer.block {
match &mut *block.lock() {
super::BufferMemoryBacking::Managed(block) => unsafe { block.unmap(&*self.shared) },
super::BufferMemoryBacking::VulkanMemory { .. } => {
crate::hal_usage_error("tried to unmap externally created buffer")
}
};
if buffer.allocation.is_some() {
// gpu-allocator maps the buffer when allocated and unmap it when free'd
} else {
crate::hal_usage_error("tried to unmap external buffer")
}
@@ -1135,27 +1043,32 @@ impl crate::Device for super::Device {
unsafe { self.shared.raw.destroy_image(image.raw, None) };
})?;
let block = unsafe {
self.mem_allocator.lock().alloc(
&*self.shared,
gpu_alloc::Request {
size: image.requirements.size,
align_mask: image.requirements.alignment - 1,
usage: gpu_alloc::UsageFlags::FAST_DEVICE_ACCESS,
memory_types: image.requirements.memory_type_bits & self.valid_ash_memory_types,
},
)
}
.inspect_err(|_| {
unsafe { self.shared.raw.destroy_image(image.raw, None) };
})?;
let name = desc.label.unwrap_or("Unlabeled texture");
self.counters.texture_memory.add(block.size() as isize);
let allocation = self
.mem_allocator
.lock()
.allocate(&gpu_allocator::vulkan::AllocationCreateDesc {
name,
requirements: vk::MemoryRequirements {
memory_type_bits: image.requirements.memory_type_bits
& self.valid_ash_memory_types,
..image.requirements
},
location: gpu_allocator::MemoryLocation::GpuOnly,
linear: false,
allocation_scheme: gpu_allocator::vulkan::AllocationScheme::GpuAllocatorManaged,
})
.inspect_err(|_| {
unsafe { self.shared.raw.destroy_image(image.raw, None) };
})?;
self.counters.texture_memory.add(allocation.size() as isize);
unsafe {
self.shared
.raw
.bind_image_memory(image.raw, *block.memory(), block.offset())
.bind_image_memory(image.raw, allocation.memory(), allocation.offset())
}
.map_err(super::map_host_device_oom_err)
.inspect_err(|_| {
@@ -1163,7 +1076,12 @@ impl crate::Device for super::Device {
})?;
Ok(unsafe {
self.texture_from_raw(image.raw, desc, None, super::TextureMemory::Block(block))
self.texture_from_raw(
image.raw,
desc,
None,
super::TextureMemory::Allocation(allocation),
)
})
}
@@ -1173,10 +1091,13 @@ impl crate::Device for super::Device {
}
match texture.memory {
super::TextureMemory::Block(block) => unsafe {
self.counters.texture_memory.sub(block.size() as isize);
self.mem_allocator.lock().dealloc(&*self.shared, block);
},
super::TextureMemory::Allocation(allocation) => {
self.counters.texture_memory.sub(allocation.size() as isize);
let result = self.mem_allocator.lock().free(allocation);
if let Err(err) = result {
log::warn!("Failed to free texture allocation: {err}");
}
}
super::TextureMemory::Dedicated(memory) => unsafe {
self.shared.raw.free_memory(memory, None);
},
@@ -2548,32 +2469,35 @@ impl crate::Device for super::Device {
.raw
.create_buffer(&vk_buffer_info, None)
.map_err(super::map_host_device_oom_and_ioca_err)?;
let req = self.shared.raw.get_buffer_memory_requirements(raw_buffer);
self.error_if_would_oom_on_resource_allocation(false, req.size)
let requirements = self.shared.raw.get_buffer_memory_requirements(raw_buffer);
self.error_if_would_oom_on_resource_allocation(false, requirements.size)
.inspect_err(|_| {
self.shared.raw.destroy_buffer(raw_buffer, None);
})?;
let block = self
let name = desc
.label
.unwrap_or("Unlabeled acceleration structure buffer");
let allocation = self
.mem_allocator
.lock()
.alloc(
&*self.shared,
gpu_alloc::Request {
size: req.size,
align_mask: req.alignment - 1,
usage: gpu_alloc::UsageFlags::FAST_DEVICE_ACCESS,
memory_types: req.memory_type_bits & self.valid_ash_memory_types,
},
)
.allocate(&gpu_allocator::vulkan::AllocationCreateDesc {
name,
requirements,
location: gpu_allocator::MemoryLocation::GpuOnly,
linear: true, // Buffers are always linear
allocation_scheme: gpu_allocator::vulkan::AllocationScheme::GpuAllocatorManaged,
})
.inspect_err(|_| {
self.shared.raw.destroy_buffer(raw_buffer, None);
})?;
self.shared
.raw
.bind_buffer_memory(raw_buffer, *block.memory(), block.offset())
.bind_buffer_memory(raw_buffer, allocation.memory(), allocation.offset())
.map_err(super::map_host_device_oom_and_ioca_err)
.inspect_err(|_| {
self.shared.raw.destroy_buffer(raw_buffer, None);
@@ -2626,7 +2550,7 @@ impl crate::Device for super::Device {
Ok(super::AccelerationStructure {
raw: raw_acceleration_structure,
buffer: raw_buffer,
block: Mutex::new(block),
allocation,
compacted_size_query: pool,
})
}
@@ -2650,9 +2574,13 @@ impl crate::Device for super::Device {
self.shared
.raw
.destroy_buffer(acceleration_structure.buffer, None);
self.mem_allocator
let result = self
.mem_allocator
.lock()
.dealloc(&*self.shared, acceleration_structure.block.into_inner());
.free(acceleration_structure.allocation);
if let Err(err) = result {
log::warn!("Failed to free buffer acceleration structure: {err}");
}
if let Some(query) = acceleration_structure.compacted_size_query {
self.shared.raw.destroy_query_pool(query, None)
}
@@ -2667,6 +2595,39 @@ impl crate::Device for super::Device {
self.counters.as_ref().clone()
}
fn generate_allocator_report(&self) -> Option<wgt::AllocatorReport> {
let gpu_allocator::AllocatorReport {
allocations,
blocks,
total_allocated_bytes,
total_capacity_bytes,
} = self.mem_allocator.lock().generate_report();
let allocations = allocations
.into_iter()
.map(|alloc| wgt::AllocationReport {
name: alloc.name,
offset: alloc.offset,
size: alloc.size,
})
.collect();
let blocks = blocks
.into_iter()
.map(|block| wgt::MemoryBlockReport {
size: block.size,
allocations: block.allocations.clone(),
})
.collect();
Some(wgt::AllocatorReport {
allocations,
blocks,
total_allocated_bytes,
total_reserved_bytes: total_capacity_bytes,
})
}
fn tlas_instance_to_bytes(&self, instance: TlasInstance) -> Vec<u8> {
const MAX_U24: u32 = (1u32 << 24u32) - 1u32;
let temp = RawTlasInstance {
@@ -2805,24 +2766,6 @@ impl super::DeviceShared {
}
}
impl From<gpu_alloc::AllocationError> for crate::DeviceError {
fn from(error: gpu_alloc::AllocationError) -> Self {
use gpu_alloc::AllocationError as Ae;
match error {
Ae::OutOfDeviceMemory | Ae::OutOfHostMemory | Ae::TooManyObjects => Self::OutOfMemory,
Ae::NoCompatibleMemoryTypes => crate::hal_usage_error(error),
}
}
}
impl From<gpu_alloc::MapError> for crate::DeviceError {
fn from(error: gpu_alloc::MapError) -> Self {
use gpu_alloc::MapError as Me;
match error {
Me::OutOfDeviceMemory | Me::OutOfHostMemory | Me::MapFailed => Self::OutOfMemory,
Me::NonHostVisible | Me::AlreadyMapped => crate::hal_usage_error(error),
}
}
}
impl From<gpu_descriptor::AllocationError> for crate::DeviceError {
fn from(error: gpu_descriptor::AllocationError) -> Self {
use gpu_descriptor::AllocationError as Ae;
+13 -13
View File
@@ -505,8 +505,7 @@ impl Drop for DeviceShared {
}
pub struct Device {
shared: Arc<DeviceShared>,
mem_allocator: Mutex<gpu_alloc::GpuAllocator<vk::DeviceMemory>>,
mem_allocator: Mutex<gpu_allocator::vulkan::Allocator>,
desc_allocator:
Mutex<gpu_descriptor::DescriptorAllocator<vk::DescriptorPool, vk::DescriptorSet>>,
valid_ash_memory_types: u32,
@@ -514,11 +513,13 @@ pub struct Device {
#[cfg(feature = "renderdoc")]
render_doc: crate::auxil::renderdoc::RenderDoc,
counters: Arc<wgt::HalCounters>,
// Struct members are dropped from first to last, put the Device last to ensure that
// all resources that depends on it are destroyed before it like the mem_allocator
shared: Arc<DeviceShared>,
}
impl Drop for Device {
fn drop(&mut self) {
unsafe { self.mem_allocator.lock().cleanup(&*self.shared) };
unsafe { self.desc_allocator.lock().cleanup(&*self.shared) };
}
}
@@ -619,7 +620,7 @@ impl Drop for Queue {
}
#[derive(Debug)]
enum BufferMemoryBacking {
Managed(gpu_alloc::MemoryBlock<vk::DeviceMemory>),
Managed(gpu_allocator::vulkan::Allocation),
VulkanMemory {
memory: vk::DeviceMemory,
offset: u64,
@@ -627,10 +628,10 @@ enum BufferMemoryBacking {
},
}
impl BufferMemoryBacking {
fn memory(&self) -> &vk::DeviceMemory {
fn memory(&self) -> vk::DeviceMemory {
match self {
Self::Managed(m) => m.memory(),
Self::VulkanMemory { memory, .. } => memory,
Self::Managed(m) => unsafe { m.memory() },
Self::VulkanMemory { memory, .. } => *memory,
}
}
fn offset(&self) -> u64 {
@@ -649,7 +650,7 @@ impl BufferMemoryBacking {
#[derive(Debug)]
pub struct Buffer {
raw: vk::Buffer,
block: Option<Mutex<BufferMemoryBacking>>,
allocation: Option<Mutex<BufferMemoryBacking>>,
}
impl Buffer {
/// # Safety
@@ -659,7 +660,7 @@ impl Buffer {
pub unsafe fn from_raw(vk_buffer: vk::Buffer) -> Self {
Self {
raw: vk_buffer,
block: None,
allocation: None,
}
}
/// # Safety
@@ -674,7 +675,7 @@ impl Buffer {
) -> Self {
Self {
raw: vk_buffer,
block: Some(Mutex::new(BufferMemoryBacking::VulkanMemory {
allocation: Some(Mutex::new(BufferMemoryBacking::VulkanMemory {
memory,
offset,
size,
@@ -689,7 +690,7 @@ impl crate::DynBuffer for Buffer {}
pub struct AccelerationStructure {
raw: vk::AccelerationStructureKHR,
buffer: vk::Buffer,
block: Mutex<gpu_alloc::MemoryBlock<vk::DeviceMemory>>,
allocation: gpu_allocator::vulkan::Allocation,
compacted_size_query: Option<vk::QueryPool>,
}
@@ -698,7 +699,7 @@ impl crate::DynAccelerationStructure for AccelerationStructure {}
#[derive(Debug)]
pub enum TextureMemory {
// shared memory in GPU allocator (owned by wgpu-hal)
Block(gpu_alloc::MemoryBlock<vk::DeviceMemory>),
Allocation(gpu_allocator::vulkan::Allocation),
// dedicated memory (owned by wgpu-hal)
Dedicated(vk::DeviceMemory),
@@ -711,7 +712,6 @@ pub enum TextureMemory {
pub struct Texture {
raw: vk::Image,
memory: TextureMemory,
format: wgt::TextureFormat,
copy_size: crate::CopyExtent,
identity: ResourceIdentity<vk::Image>,