diff --git a/Cargo.lock b/Cargo.lock index 350062365..86bd7a801 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -710,19 +710,6 @@ dependencies = [ "raw-window-handle", ] -[[package]] -name = "gfx-memory" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dccdda5d2b39412f4ca2cb15c70b5a82783a86b0606f5e985342754c8ed88f05" -dependencies = [ - "bit-set", - "fxhash", - "gfx-hal", - "log", - "slab", -] - [[package]] name = "gimli" version = "0.23.0" @@ -772,6 +759,23 @@ dependencies = [ "web-sys", ] +[[package]] +name = "gpu-alloc" +version = "0.1.0" +source = "git+https://github.com/zakarumych/gpu-alloc?rev=2bb7ef947443b1b13e34d3b98a116da6d69ac15b#2bb7ef947443b1b13e34d3b98a116da6d69ac15b" +dependencies = [ + "bitflags", + "gpu-alloc-types", +] + +[[package]] +name = "gpu-alloc-types" +version = "0.1.0" +source = "git+https://github.com/zakarumych/gpu-alloc?rev=2bb7ef947443b1b13e34d3b98a116da6d69ac15b#2bb7ef947443b1b13e34d3b98a116da6d69ac15b" +dependencies = [ + "bitflags", +] + [[package]] name = "hashbrown" version = "0.8.2" @@ -2172,7 +2176,7 @@ dependencies = [ "gfx-backend-vulkan", "gfx-descriptor", "gfx-hal", - "gfx-memory", + "gpu-alloc", "loom", "naga", "parking_lot 0.11.0", diff --git a/wgpu-core/Cargo.toml b/wgpu-core/Cargo.toml index 6c700a407..3f3e64dbf 100644 --- a/wgpu-core/Cargo.toml +++ b/wgpu-core/Cargo.toml @@ -35,7 +35,7 @@ smallvec = "1" tracing = { version = "0.1", default-features = false, features = ["std"] } thiserror = "1" gfx-descriptor = "0.2" -gfx-memory = "0.2" +gpu-alloc = { git = "https://github.com/zakarumych/gpu-alloc", rev = "2bb7ef947443b1b13e34d3b98a116da6d69ac15b" } [dependencies.naga] version = "0.2" diff --git a/wgpu-core/src/device/alloc.rs b/wgpu-core/src/device/alloc.rs new file mode 100644 index 000000000..acfbb7ea6 --- /dev/null +++ b/wgpu-core/src/device/alloc.rs @@ -0,0 +1,294 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use super::DeviceError; +use hal::device::Device; +use std::{borrow::Cow, fmt, iter, ptr::NonNull, sync::Arc}; + +pub struct MemoryAllocator(gpu_alloc::GpuAllocator>); +#[derive(Debug)] +pub struct MemoryBlock(gpu_alloc::MemoryBlock>); +struct MemoryDevice<'a, B: hal::Backend>(&'a B::Device); + +//TODO: https://github.com/zakarumych/gpu-alloc/issues/9 +impl fmt::Debug for MemoryAllocator { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "MemoryAllocator") + } +} + +impl MemoryAllocator { + pub fn new(mem_props: hal::adapter::MemoryProperties, limits: hal::Limits) -> Self { + let mem_config = gpu_alloc::Config { + dedicated_treshold: 32 << 20, + preferred_dedicated_treshold: 8 << 20, + transient_dedicated_treshold: 128 << 20, + linear_chunk: 128 << 20, + minimal_buddy_size: 1 << 10, + initial_buddy_dedicated_size: 8 << 20, + }; + let properties = gpu_alloc::DeviceProperties { + memory_types: Cow::Owned( + mem_props + .memory_types + .iter() + .map(|mt| gpu_alloc::MemoryType { + heap: mt.heap_index as u32, + props: gpu_alloc::MemoryPropertyFlags::from_bits_truncate( + mt.properties.bits() as u8, + ), + }) + .collect::>(), + ), + memory_heaps: Cow::Owned( + mem_props + .memory_heaps + .iter() + .map(|mh| gpu_alloc::MemoryHeap { size: *mh }) + .collect::>(), + ), + max_memory_allocation_count: if limits.max_memory_allocation_count == 0 { + tracing::warn!("max_memory_allocation_count is not set by gfx-rs backend"); + !0 + } else { + limits.max_memory_allocation_count.min(!0u32 as usize) as u32 + }, + max_memory_allocation_size: !0, + non_coherent_atom_size: limits.non_coherent_atom_size as u64, + buffer_device_address: false, + }; + MemoryAllocator(gpu_alloc::GpuAllocator::new(mem_config, properties)) + } + + pub fn allocate( + &mut self, + device: &B::Device, + requirements: hal::memory::Requirements, + usage: gpu_alloc::UsageFlags, + ) -> Result, DeviceError> { + assert!(requirements.alignment.is_power_of_two()); + let request = gpu_alloc::Request { + size: requirements.size, + align_mask: requirements.alignment - 1, + memory_types: requirements.type_mask, + usage, + }; + + unsafe { self.0.alloc(&MemoryDevice::(device), request) } + .map(MemoryBlock) + .map_err(|err| match err { + gpu_alloc::AllocationError::OutOfHostMemory + | gpu_alloc::AllocationError::OutOfDeviceMemory => DeviceError::OutOfMemory, + _ => panic!("Unable to allocate memory: {:?}", err), + }) + } + + pub fn free(&mut self, device: &B::Device, block: MemoryBlock) { + unsafe { self.0.dealloc(&MemoryDevice::(device), block.0) } + } + + pub fn clear(&mut self, device: &B::Device) { + unsafe { self.0.cleanup(&MemoryDevice::(device)) } + } +} + +impl MemoryBlock { + pub fn bind_buffer( + &self, + device: &B::Device, + buffer: &mut B::Buffer, + ) -> Result<(), DeviceError> { + unsafe { + device + .bind_buffer_memory(self.0.memory(), self.0.offset(), buffer) + .map_err(DeviceError::from_bind) + } + } + + pub fn bind_image(&self, device: &B::Device, image: &mut B::Image) -> Result<(), DeviceError> { + unsafe { + device + .bind_image_memory(self.0.memory(), self.0.offset(), image) + .map_err(DeviceError::from_bind) + } + } + + pub fn is_coherent(&self) -> bool { + self.0 + .props() + .contains(gpu_alloc::MemoryPropertyFlags::HOST_COHERENT) + } + + pub fn map( + &mut self, + device: &B::Device, + inner_offset: wgt::BufferAddress, + size: wgt::BufferAddress, + ) -> Result, DeviceError> { + let offset = inner_offset; + unsafe { + self.0 + .map(&MemoryDevice::(device), offset, size as usize) + .map_err(DeviceError::from) + } + } + + pub fn unmap(&mut self, device: &B::Device) { + unsafe { self.0.unmap(&MemoryDevice::(device)) } + } + + pub fn write_bytes( + &mut self, + device: &B::Device, + inner_offset: wgt::BufferAddress, + data: &[u8], + ) -> Result<(), DeviceError> { + let offset = inner_offset; + unsafe { + self.0 + .write_bytes(&MemoryDevice::(device), offset, data) + .map_err(DeviceError::from) + } + } + + pub fn read_bytes( + &mut self, + device: &B::Device, + inner_offset: wgt::BufferAddress, + data: &mut [u8], + ) -> Result<(), DeviceError> { + let offset = inner_offset; + unsafe { + self.0 + .read_bytes(&MemoryDevice::(device), offset, data) + .map_err(DeviceError::from) + } + } + + fn segment( + &self, + inner_offset: wgt::BufferAddress, + size: Option, + ) -> hal::memory::Segment { + hal::memory::Segment { + offset: self.0.offset() + inner_offset, + size: size.or(Some(self.0.size())), + } + } + + pub fn flush_range( + &self, + device: &B::Device, + inner_offset: wgt::BufferAddress, + size: Option, + ) -> Result<(), DeviceError> { + let segment = self.segment(inner_offset, size); + unsafe { + device + .flush_mapped_memory_ranges(iter::once((&**self.0.memory(), segment))) + .or(Err(DeviceError::OutOfMemory)) + } + } + + pub fn invalidate_range( + &self, + device: &B::Device, + inner_offset: wgt::BufferAddress, + size: Option, + ) -> Result<(), DeviceError> { + let segment = self.segment(inner_offset, size); + unsafe { + device + .invalidate_mapped_memory_ranges(iter::once((&**self.0.memory(), segment))) + .or(Err(DeviceError::OutOfMemory)) + } + } +} + +impl gpu_alloc::MemoryDevice> for MemoryDevice<'_, B> { + #[cfg_attr(feature = "tracing", tracing::instrument(skip(self)))] + unsafe fn allocate_memory( + &self, + size: u64, + memory_type: u32, + flags: gpu_alloc::AllocationFlags, + ) -> Result, gpu_alloc::OutOfMemory> { + assert!(flags.is_empty()); + + self.0 + .allocate_memory(hal::MemoryTypeId(memory_type as _), size) + .map(Arc::new) + .map_err(|_| gpu_alloc::OutOfMemory::OutOfDeviceMemory) + } + + #[cfg_attr(feature = "tracing", tracing::instrument(skip(self)))] + unsafe fn deallocate_memory(&self, memory: Arc) { + let memory = Arc::try_unwrap(memory).expect("Memory must not be used anywhere"); + self.0.free_memory(memory); + } + + #[cfg_attr(feature = "tracing", tracing::instrument(skip(self)))] + unsafe fn map_memory( + &self, + memory: &Arc, + offset: u64, + size: u64, + ) -> Result, gpu_alloc::DeviceMapError> { + match self.0.map_memory( + memory, + hal::memory::Segment { + offset, + size: Some(size), + }, + ) { + Ok(ptr) => Ok(NonNull::new(ptr).expect("Pointer to memory mapping must not be null")), + Err(hal::device::MapError::OutOfMemory(_)) => { + Err(gpu_alloc::DeviceMapError::OutOfDeviceMemory) + } + Err(hal::device::MapError::MappingFailed) => Err(gpu_alloc::DeviceMapError::MapFailed), + Err(other) => panic!("Unexpected map error: {:?}", other), + } + } + + #[cfg_attr(feature = "tracing", tracing::instrument(skip(self)))] + unsafe fn unmap_memory(&self, memory: &Arc) { + self.0.unmap_memory(memory); + } + + #[cfg_attr(feature = "tracing", tracing::instrument(skip(self)))] + unsafe fn invalidate_memory_ranges( + &self, + ranges: &[gpu_alloc::MappedMemoryRange<'_, Arc>], + ) -> Result<(), gpu_alloc::OutOfMemory> { + self.0 + .invalidate_mapped_memory_ranges(ranges.iter().map(|range| { + ( + &**range.memory, + hal::memory::Segment { + offset: range.offset, + size: Some(range.size), + }, + ) + })) + .map_err(|_| gpu_alloc::OutOfMemory::OutOfHostMemory) + } + + #[cfg_attr(feature = "tracing", tracing::instrument(skip(self)))] + unsafe fn flush_memory_ranges( + &self, + ranges: &[gpu_alloc::MappedMemoryRange<'_, Arc>], + ) -> Result<(), gpu_alloc::OutOfMemory> { + self.0 + .flush_mapped_memory_ranges(ranges.iter().map(|range| { + ( + &**range.memory, + hal::memory::Segment { + offset: range.offset, + size: Some(range.size), + }, + ) + })) + .map_err(|_| gpu_alloc::OutOfMemory::OutOfHostMemory) + } +} diff --git a/wgpu-core/src/device/life.rs b/wgpu-core/src/device/life.rs index 0d0fbc5a8..4d3c73ac3 100644 --- a/wgpu-core/src/device/life.rs +++ b/wgpu-core/src/device/life.rs @@ -5,7 +5,7 @@ #[cfg(feature = "trace")] use crate::device::trace; use crate::{ - device::{queue::TempResource, DeviceError}, + device::{alloc, queue::TempResource, DeviceError}, hub::{GfxBackend, GlobalIdentityHandlerFactory, Hub, Token}, id, resource, track::TrackerSet, @@ -14,7 +14,6 @@ use crate::{ use copyless::VecHelper as _; use gfx_descriptor::{DescriptorAllocator, DescriptorSet}; -use gfx_memory::{Heaps, MemoryBlock}; use hal::device::Device as _; use parking_lot::Mutex; use thiserror::Error; @@ -84,8 +83,8 @@ impl SuspectedResources { /// A struct that keeps lists of resources that are no longer needed. #[derive(Debug)] struct NonReferencedResources { - buffers: Vec<(B::Buffer, MemoryBlock)>, - images: Vec<(B::Image, MemoryBlock)>, + buffers: Vec<(B::Buffer, alloc::MemoryBlock)>, + images: Vec<(B::Image, alloc::MemoryBlock)>, // Note: we keep the associated ID here in order to be able to check // at any point what resources are used in a submission. image_views: Vec<(id::Valid, B::ImageView)>, @@ -130,20 +129,20 @@ impl NonReferencedResources { unsafe fn clean( &mut self, device: &B::Device, - heaps_mutex: &Mutex>, + memory_allocator_mutex: &Mutex>, descriptor_allocator_mutex: &Mutex>, ) { if !self.buffers.is_empty() || !self.images.is_empty() { - let mut heaps = heaps_mutex.lock(); + let mut allocator = memory_allocator_mutex.lock(); for (raw, memory) in self.buffers.drain(..) { tracing::trace!("Buffer {:?} is destroyed with memory {:?}", raw, memory); device.destroy_buffer(raw); - heaps.free(device, memory); + allocator.free(device, memory); } for (raw, memory) in self.images.drain(..) { tracing::trace!("Image {:?} is destroyed with memory {:?}", raw, memory); device.destroy_image(raw); - heaps.free(device, memory); + allocator.free(device, memory); } } @@ -241,7 +240,7 @@ impl LifetimeTracker { index: SubmissionIndex, fence: B::Fence, new_suspects: &SuspectedResources, - temp_resources: impl Iterator, MemoryBlock)>, + temp_resources: impl Iterator, alloc::MemoryBlock)>, ) { let mut last_resources = NonReferencedResources::new(); for (res, memory) in temp_resources { @@ -334,12 +333,12 @@ impl LifetimeTracker { pub fn cleanup( &mut self, device: &B::Device, - heaps_mutex: &Mutex>, + memory_allocator_mutex: &Mutex>, descriptor_allocator_mutex: &Mutex>, ) { unsafe { self.free_resources - .clean(device, heaps_mutex, descriptor_allocator_mutex); + .clean(device, memory_allocator_mutex, descriptor_allocator_mutex); descriptor_allocator_mutex.lock().cleanup(device); } } @@ -347,7 +346,7 @@ impl LifetimeTracker { pub fn schedule_resource_destruction( &mut self, temp_resource: TempResource, - memory: MemoryBlock, + memory: alloc::MemoryBlock, last_submit_index: SubmissionIndex, ) { let resources = self @@ -725,14 +724,18 @@ impl LifetimeTracker { resource::BufferMapState::Waiting(pending_mapping) => pending_mapping, _ => panic!("No pending mapping."), }; - let status = if mapping.sub_range.size.map_or(true, |x| x != 0) { + let status = if mapping.range.start != mapping.range.end { tracing::debug!("Buffer {:?} map state -> Active", buffer_id); let host = mapping.op.host; - match super::map_buffer(raw, buffer, mapping.sub_range.clone(), host) { + let size = mapping.range.end - mapping.range.start; + match super::map_buffer(raw, buffer, mapping.range.start, size, host) { Ok(ptr) => { buffer.map_state = resource::BufferMapState::Active { ptr, - sub_range: mapping.sub_range, + sub_range: hal::buffer::SubRange { + offset: mapping.range.start, + size: Some(size), + }, host, }; resource::BufferMapAsyncStatus::Success diff --git a/wgpu-core/src/device/mod.rs b/wgpu-core/src/device/mod.rs index af8d4a12e..39e9e8155 100644 --- a/wgpu-core/src/device/mod.rs +++ b/wgpu-core/src/device/mod.rs @@ -19,7 +19,6 @@ use crate::{ use arrayvec::ArrayVec; use copyless::VecHelper as _; use gfx_descriptor::DescriptorAllocator; -use gfx_memory::{Block, Heaps}; use hal::{ command::CommandBuffer as _, device::Device as _, @@ -42,6 +41,7 @@ use std::{ sync::atomic::Ordering, }; +pub mod alloc; mod life; mod queue; #[cfg(any(feature = "trace", feature = "replay"))] @@ -163,34 +163,25 @@ type BufferMapPendingCallback = (resource::BufferMapOperation, resource::BufferM fn map_buffer( raw: &B::Device, buffer: &mut resource::Buffer, - sub_range: hal::buffer::SubRange, + offset: hal::buffer::Offset, + size: BufferAddress, kind: HostMap, ) -> Result, resource::BufferAccessError> { - let &mut (_, ref mut memory) = buffer + let &mut (_, ref mut block) = buffer .raw .as_mut() .ok_or(resource::BufferAccessError::Destroyed)?; - let (ptr, segment, needs_sync) = { - let segment = hal::memory::Segment { - offset: sub_range.offset, - size: sub_range.size, - }; - let mapped = memory.map(raw, segment)?; - let mr = mapped.range(); - let segment = hal::memory::Segment { - offset: mr.start, - size: Some(mr.end - mr.start), - }; - (mapped.ptr(), segment, !mapped.is_coherent()) - }; + let ptr = block.map(raw, offset, size).map_err(DeviceError::from)?; buffer.sync_mapped_writes = match kind { - HostMap::Read if needs_sync => unsafe { - raw.invalidate_mapped_memory_ranges(iter::once((memory.memory(), segment))) - .or(Err(DeviceError::OutOfMemory))?; + HostMap::Read if !block.is_coherent() => { + block.invalidate_range(raw, offset, Some(size))?; None - }, - HostMap::Write if needs_sync => Some(segment), + } + HostMap::Write if !block.is_coherent() => Some(hal::memory::Segment { + offset, + size: Some(size), + }), _ => None, }; Ok(ptr) @@ -200,16 +191,14 @@ fn unmap_buffer( raw: &B::Device, buffer: &mut resource::Buffer, ) -> Result<(), resource::BufferAccessError> { - let &(_, ref memory) = buffer + let &mut (_, ref mut block) = buffer .raw - .as_ref() + .as_mut() .ok_or(resource::BufferAccessError::Destroyed)?; if let Some(segment) = buffer.sync_mapped_writes.take() { - unsafe { - raw.flush_mapped_memory_ranges(iter::once((memory.memory(), segment))) - .or(Err(DeviceError::OutOfMemory))?; - } + block.flush_range(raw, segment.offset, segment.size)?; } + block.unmap(raw); Ok(()) } @@ -227,7 +216,7 @@ pub struct Device { pub(crate) adapter_id: Stored, pub(crate) queue_group: hal::queue::QueueGroup, pub(crate) cmd_allocator: command::CommandAllocator, - mem_allocator: Mutex>, + mem_allocator: Mutex>, desc_allocator: Mutex>, //Note: The submission index here corresponds to the last submission that is done. pub(crate) life_guard: LifeGuard, @@ -268,20 +257,8 @@ impl Device { ) -> Result { let cmd_allocator = command::CommandAllocator::new(queue_group.family, &raw) .or(Err(CreateDeviceError::OutOfMemory))?; - let heaps = unsafe { - Heaps::new( - &mem_props, - gfx_memory::GeneralConfig { - block_size_granularity: 0x100, - max_chunk_size: 0x100_0000, - min_device_allocation: 0x1_0000, - }, - gfx_memory::LinearConfig { - line_size: 0x100_0000, - }, - hal_limits.non_coherent_atom_size as u64, - ) - }; + + let mem_allocator = alloc::MemoryAllocator::new(mem_props, hal_limits); let descriptors = unsafe { DescriptorAllocator::new() }; #[cfg(not(feature = "trace"))] match trace_path { @@ -293,7 +270,7 @@ impl Device { raw, adapter_id, cmd_allocator, - mem_allocator: Mutex::new(heaps), + mem_allocator: Mutex::new(mem_allocator), desc_allocator: Mutex::new(descriptors), queue_group, life_guard: LifeGuard::new(), @@ -436,7 +413,7 @@ impl Device { &self, self_id: id::DeviceId, desc: &resource::BufferDescriptor, - memory_kind: gfx_memory::Kind, + transient: bool, ) -> Result, resource::CreateBufferError> { debug_assert_eq!(self_id.backend(), B::VARIANT); let (mut usage, _memory_properties) = conv::map_buffer_usage(desc.usage); @@ -446,27 +423,42 @@ impl Device { } let mem_usage = { - use gfx_memory::MemoryUsage; + use gpu_alloc::UsageFlags as Uf; use wgt::BufferUsage as Bu; - //TODO: use linear allocation when we can ensure the freeing is linear - if !desc.usage.intersects(Bu::MAP_READ | Bu::MAP_WRITE) { - MemoryUsage::Private - } else if (Bu::MAP_WRITE | Bu::COPY_SRC).contains(desc.usage) { - MemoryUsage::Staging { read_back: false } - } else if (Bu::MAP_READ | Bu::COPY_DST).contains(desc.usage) { - MemoryUsage::Staging { read_back: true } - } else { + let mut flags = Uf::empty(); + let map_flags = desc.usage & (Bu::MAP_READ | Bu::MAP_WRITE); + if !(desc.usage - map_flags).is_empty() { + flags |= Uf::FAST_DEVICE_ACCESS; + } + if transient { + flags |= Uf::TRANSIENT; + } + + if !map_flags.is_empty() { + let upload_usage = Bu::MAP_WRITE | Bu::COPY_SRC; + let download_usage = Bu::MAP_READ | Bu::COPY_DST; + + flags |= Uf::HOST_ACCESS; + if desc.usage.contains(upload_usage) { + flags |= Uf::UPLOAD; + } + if desc.usage.contains(download_usage) { + flags |= Uf::DOWNLOAD; + } + let is_native_only = self .features .contains(wgt::Features::MAPPABLE_PRIMARY_BUFFERS); - if !is_native_only { + if !is_native_only + && !upload_usage.contains(desc.usage) + && !download_usage.contains(desc.usage) + { return Err(resource::CreateBufferError::UsageMismatch(desc.usage)); } - MemoryUsage::Dynamic { - sparse_updates: false, - } } + + flags }; let mut buffer = unsafe { self.raw.create_buffer(desc.size.max(1), usage) }.map_err( @@ -480,19 +472,14 @@ impl Device { } let requirements = unsafe { self.raw.get_buffer_requirements(&buffer) }; - let memory = self + let block = self .mem_allocator .lock() - .allocate(&self.raw, &requirements, mem_usage, memory_kind) - .map_err(DeviceError::from_heaps)?; - unsafe { - self.raw - .bind_buffer_memory(memory.memory(), memory.segment().offset, &mut buffer) - } - .map_err(DeviceError::from_bind)?; + .allocate(&self.raw, requirements, mem_usage)?; + block.bind_buffer(&self.raw, &mut buffer)?; Ok(resource::Buffer { - raw: Some((buffer, memory)), + raw: Some((buffer, block)), device_id: Stored { value: id::Valid(self_id), ref_count: self.life_guard.add_ref(), @@ -572,24 +559,15 @@ impl Device { }; let requirements = unsafe { self.raw.get_image_requirements(&image) }; - let memory = self - .mem_allocator - .lock() - .allocate( - &self.raw, - &requirements, - gfx_memory::MemoryUsage::Private, - gfx_memory::Kind::General, - ) - .map_err(DeviceError::from_heaps)?; - unsafe { - self.raw - .bind_image_memory(memory.memory(), memory.segment().offset, &mut image) - } - .map_err(DeviceError::from_bind)?; + let block = self.mem_allocator.lock().allocate( + &self.raw, + requirements, + gpu_alloc::UsageFlags::FAST_DEVICE_ACCESS, + )?; + block.bind_image(&self.raw, &mut image)?; Ok(resource::Texture { - raw: Some((image, memory)), + raw: Some((image, block)), device_id: Stored { value: id::Valid(self_id), ref_count: self.life_guard.add_ref(), @@ -949,16 +927,18 @@ impl From for DeviceError { } } -impl DeviceError { - fn from_heaps(err: gfx_memory::HeapsError) -> Self { +impl From for DeviceError { + fn from(err: gpu_alloc::MapError) -> Self { match err { - gfx_memory::HeapsError::AllocationError(hal::device::AllocationError::OutOfMemory( - _, - )) => Self::OutOfMemory, - _ => panic!("Unable to allocate memory: {:?}", err), + gpu_alloc::MapError::OutOfDeviceMemory | gpu_alloc::MapError::OutOfHostMemory => { + DeviceError::OutOfMemory + } + _ => panic!("failed to map buffer: {}", err), } } +} +impl DeviceError { fn from_bind(err: hal::device::BindError) -> Self { match err { hal::device::BindError::OutOfMemory(_) => Self::OutOfMemory, @@ -1022,19 +1002,15 @@ impl Global { let device = device_guard .get(device_id) .map_err(|_| DeviceError::Invalid)?; - let mut buffer = device.create_buffer(device_id, desc, gfx_memory::Kind::General)?; + let mut buffer = device.create_buffer(device_id, desc, false)?; let ref_count = buffer.life_guard.add_ref(); let buffer_use = if !desc.mapped_at_creation { resource::BufferUse::EMPTY } else if desc.usage.contains(wgt::BufferUsage::MAP_WRITE) { // buffer is mappable, so we are just doing that at start - let ptr = map_buffer( - &device.raw, - &mut buffer, - hal::buffer::SubRange::WHOLE, - HostMap::Write, - )?; + let size = buffer.size; + let ptr = map_buffer(&device.raw, &mut buffer, 0, size, HostMap::Write)?; buffer.map_state = resource::BufferMapState::Active { ptr, @@ -1053,15 +1029,13 @@ impl Global { usage: wgt::BufferUsage::MAP_WRITE | wgt::BufferUsage::COPY_SRC, mapped_at_creation: false, }, - gfx_memory::Kind::Linear, + true, )?; let (stage_buffer, mut stage_memory) = stage.raw.unwrap(); - let mapped = stage_memory - .map(&device.raw, hal::memory::Segment::ALL) - .map_err(resource::BufferAccessError::from)?; + let ptr = stage_memory.map(&device.raw, 0, stage.size)?; buffer.map_state = resource::BufferMapState::Init { - ptr: mapped.ptr(), - needs_flush: !mapped.is_coherent(), + ptr, + needs_flush: !stage_memory.is_coherent(), stage_buffer, stage_memory, }; @@ -1129,7 +1103,7 @@ impl Global { let device = device_guard .get(device_id) .map_err(|_| DeviceError::Invalid)?; - let mut buffer = buffer_guard + let buffer = buffer_guard .get_mut(buffer_id) .map_err(|_| resource::BufferAccessError::Invalid)?; check_buffer_usage(buffer.usage, wgt::BufferUsage::MAP_WRITE)?; @@ -1147,21 +1121,8 @@ impl Global { }); } - let ptr = map_buffer( - &device.raw, - &mut buffer, - hal::buffer::SubRange { - offset, - size: Some(data.len() as BufferAddress), - }, - HostMap::Write, - )?; - - unsafe { - ptr::copy_nonoverlapping(data.as_ptr(), ptr.as_ptr(), data.len()); - } - - unmap_buffer(&device.raw, buffer)?; + let (_, block) = buffer.raw.as_mut().unwrap(); + block.write_bytes(&device.raw, offset, data)?; Ok(()) } @@ -1183,27 +1144,14 @@ impl Global { let device = device_guard .get(device_id) .map_err(|_| DeviceError::Invalid)?; - let mut buffer = buffer_guard + let buffer = buffer_guard .get_mut(buffer_id) .map_err(|_| resource::BufferAccessError::Invalid)?; check_buffer_usage(buffer.usage, wgt::BufferUsage::MAP_READ)?; //assert!(buffer isn't used by the GPU); - let ptr = map_buffer( - &device.raw, - &mut buffer, - hal::buffer::SubRange { - offset, - size: Some(data.len() as BufferAddress), - }, - HostMap::Read, - )?; - - unsafe { - ptr::copy_nonoverlapping(ptr.as_ptr(), data.as_mut_ptr(), data.len()); - } - - unmap_buffer(&device.raw, buffer)?; + let (_, block) = buffer.raw.as_mut().unwrap(); + block.read_bytes(&device.raw, offset, data)?; Ok(()) } @@ -1739,9 +1687,7 @@ impl Global { let actual_border: hal::image::PackedColor = desc .border_color .map(|c| match c { - wgt::SamplerBorderColor::TransparentBlack => { - [0., 0., 0., 0.].into() - } + wgt::SamplerBorderColor::TransparentBlack => [0., 0., 0., 0.].into(), wgt::SamplerBorderColor::OpaqueBlack => [0., 0., 0., 1.].into(), wgt::SamplerBorderColor::OpaqueWhite => [1., 1., 1., 1.].into(), }) @@ -2379,7 +2325,7 @@ impl Global { Ok(hal::pso::Descriptor::Image(raw, image_layout)) } resource::TextureViewInner::SwapChain { .. } => { - return Err(CreateBindGroupError::SwapChainImage) + Err(CreateBindGroupError::SwapChainImage) } } }) @@ -3799,10 +3745,7 @@ impl Global { } resource::BufferMapState::Idle => { resource::BufferMapState::Waiting(resource::BufferPendingMapping { - sub_range: hal::buffer::SubRange { - offset: range.start, - size: Some(range.end - range.start), - }, + range, op, parent_ref_count: buffer.life_guard.add_ref(), }) @@ -3894,17 +3837,7 @@ impl Global { let _ = ptr; if needs_flush { - //TODO: gfx-memory needs a helper method for this. - // It needs to align the mapped range to the non-coherent atom size. - unsafe { - device - .raw - .flush_mapped_memory_ranges(iter::once(( - stage_memory.memory(), - stage_memory.segment(), - ))) - .unwrap() - }; + stage_memory.flush_range(&device.raw, 0, None)?; } let &(ref buf_raw, _) = buffer diff --git a/wgpu-core/src/device/queue.rs b/wgpu-core/src/device/queue.rs index ac7b3429b..a03a499e5 100644 --- a/wgpu-core/src/device/queue.rs +++ b/wgpu-core/src/device/queue.rs @@ -10,22 +10,21 @@ use crate::{ CommandAllocator, CommandBuffer, CopySide, TextureCopyView, TransferError, BITS_PER_BYTE, }, conv, - device::{DeviceError, WaitIdleError}, + device::{alloc, DeviceError, WaitIdleError}, hub::{GfxBackend, Global, GlobalIdentityHandlerFactory, Token}, id, resource::{BufferAccessError, BufferMapState, BufferUse, TextureUse}, span, FastHashSet, }; -use gfx_memory::{Block, Heaps, MemoryBlock}; use hal::{command::CommandBuffer as _, device::Device as _, queue::CommandQueue as _}; use smallvec::SmallVec; -use std::iter; +use std::{iter, ptr}; use thiserror::Error; struct StagingData { buffer: B::Buffer, - memory: MemoryBlock, + memory: alloc::MemoryBlock, cmdbuf: B::CommandBuffer, } @@ -38,7 +37,7 @@ pub enum TempResource { #[derive(Debug)] pub(crate) struct PendingWrites { pub command_buffer: Option, - pub temp_resources: Vec<(TempResource, MemoryBlock)>, + pub temp_resources: Vec<(TempResource, alloc::MemoryBlock)>, pub dst_buffers: FastHashSet, pub dst_textures: FastHashSet, } @@ -57,7 +56,7 @@ impl PendingWrites { self, device: &B::Device, cmd_allocator: &CommandAllocator, - mem_allocator: &mut Heaps, + mem_allocator: &mut alloc::MemoryAllocator, ) { if let Some(raw) = self.command_buffer { cmd_allocator.discard_internal(raw); @@ -75,7 +74,7 @@ impl PendingWrites { } } - pub fn consume_temp(&mut self, resource: TempResource, memory: MemoryBlock) { + pub fn consume_temp(&mut self, resource: TempResource, memory: alloc::MemoryBlock) { self.temp_resources.push((resource, memory)); } @@ -118,24 +117,17 @@ impl super::Device { })? }; //TODO: do we need to transition into HOST_WRITE access first? - let requirements = unsafe { self.raw.get_buffer_requirements(&buffer) }; - - let memory = self - .mem_allocator - .lock() - .allocate( - &self.raw, - &requirements, - gfx_memory::MemoryUsage::Staging { read_back: false }, - gfx_memory::Kind::Linear, - ) - .map_err(DeviceError::from_heaps)?; - unsafe { + let requirements = unsafe { self.raw.set_buffer_name(&mut buffer, ""); - self.raw - .bind_buffer_memory(memory.memory(), memory.segment().offset, &mut buffer) - .map_err(DeviceError::from_bind)?; - } + self.raw.get_buffer_requirements(&buffer) + }; + + let block = self.mem_allocator.lock().allocate( + &self.raw, + requirements, + gpu_alloc::UsageFlags::UPLOAD | gpu_alloc::UsageFlags::TRANSIENT, + )?; + block.bind_buffer(&self.raw, &mut buffer)?; let cmdbuf = match self.pending_writes.command_buffer.take() { Some(cmdbuf) => cmdbuf, @@ -149,7 +141,7 @@ impl super::Device { }; Ok(StagingData { buffer, - memory, + memory: block, cmdbuf, }) } @@ -220,19 +212,7 @@ impl Global { } let mut stage = device.prepare_stage(data_size)?; - { - let mut mapped = stage - .memory - .map(&device.raw, hal::memory::Segment::ALL) - .map_err(|err| match err { - hal::device::MapError::OutOfMemory(_) => DeviceError::OutOfMemory, - _ => panic!("failed to map buffer: {}", err), - })?; - unsafe { mapped.write(&device.raw, hal::memory::Segment::ALL) } - .expect("failed to get writer to mapped staging buffer") - .slice[..data.len()] - .copy_from_slice(data); - } + stage.memory.write_bytes(&device.raw, 0, data)?; let mut trackers = device.trackers.lock(); let (dst, transition) = trackers @@ -388,19 +368,12 @@ impl Global { )?; dst.life_guard.use_at(device.active_submission_index + 1); - { - let mut mapped = stage - .memory - .map(&device.raw, hal::memory::Segment::ALL) - .map_err(|err| match err { - hal::device::MapError::OutOfMemory(_) => DeviceError::OutOfMemory, - _ => panic!("failed to map staging buffer: {}", err), - })?; - let mapping = unsafe { mapped.write(&device.raw, hal::memory::Segment::ALL) } - .expect("failed to get writer to mapped staging buffer"); + let ptr = stage.memory.map(&device.raw, 0, stage_size)?; + unsafe { + //TODO: https://github.com/zakarumych/gpu-alloc/issues/13 if stage_bytes_per_row == data_layout.bytes_per_row { // Fast path if the data isalready being aligned optimally. - mapping.slice[..stage_size as usize].copy_from_slice(data); + ptr::copy_nonoverlapping(data.as_ptr(), ptr.as_ptr(), stage_size as usize); } else { // Copy row by row into the optimal alignment. let copy_bytes_per_row = @@ -408,16 +381,20 @@ impl Global { for layer in 0..size.depth { let rows_offset = layer * block_rows_per_image; for row in 0..height_blocks { - let data_offset = - (rows_offset + row) as usize * data_layout.bytes_per_row as usize; - let stage_offset = - (rows_offset + row) as usize * stage_bytes_per_row as usize; - mapping.slice[stage_offset..stage_offset + copy_bytes_per_row] - .copy_from_slice(&data[data_offset..data_offset + copy_bytes_per_row]); + ptr::copy_nonoverlapping( + data.as_ptr().offset( + (rows_offset + row) as isize * data_layout.bytes_per_row as isize, + ), + ptr.as_ptr().offset( + (rows_offset + row) as isize * stage_bytes_per_row as isize, + ), + copy_bytes_per_row, + ); } } } } + stage.memory.unmap(&device.raw); let region = hal::command::BufferImageCopy { buffer_offset: 0, diff --git a/wgpu-core/src/resource.rs b/wgpu-core/src/resource.rs index 4b8b0b851..7770c116c 100644 --- a/wgpu-core/src/resource.rs +++ b/wgpu-core/src/resource.rs @@ -3,19 +3,19 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ use crate::{ - device::DeviceError, + device::{alloc::MemoryBlock, DeviceError}, id::{DeviceId, SwapChainId, TextureId}, track::{TextureSelector, DUMMY_SELECTOR}, validation::MissingBufferUsageError, Label, LifeGuard, RefCount, Stored, }; -use gfx_memory::MemoryBlock; use thiserror::Error; use std::{ borrow::Borrow, num::{NonZeroU32, NonZeroU8}, + ops::Range, ptr::NonNull, }; @@ -143,20 +143,9 @@ pub enum BufferAccessError { UnalignedRange, } -impl From for BufferAccessError { - fn from(error: hal::device::MapError) -> Self { - match error { - hal::device::MapError::OutOfMemory(_) => { - BufferAccessError::Device(DeviceError::OutOfMemory) - } - _ => panic!("failed to map buffer: {}", error), - } - } -} - #[derive(Debug)] pub(crate) struct BufferPendingMapping { - pub sub_range: hal::buffer::SubRange, + pub range: Range, pub op: BufferMapOperation, // hold the parent alive while the mapping is active pub parent_ref_count: RefCount,