Replace gfx-memory by gpu-alloc

This commit is contained in:
Dzmitry Malyshau committed 2020-11-09 12:16:34 -05:00
1 parent 789b09c7bd
commit 22efbeae70
7 files changed
+451 -251

No files matched your search

Generated
+18 -14
View File
@@ -710,19 +710,6 @@ dependencies = [
"raw-window-handle",
]
[[package]]
name = "gfx-memory"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dccdda5d2b39412f4ca2cb15c70b5a82783a86b0606f5e985342754c8ed88f05"
dependencies = [
"bit-set",
"fxhash",
"gfx-hal",
"log",
"slab",
]
[[package]]
name = "gimli"
version = "0.23.0"
@@ -772,6 +759,23 @@ dependencies = [
"web-sys",
]
[[package]]
name = "gpu-alloc"
version = "0.1.0"
source = "git+https://github.com/zakarumych/gpu-alloc?rev=2bb7ef947443b1b13e34d3b98a116da6d69ac15b#2bb7ef947443b1b13e34d3b98a116da6d69ac15b"
dependencies = [
"bitflags",
"gpu-alloc-types",
]
[[package]]
name = "gpu-alloc-types"
version = "0.1.0"
source = "git+https://github.com/zakarumych/gpu-alloc?rev=2bb7ef947443b1b13e34d3b98a116da6d69ac15b#2bb7ef947443b1b13e34d3b98a116da6d69ac15b"
dependencies = [
"bitflags",
]
[[package]]
name = "hashbrown"
version = "0.8.2"
@@ -2172,7 +2176,7 @@ dependencies = [
"gfx-backend-vulkan",
"gfx-descriptor",
"gfx-hal",
"gfx-memory",
"gpu-alloc",
"loom",
"naga",
"parking_lot 0.11.0",
+1 -1
View File
@@ -35,7 +35,7 @@ smallvec = "1"
tracing = { version = "0.1", default-features = false, features = ["std"] }
thiserror = "1"
gfx-descriptor = "0.2"
gfx-memory = "0.2"
gpu-alloc = { git = "https://github.com/zakarumych/gpu-alloc", rev = "2bb7ef947443b1b13e34d3b98a116da6d69ac15b" }
[dependencies.naga]
version = "0.2"
+294
View File
@@ -0,0 +1,294 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
use super::DeviceError;
use hal::device::Device;
use std::{borrow::Cow, fmt, iter, ptr::NonNull, sync::Arc};
pub struct MemoryAllocator<B: hal::Backend>(gpu_alloc::GpuAllocator<Arc<B::Memory>>);
#[derive(Debug)]
pub struct MemoryBlock<B: hal::Backend>(gpu_alloc::MemoryBlock<Arc<B::Memory>>);
struct MemoryDevice<'a, B: hal::Backend>(&'a B::Device);
//TODO: https://github.com/zakarumych/gpu-alloc/issues/9
impl<B: hal::Backend> fmt::Debug for MemoryAllocator<B> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "MemoryAllocator")
}
}
impl<B: hal::Backend> MemoryAllocator<B> {
pub fn new(mem_props: hal::adapter::MemoryProperties, limits: hal::Limits) -> Self {
let mem_config = gpu_alloc::Config {
dedicated_treshold: 32 << 20,
preferred_dedicated_treshold: 8 << 20,
transient_dedicated_treshold: 128 << 20,
linear_chunk: 128 << 20,
minimal_buddy_size: 1 << 10,
initial_buddy_dedicated_size: 8 << 20,
};
let properties = gpu_alloc::DeviceProperties {
memory_types: Cow::Owned(
mem_props
.memory_types
.iter()
.map(|mt| gpu_alloc::MemoryType {
heap: mt.heap_index as u32,
props: gpu_alloc::MemoryPropertyFlags::from_bits_truncate(
mt.properties.bits() as u8,
),
})
.collect::<Vec<_>>(),
),
memory_heaps: Cow::Owned(
mem_props
.memory_heaps
.iter()
.map(|mh| gpu_alloc::MemoryHeap { size: *mh })
.collect::<Vec<_>>(),
),
max_memory_allocation_count: if limits.max_memory_allocation_count == 0 {
tracing::warn!("max_memory_allocation_count is not set by gfx-rs backend");
!0
} else {
limits.max_memory_allocation_count.min(!0u32 as usize) as u32
},
max_memory_allocation_size: !0,
non_coherent_atom_size: limits.non_coherent_atom_size as u64,
buffer_device_address: false,
};
MemoryAllocator(gpu_alloc::GpuAllocator::new(mem_config, properties))
}
pub fn allocate(
&mut self,
device: &B::Device,
requirements: hal::memory::Requirements,
usage: gpu_alloc::UsageFlags,
) -> Result<MemoryBlock<B>, DeviceError> {
assert!(requirements.alignment.is_power_of_two());
let request = gpu_alloc::Request {
size: requirements.size,
align_mask: requirements.alignment - 1,
memory_types: requirements.type_mask,
usage,
};
unsafe { self.0.alloc(&MemoryDevice::<B>(device), request) }
.map(MemoryBlock)
.map_err(|err| match err {
gpu_alloc::AllocationError::OutOfHostMemory
| gpu_alloc::AllocationError::OutOfDeviceMemory => DeviceError::OutOfMemory,
_ => panic!("Unable to allocate memory: {:?}", err),
})
}
pub fn free(&mut self, device: &B::Device, block: MemoryBlock<B>) {
unsafe { self.0.dealloc(&MemoryDevice::<B>(device), block.0) }
}
pub fn clear(&mut self, device: &B::Device) {
unsafe { self.0.cleanup(&MemoryDevice::<B>(device)) }
}
}
impl<B: hal::Backend> MemoryBlock<B> {
pub fn bind_buffer(
&self,
device: &B::Device,
buffer: &mut B::Buffer,
) -> Result<(), DeviceError> {
unsafe {
device
.bind_buffer_memory(self.0.memory(), self.0.offset(), buffer)
.map_err(DeviceError::from_bind)
}
}
pub fn bind_image(&self, device: &B::Device, image: &mut B::Image) -> Result<(), DeviceError> {
unsafe {
device
.bind_image_memory(self.0.memory(), self.0.offset(), image)
.map_err(DeviceError::from_bind)
}
}
pub fn is_coherent(&self) -> bool {
self.0
.props()
.contains(gpu_alloc::MemoryPropertyFlags::HOST_COHERENT)
}
pub fn map(
&mut self,
device: &B::Device,
inner_offset: wgt::BufferAddress,
size: wgt::BufferAddress,
) -> Result<NonNull<u8>, DeviceError> {
let offset = inner_offset;
unsafe {
self.0
.map(&MemoryDevice::<B>(device), offset, size as usize)
.map_err(DeviceError::from)
}
}
pub fn unmap(&mut self, device: &B::Device) {
unsafe { self.0.unmap(&MemoryDevice::<B>(device)) }
}
pub fn write_bytes(
&mut self,
device: &B::Device,
inner_offset: wgt::BufferAddress,
data: &[u8],
) -> Result<(), DeviceError> {
let offset = inner_offset;
unsafe {
self.0
.write_bytes(&MemoryDevice::<B>(device), offset, data)
.map_err(DeviceError::from)
}
}
pub fn read_bytes(
&mut self,
device: &B::Device,
inner_offset: wgt::BufferAddress,
data: &mut [u8],
) -> Result<(), DeviceError> {
let offset = inner_offset;
unsafe {
self.0
.read_bytes(&MemoryDevice::<B>(device), offset, data)
.map_err(DeviceError::from)
}
}
fn segment(
&self,
inner_offset: wgt::BufferAddress,
size: Option<wgt::BufferAddress>,
) -> hal::memory::Segment {
hal::memory::Segment {
offset: self.0.offset() + inner_offset,
size: size.or(Some(self.0.size())),
}
}
pub fn flush_range(
&self,
device: &B::Device,
inner_offset: wgt::BufferAddress,
size: Option<wgt::BufferAddress>,
) -> Result<(), DeviceError> {
let segment = self.segment(inner_offset, size);
unsafe {
device
.flush_mapped_memory_ranges(iter::once((&**self.0.memory(), segment)))
.or(Err(DeviceError::OutOfMemory))
}
}
pub fn invalidate_range(
&self,
device: &B::Device,
inner_offset: wgt::BufferAddress,
size: Option<wgt::BufferAddress>,
) -> Result<(), DeviceError> {
let segment = self.segment(inner_offset, size);
unsafe {
device
.invalidate_mapped_memory_ranges(iter::once((&**self.0.memory(), segment)))
.or(Err(DeviceError::OutOfMemory))
}
}
}
impl<B: hal::Backend> gpu_alloc::MemoryDevice<Arc<B::Memory>> for MemoryDevice<'_, B> {
#[cfg_attr(feature = "tracing", tracing::instrument(skip(self)))]
unsafe fn allocate_memory(
&self,
size: u64,
memory_type: u32,
flags: gpu_alloc::AllocationFlags,
) -> Result<Arc<B::Memory>, gpu_alloc::OutOfMemory> {
assert!(flags.is_empty());
self.0
.allocate_memory(hal::MemoryTypeId(memory_type as _), size)
.map(Arc::new)
.map_err(|_| gpu_alloc::OutOfMemory::OutOfDeviceMemory)
}
#[cfg_attr(feature = "tracing", tracing::instrument(skip(self)))]
unsafe fn deallocate_memory(&self, memory: Arc<B::Memory>) {
let memory = Arc::try_unwrap(memory).expect("Memory must not be used anywhere");
self.0.free_memory(memory);
}
#[cfg_attr(feature = "tracing", tracing::instrument(skip(self)))]
unsafe fn map_memory(
&self,
memory: &Arc<B::Memory>,
offset: u64,
size: u64,
) -> Result<NonNull<u8>, gpu_alloc::DeviceMapError> {
match self.0.map_memory(
memory,
hal::memory::Segment {
offset,
size: Some(size),
},
) {
Ok(ptr) => Ok(NonNull::new(ptr).expect("Pointer to memory mapping must not be null")),
Err(hal::device::MapError::OutOfMemory(_)) => {
Err(gpu_alloc::DeviceMapError::OutOfDeviceMemory)
}
Err(hal::device::MapError::MappingFailed) => Err(gpu_alloc::DeviceMapError::MapFailed),
Err(other) => panic!("Unexpected map error: {:?}", other),
}
}
#[cfg_attr(feature = "tracing", tracing::instrument(skip(self)))]
unsafe fn unmap_memory(&self, memory: &Arc<B::Memory>) {
self.0.unmap_memory(memory);
}
#[cfg_attr(feature = "tracing", tracing::instrument(skip(self)))]
unsafe fn invalidate_memory_ranges(
&self,
ranges: &[gpu_alloc::MappedMemoryRange<'_, Arc<B::Memory>>],
) -> Result<(), gpu_alloc::OutOfMemory> {
self.0
.invalidate_mapped_memory_ranges(ranges.iter().map(|range| {
(
&**range.memory,
hal::memory::Segment {
offset: range.offset,
size: Some(range.size),
},
)
}))
.map_err(|_| gpu_alloc::OutOfMemory::OutOfHostMemory)
}
#[cfg_attr(feature = "tracing", tracing::instrument(skip(self)))]
unsafe fn flush_memory_ranges(
&self,
ranges: &[gpu_alloc::MappedMemoryRange<'_, Arc<B::Memory>>],
) -> Result<(), gpu_alloc::OutOfMemory> {
self.0
.flush_mapped_memory_ranges(ranges.iter().map(|range| {
(
&**range.memory,
hal::memory::Segment {
offset: range.offset,
size: Some(range.size),
},
)
}))
.map_err(|_| gpu_alloc::OutOfMemory::OutOfHostMemory)
}
}
+18 -15
View File
@@ -5,7 +5,7 @@
#[cfg(feature = "trace")]
use crate::device::trace;
use crate::{
device::{queue::TempResource, DeviceError},
device::{alloc, queue::TempResource, DeviceError},
hub::{GfxBackend, GlobalIdentityHandlerFactory, Hub, Token},
id, resource,
track::TrackerSet,
@@ -14,7 +14,6 @@ use crate::{
use copyless::VecHelper as _;
use gfx_descriptor::{DescriptorAllocator, DescriptorSet};
use gfx_memory::{Heaps, MemoryBlock};
use hal::device::Device as _;
use parking_lot::Mutex;
use thiserror::Error;
@@ -84,8 +83,8 @@ impl SuspectedResources {
/// A struct that keeps lists of resources that are no longer needed.
#[derive(Debug)]
struct NonReferencedResources<B: hal::Backend> {
buffers: Vec<(B::Buffer, MemoryBlock<B>)>,
images: Vec<(B::Image, MemoryBlock<B>)>,
buffers: Vec<(B::Buffer, alloc::MemoryBlock<B>)>,
images: Vec<(B::Image, alloc::MemoryBlock<B>)>,
// Note: we keep the associated ID here in order to be able to check
// at any point what resources are used in a submission.
image_views: Vec<(id::Valid<id::TextureViewId>, B::ImageView)>,
@@ -130,20 +129,20 @@ impl<B: hal::Backend> NonReferencedResources<B> {
unsafe fn clean(
&mut self,
device: &B::Device,
heaps_mutex: &Mutex<Heaps<B>>,
memory_allocator_mutex: &Mutex<alloc::MemoryAllocator<B>>,
descriptor_allocator_mutex: &Mutex<DescriptorAllocator<B>>,
) {
if !self.buffers.is_empty() || !self.images.is_empty() {
let mut heaps = heaps_mutex.lock();
let mut allocator = memory_allocator_mutex.lock();
for (raw, memory) in self.buffers.drain(..) {
tracing::trace!("Buffer {:?} is destroyed with memory {:?}", raw, memory);
device.destroy_buffer(raw);
heaps.free(device, memory);
allocator.free(device, memory);
}
for (raw, memory) in self.images.drain(..) {
tracing::trace!("Image {:?} is destroyed with memory {:?}", raw, memory);
device.destroy_image(raw);
heaps.free(device, memory);
allocator.free(device, memory);
}
}
@@ -241,7 +240,7 @@ impl<B: hal::Backend> LifetimeTracker<B> {
index: SubmissionIndex,
fence: B::Fence,
new_suspects: &SuspectedResources,
temp_resources: impl Iterator<Item = (TempResource<B>, MemoryBlock<B>)>,
temp_resources: impl Iterator<Item = (TempResource<B>, alloc::MemoryBlock<B>)>,
) {
let mut last_resources = NonReferencedResources::new();
for (res, memory) in temp_resources {
@@ -334,12 +333,12 @@ impl<B: hal::Backend> LifetimeTracker<B> {
pub fn cleanup(
&mut self,
device: &B::Device,
heaps_mutex: &Mutex<Heaps<B>>,
memory_allocator_mutex: &Mutex<alloc::MemoryAllocator<B>>,
descriptor_allocator_mutex: &Mutex<DescriptorAllocator<B>>,
) {
unsafe {
self.free_resources
.clean(device, heaps_mutex, descriptor_allocator_mutex);
.clean(device, memory_allocator_mutex, descriptor_allocator_mutex);
descriptor_allocator_mutex.lock().cleanup(device);
}
}
@@ -347,7 +346,7 @@ impl<B: hal::Backend> LifetimeTracker<B> {
pub fn schedule_resource_destruction(
&mut self,
temp_resource: TempResource<B>,
memory: MemoryBlock<B>,
memory: alloc::MemoryBlock<B>,
last_submit_index: SubmissionIndex,
) {
let resources = self
@@ -725,14 +724,18 @@ impl<B: GfxBackend> LifetimeTracker<B> {
resource::BufferMapState::Waiting(pending_mapping) => pending_mapping,
_ => panic!("No pending mapping."),
};
let status = if mapping.sub_range.size.map_or(true, |x| x != 0) {
let status = if mapping.range.start != mapping.range.end {
tracing::debug!("Buffer {:?} map state -> Active", buffer_id);
let host = mapping.op.host;
match super::map_buffer(raw, buffer, mapping.sub_range.clone(), host) {
let size = mapping.range.end - mapping.range.start;
match super::map_buffer(raw, buffer, mapping.range.start, size, host) {
Ok(ptr) => {
buffer.map_state = resource::BufferMapState::Active {
ptr,
sub_range: mapping.sub_range,
sub_range: hal::buffer::SubRange {
offset: mapping.range.start,
size: Some(size),
},
host,
};
resource::BufferMapAsyncStatus::Success
+85 -152
View File
@@ -19,7 +19,6 @@ use crate::{
use arrayvec::ArrayVec;
use copyless::VecHelper as _;
use gfx_descriptor::DescriptorAllocator;
use gfx_memory::{Block, Heaps};
use hal::{
command::CommandBuffer as _,
device::Device as _,
@@ -42,6 +41,7 @@ use std::{
sync::atomic::Ordering,
};
pub mod alloc;
mod life;
mod queue;
#[cfg(any(feature = "trace", feature = "replay"))]
@@ -163,34 +163,25 @@ type BufferMapPendingCallback = (resource::BufferMapOperation, resource::BufferM
fn map_buffer<B: hal::Backend>(
raw: &B::Device,
buffer: &mut resource::Buffer<B>,
sub_range: hal::buffer::SubRange,
offset: hal::buffer::Offset,
size: BufferAddress,
kind: HostMap,
) -> Result<ptr::NonNull<u8>, resource::BufferAccessError> {
let &mut (_, ref mut memory) = buffer
let &mut (_, ref mut block) = buffer
.raw
.as_mut()
.ok_or(resource::BufferAccessError::Destroyed)?;
let (ptr, segment, needs_sync) = {
let segment = hal::memory::Segment {
offset: sub_range.offset,
size: sub_range.size,
};
let mapped = memory.map(raw, segment)?;
let mr = mapped.range();
let segment = hal::memory::Segment {
offset: mr.start,
size: Some(mr.end - mr.start),
};
(mapped.ptr(), segment, !mapped.is_coherent())
};
let ptr = block.map(raw, offset, size).map_err(DeviceError::from)?;
buffer.sync_mapped_writes = match kind {
HostMap::Read if needs_sync => unsafe {
raw.invalidate_mapped_memory_ranges(iter::once((memory.memory(), segment)))
.or(Err(DeviceError::OutOfMemory))?;
HostMap::Read if !block.is_coherent() => {
block.invalidate_range(raw, offset, Some(size))?;
None
},
HostMap::Write if needs_sync => Some(segment),
}
HostMap::Write if !block.is_coherent() => Some(hal::memory::Segment {
offset,
size: Some(size),
}),
_ => None,
};
Ok(ptr)
@@ -200,16 +191,14 @@ fn unmap_buffer<B: hal::Backend>(
raw: &B::Device,
buffer: &mut resource::Buffer<B>,
) -> Result<(), resource::BufferAccessError> {
let &(_, ref memory) = buffer
let &mut (_, ref mut block) = buffer
.raw
.as_ref()
.as_mut()
.ok_or(resource::BufferAccessError::Destroyed)?;
if let Some(segment) = buffer.sync_mapped_writes.take() {
unsafe {
raw.flush_mapped_memory_ranges(iter::once((memory.memory(), segment)))
.or(Err(DeviceError::OutOfMemory))?;
}
block.flush_range(raw, segment.offset, segment.size)?;
}
block.unmap(raw);
Ok(())
}
@@ -227,7 +216,7 @@ pub struct Device<B: hal::Backend> {
pub(crate) adapter_id: Stored<id::AdapterId>,
pub(crate) queue_group: hal::queue::QueueGroup<B>,
pub(crate) cmd_allocator: command::CommandAllocator<B>,
mem_allocator: Mutex<Heaps<B>>,
mem_allocator: Mutex<alloc::MemoryAllocator<B>>,
desc_allocator: Mutex<DescriptorAllocator<B>>,
//Note: The submission index here corresponds to the last submission that is done.
pub(crate) life_guard: LifeGuard,
@@ -268,20 +257,8 @@ impl<B: GfxBackend> Device<B> {
) -> Result<Self, CreateDeviceError> {
let cmd_allocator = command::CommandAllocator::new(queue_group.family, &raw)
.or(Err(CreateDeviceError::OutOfMemory))?;
let heaps = unsafe {
Heaps::new(
&mem_props,
gfx_memory::GeneralConfig {
block_size_granularity: 0x100,
max_chunk_size: 0x100_0000,
min_device_allocation: 0x1_0000,
},
gfx_memory::LinearConfig {
line_size: 0x100_0000,
},
hal_limits.non_coherent_atom_size as u64,
)
};
let mem_allocator = alloc::MemoryAllocator::new(mem_props, hal_limits);
let descriptors = unsafe { DescriptorAllocator::new() };
#[cfg(not(feature = "trace"))]
match trace_path {
@@ -293,7 +270,7 @@ impl<B: GfxBackend> Device<B> {
raw,
adapter_id,
cmd_allocator,
mem_allocator: Mutex::new(heaps),
mem_allocator: Mutex::new(mem_allocator),
desc_allocator: Mutex::new(descriptors),
queue_group,
life_guard: LifeGuard::new(),
@@ -436,7 +413,7 @@ impl<B: GfxBackend> Device<B> {
&self,
self_id: id::DeviceId,
desc: &resource::BufferDescriptor,
memory_kind: gfx_memory::Kind,
transient: bool,
) -> Result<resource::Buffer<B>, resource::CreateBufferError> {
debug_assert_eq!(self_id.backend(), B::VARIANT);
let (mut usage, _memory_properties) = conv::map_buffer_usage(desc.usage);
@@ -446,27 +423,42 @@ impl<B: GfxBackend> Device<B> {
}
let mem_usage = {
use gfx_memory::MemoryUsage;
use gpu_alloc::UsageFlags as Uf;
use wgt::BufferUsage as Bu;
//TODO: use linear allocation when we can ensure the freeing is linear
if !desc.usage.intersects(Bu::MAP_READ | Bu::MAP_WRITE) {
MemoryUsage::Private
} else if (Bu::MAP_WRITE | Bu::COPY_SRC).contains(desc.usage) {
MemoryUsage::Staging { read_back: false }
} else if (Bu::MAP_READ | Bu::COPY_DST).contains(desc.usage) {
MemoryUsage::Staging { read_back: true }
} else {
let mut flags = Uf::empty();
let map_flags = desc.usage & (Bu::MAP_READ | Bu::MAP_WRITE);
if !(desc.usage - map_flags).is_empty() {
flags |= Uf::FAST_DEVICE_ACCESS;
}
if transient {
flags |= Uf::TRANSIENT;
}
if !map_flags.is_empty() {
let upload_usage = Bu::MAP_WRITE | Bu::COPY_SRC;
let download_usage = Bu::MAP_READ | Bu::COPY_DST;
flags |= Uf::HOST_ACCESS;
if desc.usage.contains(upload_usage) {
flags |= Uf::UPLOAD;
}
if desc.usage.contains(download_usage) {
flags |= Uf::DOWNLOAD;
}
let is_native_only = self
.features
.contains(wgt::Features::MAPPABLE_PRIMARY_BUFFERS);
if !is_native_only {
if !is_native_only
&& !upload_usage.contains(desc.usage)
&& !download_usage.contains(desc.usage)
{
return Err(resource::CreateBufferError::UsageMismatch(desc.usage));
}
MemoryUsage::Dynamic {
sparse_updates: false,
}
}
flags
};
let mut buffer = unsafe { self.raw.create_buffer(desc.size.max(1), usage) }.map_err(
@@ -480,19 +472,14 @@ impl<B: GfxBackend> Device<B> {
}
let requirements = unsafe { self.raw.get_buffer_requirements(&buffer) };
let memory = self
let block = self
.mem_allocator
.lock()
.allocate(&self.raw, &requirements, mem_usage, memory_kind)
.map_err(DeviceError::from_heaps)?;
unsafe {
self.raw
.bind_buffer_memory(memory.memory(), memory.segment().offset, &mut buffer)
}
.map_err(DeviceError::from_bind)?;
.allocate(&self.raw, requirements, mem_usage)?;
block.bind_buffer(&self.raw, &mut buffer)?;
Ok(resource::Buffer {
raw: Some((buffer, memory)),
raw: Some((buffer, block)),
device_id: Stored {
value: id::Valid(self_id),
ref_count: self.life_guard.add_ref(),
@@ -572,24 +559,15 @@ impl<B: GfxBackend> Device<B> {
};
let requirements = unsafe { self.raw.get_image_requirements(&image) };
let memory = self
.mem_allocator
.lock()
.allocate(
&self.raw,
&requirements,
gfx_memory::MemoryUsage::Private,
gfx_memory::Kind::General,
)
.map_err(DeviceError::from_heaps)?;
unsafe {
self.raw
.bind_image_memory(memory.memory(), memory.segment().offset, &mut image)
}
.map_err(DeviceError::from_bind)?;
let block = self.mem_allocator.lock().allocate(
&self.raw,
requirements,
gpu_alloc::UsageFlags::FAST_DEVICE_ACCESS,
)?;
block.bind_image(&self.raw, &mut image)?;
Ok(resource::Texture {
raw: Some((image, memory)),
raw: Some((image, block)),
device_id: Stored {
value: id::Valid(self_id),
ref_count: self.life_guard.add_ref(),
@@ -949,16 +927,18 @@ impl From<hal::device::OomOrDeviceLost> for DeviceError {
}
}
impl DeviceError {
fn from_heaps(err: gfx_memory::HeapsError) -> Self {
impl From<gpu_alloc::MapError> for DeviceError {
fn from(err: gpu_alloc::MapError) -> Self {
match err {
gfx_memory::HeapsError::AllocationError(hal::device::AllocationError::OutOfMemory(
_,
)) => Self::OutOfMemory,
_ => panic!("Unable to allocate memory: {:?}", err),
gpu_alloc::MapError::OutOfDeviceMemory | gpu_alloc::MapError::OutOfHostMemory => {
DeviceError::OutOfMemory
}
_ => panic!("failed to map buffer: {}", err),
}
}
}
impl DeviceError {
fn from_bind(err: hal::device::BindError) -> Self {
match err {
hal::device::BindError::OutOfMemory(_) => Self::OutOfMemory,
@@ -1022,19 +1002,15 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
let device = device_guard
.get(device_id)
.map_err(|_| DeviceError::Invalid)?;
let mut buffer = device.create_buffer(device_id, desc, gfx_memory::Kind::General)?;
let mut buffer = device.create_buffer(device_id, desc, false)?;
let ref_count = buffer.life_guard.add_ref();
let buffer_use = if !desc.mapped_at_creation {
resource::BufferUse::EMPTY
} else if desc.usage.contains(wgt::BufferUsage::MAP_WRITE) {
// buffer is mappable, so we are just doing that at start
let ptr = map_buffer(
&device.raw,
&mut buffer,
hal::buffer::SubRange::WHOLE,
HostMap::Write,
)?;
let size = buffer.size;
let ptr = map_buffer(&device.raw, &mut buffer, 0, size, HostMap::Write)?;
buffer.map_state = resource::BufferMapState::Active {
ptr,
@@ -1053,15 +1029,13 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
usage: wgt::BufferUsage::MAP_WRITE | wgt::BufferUsage::COPY_SRC,
mapped_at_creation: false,
},
gfx_memory::Kind::Linear,
true,
)?;
let (stage_buffer, mut stage_memory) = stage.raw.unwrap();
let mapped = stage_memory
.map(&device.raw, hal::memory::Segment::ALL)
.map_err(resource::BufferAccessError::from)?;
let ptr = stage_memory.map(&device.raw, 0, stage.size)?;
buffer.map_state = resource::BufferMapState::Init {
ptr: mapped.ptr(),
needs_flush: !mapped.is_coherent(),
ptr,
needs_flush: !stage_memory.is_coherent(),
stage_buffer,
stage_memory,
};
@@ -1129,7 +1103,7 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
let device = device_guard
.get(device_id)
.map_err(|_| DeviceError::Invalid)?;
let mut buffer = buffer_guard
let buffer = buffer_guard
.get_mut(buffer_id)
.map_err(|_| resource::BufferAccessError::Invalid)?;
check_buffer_usage(buffer.usage, wgt::BufferUsage::MAP_WRITE)?;
@@ -1147,21 +1121,8 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
});
}
let ptr = map_buffer(
&device.raw,
&mut buffer,
hal::buffer::SubRange {
offset,
size: Some(data.len() as BufferAddress),
},
HostMap::Write,
)?;
unsafe {
ptr::copy_nonoverlapping(data.as_ptr(), ptr.as_ptr(), data.len());
}
unmap_buffer(&device.raw, buffer)?;
let (_, block) = buffer.raw.as_mut().unwrap();
block.write_bytes(&device.raw, offset, data)?;
Ok(())
}
@@ -1183,27 +1144,14 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
let device = device_guard
.get(device_id)
.map_err(|_| DeviceError::Invalid)?;
let mut buffer = buffer_guard
let buffer = buffer_guard
.get_mut(buffer_id)
.map_err(|_| resource::BufferAccessError::Invalid)?;
check_buffer_usage(buffer.usage, wgt::BufferUsage::MAP_READ)?;
//assert!(buffer isn't used by the GPU);
let ptr = map_buffer(
&device.raw,
&mut buffer,
hal::buffer::SubRange {
offset,
size: Some(data.len() as BufferAddress),
},
HostMap::Read,
)?;
unsafe {
ptr::copy_nonoverlapping(ptr.as_ptr(), data.as_mut_ptr(), data.len());
}
unmap_buffer(&device.raw, buffer)?;
let (_, block) = buffer.raw.as_mut().unwrap();
block.read_bytes(&device.raw, offset, data)?;
Ok(())
}
@@ -1739,9 +1687,7 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
let actual_border: hal::image::PackedColor = desc
.border_color
.map(|c| match c {
wgt::SamplerBorderColor::TransparentBlack => {
[0., 0., 0., 0.].into()
}
wgt::SamplerBorderColor::TransparentBlack => [0., 0., 0., 0.].into(),
wgt::SamplerBorderColor::OpaqueBlack => [0., 0., 0., 1.].into(),
wgt::SamplerBorderColor::OpaqueWhite => [1., 1., 1., 1.].into(),
})
@@ -2379,7 +2325,7 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
Ok(hal::pso::Descriptor::Image(raw, image_layout))
}
resource::TextureViewInner::SwapChain { .. } => {
return Err(CreateBindGroupError::SwapChainImage)
Err(CreateBindGroupError::SwapChainImage)
}
}
})
@@ -3799,10 +3745,7 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
}
resource::BufferMapState::Idle => {
resource::BufferMapState::Waiting(resource::BufferPendingMapping {
sub_range: hal::buffer::SubRange {
offset: range.start,
size: Some(range.end - range.start),
},
range,
op,
parent_ref_count: buffer.life_guard.add_ref(),
})
@@ -3894,17 +3837,7 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
let _ = ptr;
if needs_flush {
//TODO: gfx-memory needs a helper method for this.
// It needs to align the mapped range to the non-coherent atom size.
unsafe {
device
.raw
.flush_mapped_memory_ranges(iter::once((
stage_memory.memory(),
stage_memory.segment(),
)))
.unwrap()
};
stage_memory.flush_range(&device.raw, 0, None)?;
}
let &(ref buf_raw, _) = buffer
+32 -55
View File
@@ -10,22 +10,21 @@ use crate::{
CommandAllocator, CommandBuffer, CopySide, TextureCopyView, TransferError, BITS_PER_BYTE,
},
conv,
device::{DeviceError, WaitIdleError},
device::{alloc, DeviceError, WaitIdleError},
hub::{GfxBackend, Global, GlobalIdentityHandlerFactory, Token},
id,
resource::{BufferAccessError, BufferMapState, BufferUse, TextureUse},
span, FastHashSet,
};
use gfx_memory::{Block, Heaps, MemoryBlock};
use hal::{command::CommandBuffer as _, device::Device as _, queue::CommandQueue as _};
use smallvec::SmallVec;
use std::iter;
use std::{iter, ptr};
use thiserror::Error;
struct StagingData<B: hal::Backend> {
buffer: B::Buffer,
memory: MemoryBlock<B>,
memory: alloc::MemoryBlock<B>,
cmdbuf: B::CommandBuffer,
}
@@ -38,7 +37,7 @@ pub enum TempResource<B: hal::Backend> {
#[derive(Debug)]
pub(crate) struct PendingWrites<B: hal::Backend> {
pub command_buffer: Option<B::CommandBuffer>,
pub temp_resources: Vec<(TempResource<B>, MemoryBlock<B>)>,
pub temp_resources: Vec<(TempResource<B>, alloc::MemoryBlock<B>)>,
pub dst_buffers: FastHashSet<id::BufferId>,
pub dst_textures: FastHashSet<id::TextureId>,
}
@@ -57,7 +56,7 @@ impl<B: hal::Backend> PendingWrites<B> {
self,
device: &B::Device,
cmd_allocator: &CommandAllocator<B>,
mem_allocator: &mut Heaps<B>,
mem_allocator: &mut alloc::MemoryAllocator<B>,
) {
if let Some(raw) = self.command_buffer {
cmd_allocator.discard_internal(raw);
@@ -75,7 +74,7 @@ impl<B: hal::Backend> PendingWrites<B> {
}
}
pub fn consume_temp(&mut self, resource: TempResource<B>, memory: MemoryBlock<B>) {
pub fn consume_temp(&mut self, resource: TempResource<B>, memory: alloc::MemoryBlock<B>) {
self.temp_resources.push((resource, memory));
}
@@ -118,24 +117,17 @@ impl<B: hal::Backend> super::Device<B> {
})?
};
//TODO: do we need to transition into HOST_WRITE access first?
let requirements = unsafe { self.raw.get_buffer_requirements(&buffer) };
let memory = self
.mem_allocator
.lock()
.allocate(
&self.raw,
&requirements,
gfx_memory::MemoryUsage::Staging { read_back: false },
gfx_memory::Kind::Linear,
)
.map_err(DeviceError::from_heaps)?;
unsafe {
let requirements = unsafe {
self.raw.set_buffer_name(&mut buffer, "<write_buffer_temp>");
self.raw
.bind_buffer_memory(memory.memory(), memory.segment().offset, &mut buffer)
.map_err(DeviceError::from_bind)?;
}
self.raw.get_buffer_requirements(&buffer)
};
let block = self.mem_allocator.lock().allocate(
&self.raw,
requirements,
gpu_alloc::UsageFlags::UPLOAD | gpu_alloc::UsageFlags::TRANSIENT,
)?;
block.bind_buffer(&self.raw, &mut buffer)?;
let cmdbuf = match self.pending_writes.command_buffer.take() {
Some(cmdbuf) => cmdbuf,
@@ -149,7 +141,7 @@ impl<B: hal::Backend> super::Device<B> {
};
Ok(StagingData {
buffer,
memory,
memory: block,
cmdbuf,
})
}
@@ -220,19 +212,7 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
}
let mut stage = device.prepare_stage(data_size)?;
{
let mut mapped = stage
.memory
.map(&device.raw, hal::memory::Segment::ALL)
.map_err(|err| match err {
hal::device::MapError::OutOfMemory(_) => DeviceError::OutOfMemory,
_ => panic!("failed to map buffer: {}", err),
})?;
unsafe { mapped.write(&device.raw, hal::memory::Segment::ALL) }
.expect("failed to get writer to mapped staging buffer")
.slice[..data.len()]
.copy_from_slice(data);
}
stage.memory.write_bytes(&device.raw, 0, data)?;
let mut trackers = device.trackers.lock();
let (dst, transition) = trackers
@@ -388,19 +368,12 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
)?;
dst.life_guard.use_at(device.active_submission_index + 1);
{
let mut mapped = stage
.memory
.map(&device.raw, hal::memory::Segment::ALL)
.map_err(|err| match err {
hal::device::MapError::OutOfMemory(_) => DeviceError::OutOfMemory,
_ => panic!("failed to map staging buffer: {}", err),
})?;
let mapping = unsafe { mapped.write(&device.raw, hal::memory::Segment::ALL) }
.expect("failed to get writer to mapped staging buffer");
let ptr = stage.memory.map(&device.raw, 0, stage_size)?;
unsafe {
//TODO: https://github.com/zakarumych/gpu-alloc/issues/13
if stage_bytes_per_row == data_layout.bytes_per_row {
// Fast path if the data isalready being aligned optimally.
mapping.slice[..stage_size as usize].copy_from_slice(data);
ptr::copy_nonoverlapping(data.as_ptr(), ptr.as_ptr(), stage_size as usize);
} else {
// Copy row by row into the optimal alignment.
let copy_bytes_per_row =
@@ -408,16 +381,20 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
for layer in 0..size.depth {
let rows_offset = layer * block_rows_per_image;
for row in 0..height_blocks {
let data_offset =
(rows_offset + row) as usize * data_layout.bytes_per_row as usize;
let stage_offset =
(rows_offset + row) as usize * stage_bytes_per_row as usize;
mapping.slice[stage_offset..stage_offset + copy_bytes_per_row]
.copy_from_slice(&data[data_offset..data_offset + copy_bytes_per_row]);
ptr::copy_nonoverlapping(
data.as_ptr().offset(
(rows_offset + row) as isize * data_layout.bytes_per_row as isize,
),
ptr.as_ptr().offset(
(rows_offset + row) as isize * stage_bytes_per_row as isize,
),
copy_bytes_per_row,
);
}
}
}
}
stage.memory.unmap(&device.raw);
let region = hal::command::BufferImageCopy {
buffer_offset: 0,
+3 -14
View File
@@ -3,19 +3,19 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
use crate::{
device::DeviceError,
device::{alloc::MemoryBlock, DeviceError},
id::{DeviceId, SwapChainId, TextureId},
track::{TextureSelector, DUMMY_SELECTOR},
validation::MissingBufferUsageError,
Label, LifeGuard, RefCount, Stored,
};
use gfx_memory::MemoryBlock;
use thiserror::Error;
use std::{
borrow::Borrow,
num::{NonZeroU32, NonZeroU8},
ops::Range,
ptr::NonNull,
};
@@ -143,20 +143,9 @@ pub enum BufferAccessError {
UnalignedRange,
}
impl From<hal::device::MapError> for BufferAccessError {
fn from(error: hal::device::MapError) -> Self {
match error {
hal::device::MapError::OutOfMemory(_) => {
BufferAccessError::Device(DeviceError::OutOfMemory)
}
_ => panic!("failed to map buffer: {}", error),
}
}
}
#[derive(Debug)]
pub(crate) struct BufferPendingMapping {
pub sub_range: hal::buffer::SubRange,
pub range: Range<wgt::BufferAddress>,
pub op: BufferMapOperation,
// hold the parent alive while the mapping is active
pub parent_ref_count: RefCount,