From a1fceed0710fafed9abe0f7fb16c0a19bb390bf2 Mon Sep 17 00:00:00 2001 From: Vecvec Date: Fri, 29 May 2026 09:49:12 +1200 Subject: [PATCH] Move fence synchronisation into wgpu-hal (#9475) --- CHANGELOG.md | 1 + wgpu-core/src/as_hal.rs | 72 +------------------ wgpu-core/src/device/mod.rs | 1 - wgpu-core/src/device/queue.rs | 44 ++++++------ wgpu-core/src/device/resource.rs | 44 ++++++------ wgpu-core/src/lock/observing.rs | 19 ++---- wgpu-core/src/lock/rank.rs | 6 +- wgpu-core/src/lock/ranked.rs | 19 ++---- wgpu-core/src/lock/vanilla.rs | 6 -- wgpu-core/src/present.rs | 11 ++- wgpu-hal/src/dx12/mod.rs | 2 +- wgpu-hal/src/dynamic/mod.rs | 10 --- wgpu-hal/src/dynamic/queue.rs | 6 +- wgpu-hal/src/gles/fence.rs | 83 +++++++++++++++------- wgpu-hal/src/gles/queue.rs | 2 +- wgpu-hal/src/lib.rs | 7 +- wgpu-hal/src/metal/device.rs | 16 +++-- wgpu-hal/src/metal/mod.rs | 19 ++++-- wgpu-hal/src/noop/mod.rs | 2 +- wgpu-hal/src/vulkan/device.rs | 47 ++++++++----- wgpu-hal/src/vulkan/mod.rs | 114 +++++++++++++++++++++---------- 21 files changed, 260 insertions(+), 271 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9896a8d1..85147c7f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -185,6 +185,7 @@ By @beholdnec in [#8505](https://github.com/gfx-rs/wgpu/pull/8505). - BREAKING: `map_label` helpers have changed slightly. By @beicause and @andyleiserson in [#9480](https://github.com/gfx-rs/wgpu/pull/9480), [#9481](https://github.com/gfx-rs/wgpu/pull/9481), and [#9526](https://github.com/gfx-rs/wgpu/pull/9526). - `TextureDescriptor::map_label_and_view_formats` and `SurfaceConfiguration::map_view_formats` now take `FnOnce(&V)` instead of `FnOnce(V)`. - All `map_label` helpers except `CreateShaderModuleDescriptorPassthrough` now have the signature `map_label<'a, K>(&'a self, fun: impl FnOnce(&'a L) -> K)` (previously the lifetimes were implicit and thus could differ). +- Relaxed locking within `wgpu-core` to enable queue submission processing on one thread to proceed while another thread is blocked in a device poll. To facilitate this, `wgpu-hal` fences are now internally synchronized. By @Vecvec in [#9475](https://github.com/gfx-rs/wgpu/pull/9475). #### Validation diff --git a/wgpu-core/src/as_hal.rs b/wgpu-core/src/as_hal.rs index cb09f03ab..7f997af6d 100644 --- a/wgpu-core/src/as_hal.rs +++ b/wgpu-core/src/as_hal.rs @@ -4,13 +4,12 @@ use alloc::sync::Arc; use hal::DynResource; use crate::{ - device::Device, global::Global, id::{ AdapterId, BlasId, BufferId, CommandEncoderId, DeviceId, QueueId, SurfaceId, TextureId, TextureViewId, TlasId, }, - lock::{RankData, RwLockReadGuard}, + lock::RankData, resource::RawResourceAccess, snatch::SnatchGuard, }; @@ -154,73 +153,6 @@ where { } -/// A guard which holds alive a device and the device's fence lock, dereferencing to the Hal type. -struct FenceGuard { - device: Arc, - fence_lock_rank_data: ManuallyDrop, - ptr: *const Fence, -} - -impl FenceGuard -where - Fence: 'static, -{ - /// Creates a new guard over a device's fence. - /// - /// Returns `None` if: - /// - The device's fence is not of the expected Hal type. - pub fn new(device: Arc) -> Option { - // Grab the fence lock. - let fence_guard = device.fence.read(); - - // Get the raw fence and downcast it to the expected Hal type, coercing it to a pointer - // to get rid of the lifetime connecting us to the fence guard. - let ptr: *const Fence = fence_guard.as_any().downcast_ref::()?; - - // SAFETY: At this point all panicking or divergance has already happened, - // so we can safely forget the fence guard without causing the lock to be left open. - let fence_lock_rank_data = RwLockReadGuard::forget(fence_guard); - - // SAFETY: We only construct this guard while the fence lock is held, - // as the `drop` implementation of this guard will unsafely release the lock. - Some(Self { - device, - fence_lock_rank_data: ManuallyDrop::new(fence_lock_rank_data), - ptr, - }) - } -} - -impl Deref for FenceGuard { - type Target = Fence; - - fn deref(&self) -> &Self::Target { - // SAFETY: The pointer is guaranteed to be valid as the original device's fence - // is still alive and the fence lock is still being held due to the forgotten - // fence guard. - unsafe { &*self.ptr } - } -} - -impl Drop for FenceGuard { - fn drop(&mut self) { - // SAFETY: - // - We are not going to access the rank data anymore. - let data = unsafe { ManuallyDrop::take(&mut self.fence_lock_rank_data) }; - - // SAFETY: - // - The pointer is no longer going to be accessed. - // - The fence lock is being held because this type was not created - // until after the fence lock was forgotten. - unsafe { - self.device.fence.force_unlock_read(data); - }; - } -} - -unsafe impl Send for FenceGuard where Fence: Send {} -unsafe impl Sync for FenceGuard where Fence: Sync {} - impl Global { /// # Safety /// @@ -312,7 +244,7 @@ impl Global { let device = self.hub.devices.get(id); - FenceGuard::new(device) + SimpleResourceGuard::new(device, move |device| device.fence.as_any().downcast_ref()) } /// # Safety diff --git a/wgpu-core/src/device/mod.rs b/wgpu-core/src/device/mod.rs index ab3045b77..864803aa1 100644 --- a/wgpu-core/src/device/mod.rs +++ b/wgpu-core/src/device/mod.rs @@ -28,7 +28,6 @@ pub mod ray_tracing; pub mod resource; #[cfg(any(feature = "trace", feature = "replay"))] pub mod trace; -pub(crate) use resource::{FenceReadGuard, FenceWriteGuard}; pub use {life::WaitIdleError, resource::Device}; pub const SHADER_STAGE_COUNT: usize = hal::MAX_CONCURRENT_SHADER_STAGES; diff --git a/wgpu-core/src/device/queue.rs b/wgpu-core/src/device/queue.rs index f823579d8..ee5a16a0e 100644 --- a/wgpu-core/src/device/queue.rs +++ b/wgpu-core/src/device/queue.rs @@ -24,7 +24,7 @@ use crate::{ CommandAllocator, CommandBuffer, CommandEncoder, CommandEncoderError, CopySide, TransferError, }, - device::{DeviceError, FenceReadGuard, FenceWriteGuard, WaitIdleError}, + device::{DeviceError, WaitIdleError}, get_lowest_common_denom, global::Global, hal_label, @@ -227,6 +227,13 @@ impl Queue { Ok(()) } + /// Maintains the queue's list of finished command buffers. + /// + /// Command buffers submitted before `submission_index` was submitted + /// stop being tracked and callbacks which are waiting for them are + /// returned. Also returned is whether the queue is empty. This may + /// be stale unless new submissions are prevented by locking + /// [`Device::command_indices`] pub(crate) fn maintain( &self, submission_index: u64, @@ -575,7 +582,6 @@ impl WebGpuError for QueueSubmitError { pub(crate) struct PendingSubmission<'a> { queue: &'a Queue, snatch_guard: SnatchGuard<'a>, - fence: FenceWriteGuard<'a>, command_index_guard: RwLockWriteGuard<'a, CommandIndices>, // Command buffers to be executed, along with trackers for the resources they use. pub executions: Vec, @@ -587,7 +593,6 @@ pub(crate) struct PendingSubmission<'a> { } pub(crate) struct SubmissionResult<'a> { - pub fence: FenceReadGuard<'a>, pub snatch_guard: SnatchGuard<'a>, } @@ -1494,10 +1499,7 @@ impl Queue { let pending_writes = self.pending_writes.lock(); - let SubmissionResult { - fence, - snatch_guard, - } = match submission.submit(pending_writes) { + let SubmissionResult { snatch_guard } = match submission.submit(pending_writes) { Ok(result) => result, Err(e) => break 'error Err(e.into()), }; @@ -1507,9 +1509,7 @@ impl Queue { // This will schedule destruction of all resources that are no longer needed // by the user but used in the command stream, among other things. // `device.maintain` consumes and will release the snatch guard. - let (closures, result) = self - .device - .maintain(fence, wgt::PollType::Poll, snatch_guard); + let (closures, result) = self.device.maintain(wgt::PollType::Poll, snatch_guard); match result { Ok(status) => { debug_assert!(matches!( @@ -1568,10 +1568,6 @@ impl Queue { &'a self, snatch_guard: SnatchGuard<'a>, ) -> Result, (SubmissionIndex, DeviceError)> { - // Lock ordering requires that the fence lock be acquired after the snatch lock and - // before the command index lock. - let fence = self.device.fence.write(); - let mut command_index_guard = self.device.command_indices.write(); command_index_guard.active_submission_index += 1; let index = command_index_guard.active_submission_index; @@ -1583,7 +1579,6 @@ impl Queue { let submission = PendingSubmission { queue: self, snatch_guard, - fence, command_index_guard, executions: Vec::new(), surface_textures: FastHashMap::default(), @@ -1607,8 +1602,7 @@ impl Queue { /// [`PendingSubmission`]. /// - Advances `last_successful_submission_index` and registers the /// submission with the lifetime tracker. - /// - Returns a [`SubmissionResult`], which contains the snatch guard - /// and a downgraded [`FenceReadGuard`]. + /// - Returns a [`SubmissionResult`], which contains the snatch guard. fn submit_pending_submission<'a>( &self, mut pending_writes: MutexGuard<'_, PendingWrites>, @@ -1617,7 +1611,6 @@ impl Queue { let PendingSubmission { queue: _, snatch_guard, - mut fence, command_index_guard, mut executions, mut surface_textures, @@ -1688,7 +1681,7 @@ impl Queue { self.raw().submit( &hal_command_buffers, &submit_surface_textures, - (fence.as_mut(), submit_index), + (self.device.fence.as_ref(), submit_index), ) } .map_err(|e| self.device.handle_hal_error(e))?; @@ -1697,7 +1690,6 @@ impl Queue { // command index guard until we have submitted, to prevent another submission // from claiming the next index and reaching `submit` before we do. drop(pending_writes); - drop(command_index_guard); // Advance the successful submission index. self.device @@ -1708,10 +1700,14 @@ impl Queue { // this will register the new submission to the life time tracker self.lock_life().track_submission(submit_index, executions); - Ok(SubmissionResult { - fence: RwLockWriteGuard::downgrade(fence), - snatch_guard, - }) + // `device.maintain` relies on being able to prevent new submissions by + // using `command_index_guard` while also checking whether there are + // no tracked submissions to guarantee no new submissions will happen + // after a device is lost. This requires `command_index_guard` to be + // held over `self.lock_life()` + drop(command_index_guard); + + Ok(SubmissionResult { snatch_guard }) } pub fn get_timestamp_period(&self) -> f32 { diff --git a/wgpu-core/src/device/resource.rs b/wgpu-core/src/device/resource.rs index 399270e44..a5ac5f02a 100644 --- a/wgpu-core/src/device/resource.rs +++ b/wgpu-core/src/device/resource.rs @@ -41,7 +41,7 @@ use crate::{ TextureInitTrackerAction, }, instance::{Adapter, RequestDeviceError}, - lock::{rank, Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard}, + lock::{rank, Mutex, RwLock}, pipeline::{self, ColorStateError}, pool::ResourcePool, present, @@ -229,9 +229,7 @@ pub struct Device { /// [`active_submission_index`]: CommandIndices::active_submission_index pub(crate) last_successful_submission_index: hal::AtomicFenceValue, - // NOTE: if both are needed, the `snatchable_lock` must be consistently acquired before the - // `fence` lock to avoid deadlocks. - pub(crate) fence: RwLock>>, + pub(crate) fence: ManuallyDrop>, pub(crate) snatchable_lock: SnatchLock, /// Is this device valid? Valid is closely associated with "lose the device", @@ -288,9 +286,6 @@ pub struct Device { pub(crate) trace: Mutex>>, } -pub(crate) type FenceReadGuard<'a> = RwLockReadGuard<'a, ManuallyDrop>>; -pub(crate) type FenceWriteGuard<'a> = RwLockWriteGuard<'a, ManuallyDrop>>; - pub(crate) enum DeferredDestroy { TextureViews(WeakVec), BindGroups(WeakVec), @@ -321,7 +316,7 @@ impl Drop for Device { let default_external_texture_params_buffer = unsafe { ManuallyDrop::take(&mut self.default_external_texture_params_buffer) }; // SAFETY: We are in the Drop impl and we don't use self.fence anymore after this point. - let fence = unsafe { ManuallyDrop::take(&mut self.fence.write()) }; + let fence = unsafe { ManuallyDrop::take(&mut self.fence) }; if let Some(indirect_validation) = self.indirect_validation.take() { indirect_validation.dispose(self.raw.as_ref()); } @@ -541,7 +536,7 @@ impl Device { }, ), last_successful_submission_index: AtomicU64::new(0), - fence: RwLock::new(rank::DEVICE_FENCE, ManuallyDrop::new(fence)), + fence: ManuallyDrop::new(fence), snatchable_lock: unsafe { SnatchLock::new(rank::DEVICE_SNATCHABLE_LOCK) }, valid: AtomicBool::new(true), device_lost_closure: Mutex::new(rank::DEVICE_LOST_CLOSURE, None), @@ -799,8 +794,7 @@ impl Device { poll_type: wgt::PollType, ) -> (UserClosures, Result) { let snatch_guard = self.snatchable_lock.read(); - let fence = self.fence.read(); - let maintain_result = self.maintain(fence, poll_type, snatch_guard); + let maintain_result = self.maintain(poll_type, snatch_guard); self.lose_if_oom(); @@ -830,7 +824,6 @@ impl Device { /// if there was a timeout or a validation error. pub(crate) fn maintain<'this>( &'this self, - fence: FenceReadGuard<'_>, poll_type: wgt::PollType, snatch_guard: SnatchGuard, ) -> (UserClosures, Result) { @@ -882,7 +875,7 @@ impl Device { let wait_result = unsafe { self.raw() - .wait(fence.as_ref(), target_submission_index, wait_timeout) + .wait(self.fence.as_ref(), target_submission_index, wait_timeout) }; // This error match is only about `DeviceErrors`. At this stage we do not care if @@ -895,7 +888,7 @@ impl Device { // Get the currently finished submission index. This may be higher than the requested // wait, or it may be less than the requested wait if the wait failed. - let fence_value_result = unsafe { self.raw().get_fence_value(fence.as_ref()) }; + let fence_value_result = unsafe { self.raw().get_fence_value(self.fence.as_ref()) }; let current_finished_submission = match fence_value_result { Ok(fence_value) => fence_value, Err(e) => { @@ -904,6 +897,15 @@ impl Device { } }; + // Prevent new commands from being submitted as we want to act on `queue_empty`. + let command_indices = self.command_indices.read(); + // Check that the device is valid. This is combined with queue empty to decide whether + // to destroy all resources. Queue.submit blocks on command indices being writable + // and rejects if invalid so if the device in now invalid, and all submissions are + // finished, there will be no more submissions. + let device_valid = self.is_valid(); + drop(command_indices); + // Maintain all finished submissions on the queue, updating the relevant user closures and // collecting if the queue is empty. // @@ -974,7 +976,7 @@ impl Device { // our caller. This will complete the steps for both destroy and for // "lose the device". let mut should_release_gpu_resource = false; - if !self.is_valid() && queue_empty { + if !device_valid && queue_empty { // We can release gpu resources associated with this device (but not // while holding the life_tracker lock). should_release_gpu_resource = true; @@ -992,9 +994,6 @@ impl Device { } } - // Don't hold the locks while calling release_gpu_resources. - drop(fence); - if should_release_gpu_resource { self.release_gpu_resources(); } @@ -4965,13 +4964,11 @@ impl Device { &self, submission_index: crate::SubmissionIndex, ) -> Result<(), DeviceError> { - let fence = self.fence.read(); - let last_done_index = unsafe { self.raw().get_fence_value(fence.as_ref()) } + let last_done_index = unsafe { self.raw().get_fence_value(self.fence.as_ref()) } .map_err(|e| self.handle_hal_error(e))?; if last_done_index < submission_index { - unsafe { self.raw().wait(fence.as_ref(), submission_index, None) } + unsafe { self.raw().wait(self.fence.as_ref(), submission_index, None) } .map_err(|e| self.handle_hal_error(e))?; - drop(fence); if let Some(queue) = self.get_queue() { let closures = queue.lock_life().triage_submissions(submission_index); assert!( @@ -5224,11 +5221,10 @@ impl Device { // Wait for all work to finish before configuring the surface. let snatch_guard = self.snatchable_lock.read(); - let fence = self.fence.read(); let maintain_result; (user_callbacks, maintain_result) = - self.maintain(fence, wgt::PollType::wait_indefinitely(), snatch_guard); + self.maintain(wgt::PollType::wait_indefinitely(), snatch_guard); match maintain_result { // We're happy diff --git a/wgpu-core/src/lock/observing.rs b/wgpu-core/src/lock/observing.rs index 3c61d613c..25c25185d 100644 --- a/wgpu-core/src/lock/observing.rs +++ b/wgpu-core/src/lock/observing.rs @@ -185,15 +185,6 @@ impl<'a, T> RwLockReadGuard<'a, T> { } } -impl<'a, T> RwLockWriteGuard<'a, T> { - pub fn downgrade(this: Self) -> RwLockReadGuard<'a, T> { - RwLockReadGuard { - inner: parking_lot::RwLockWriteGuard::downgrade(this.inner), - _state: this._state, - } - } -} - impl core::fmt::Debug for RwLock { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { self.inner.fmt(f) @@ -226,11 +217,11 @@ impl<'a, T> core::ops::DerefMut for RwLockWriteGuard<'a, T> { /// /// This type serves two purposes: /// -/// - Operations like `RwLockWriteGuard::downgrade` would like to be able to -/// destructure lock guards and reassemble their pieces into new guards, but -/// if the guard type itself implements `Drop`, we can't destructure it -/// without unsafe code or pointless `Option`s whose state is almost always -/// statically known. +/// - Operations would like to be able to destructure lock guards and +/// reassemble their pieces into new guards, but if the guard type +/// itself implements `Drop`, we can't destructure it without unsafe +/// code or pointless `Option`s whose state is almost always statically +/// known. /// /// - We can just implement `Drop` for this type once, and then use it in lock /// guards, rather than implementing `Drop` separately for each guard type. diff --git a/wgpu-core/src/lock/rank.rs b/wgpu-core/src/lock/rank.rs index ba5950057..401fb9c7b 100644 --- a/wgpu-core/src/lock/rank.rs +++ b/wgpu-core/src/lock/rank.rs @@ -118,7 +118,7 @@ define_lock_ranks! { } rank DEVICE_SNATCHABLE_LOCK "Device::snatchable_lock" followed by { BUFFER_MAP_STATE, - DEVICE_FENCE, + DEVICE_COMMAND_INDICES, QUEUE_PENDING_WRITES, TEXTURE_INITIALIZATION_STATUS, QUEUE_LIFE_TRACKER, @@ -136,10 +136,6 @@ define_lock_ranks! { // Uncomment this to see an interesting cycle. // COMMAND_BUFFER_DATA, } - rank DEVICE_FENCE "Device::fence" followed by { - DEVICE_COMMAND_INDICES, - QUEUE_LIFE_TRACKER, - } rank DEVICE_COMMAND_INDICES "Device::command_indices" followed by { BUFFER_POOL, QUEUE_PENDING_WRITES, diff --git a/wgpu-core/src/lock/ranked.rs b/wgpu-core/src/lock/ranked.rs index 2e68b04dd..745300cea 100644 --- a/wgpu-core/src/lock/ranked.rs +++ b/wgpu-core/src/lock/ranked.rs @@ -110,11 +110,11 @@ impl LockState { /// /// This type serves two purposes: /// -/// - Operations like `RwLockWriteGuard::downgrade` would like to be able to -/// destructure lock guards and reassemble their pieces into new guards, but -/// if the guard type itself implements `Drop`, we can't destructure it -/// without unsafe code or pointless `Option`s whose state is almost always -/// statically known. +/// - Operations would like to be able to destructure lock guards and +/// reassemble their pieces into new guards, but if the guard type +/// itself implements `Drop`, we can't destructure it without unsafe +/// code or pointless `Option`s whose state is almost always statically +/// known. /// /// - We can just implement `Drop` for this type once, and then use it in lock /// guards, rather than implementing `Drop` separately for each guard type. @@ -299,15 +299,6 @@ impl<'a, T> RwLockReadGuard<'a, T> { } } -impl<'a, T> RwLockWriteGuard<'a, T> { - pub fn downgrade(this: Self) -> RwLockReadGuard<'a, T> { - RwLockReadGuard { - inner: parking_lot::RwLockWriteGuard::downgrade(this.inner), - saved: this.saved, - } - } -} - impl fmt::Debug for RwLock { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.inner.fmt(f) diff --git a/wgpu-core/src/lock/vanilla.rs b/wgpu-core/src/lock/vanilla.rs index 4ed38c380..969996962 100644 --- a/wgpu-core/src/lock/vanilla.rs +++ b/wgpu-core/src/lock/vanilla.rs @@ -116,12 +116,6 @@ impl<'a, T> RwLockReadGuard<'a, T> { } } -impl<'a, T> RwLockWriteGuard<'a, T> { - pub fn downgrade(this: Self) -> RwLockReadGuard<'a, T> { - RwLockReadGuard(parking_lot::RwLockWriteGuard::downgrade(this.0)) - } -} - impl fmt::Debug for RwLock { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.0.fmt(f) diff --git a/wgpu-core/src/present.rs b/wgpu-core/src/present.rs index e010e687f..b7f92b584 100644 --- a/wgpu-core/src/present.rs +++ b/wgpu-core/src/present.rs @@ -165,18 +165,14 @@ impl Surface { return Err(SurfaceError::NotConfigured); }; - let fence = device.fence.read(); - let suf = self.raw(device.backend()).unwrap(); let (texture, status) = match unsafe { suf.acquire_texture( Some(core::time::Duration::from_millis(FRAME_TIMEOUT_MS as u64)), - fence.as_ref(), + device.fence.as_ref(), ) } { Ok(ast) => { - drop(fence); - let texture_desc = wgt::TextureDescriptor { label: hal_label( Some(alloc::borrow::Cow::Borrowed("")), @@ -343,7 +339,10 @@ impl Queue { Some(resource::TextureInner::Surface { raw }) => { let raw_surface = surface.raw(device.backend()).unwrap(); let raw_queue = self.raw(); - let _fence_lock = device.fence.write(); + // [`wgpu_hal::Queue::present`] requires the queue to be synchronized with submit calls and + // other present calls. Locking command indices prevents submits which must increment the + // submission index, and by `write`ing prevents other present calls. + let _command_indices = device.command_indices.write(); unsafe { raw_queue.present(raw_surface, raw) } } _ => unreachable!(), diff --git a/wgpu-hal/src/dx12/mod.rs b/wgpu-hal/src/dx12/mod.rs index 086374e77..0ae23aeed 100644 --- a/wgpu-hal/src/dx12/mod.rs +++ b/wgpu-hal/src/dx12/mod.rs @@ -1670,7 +1670,7 @@ impl crate::Queue for Queue { &self, command_buffers: &[&CommandBuffer], _surface_textures: &[&Texture], - (signal_fence, signal_value): (&mut Fence, crate::FenceValue), + (signal_fence, signal_value): (&Fence, crate::FenceValue), ) -> Result<(), crate::DeviceError> { let mut temp_lists = self.temp_lists.lock(); temp_lists.clear(); diff --git a/wgpu-hal/src/dynamic/mod.rs b/wgpu-hal/src/dynamic/mod.rs index 85d8ca004..f45051363 100644 --- a/wgpu-hal/src/dynamic/mod.rs +++ b/wgpu-hal/src/dynamic/mod.rs @@ -57,10 +57,6 @@ trait DynResourceExt { /// /// - Panics if `self` is not downcastable to `T`. fn expect_downcast_ref(&self) -> &T; - /// # Panics - /// - /// - Panics if `self` is not downcastable to `T`. - fn expect_downcast_mut(&mut self) -> &mut T; /// Unboxes a `Box` to a concrete type. /// @@ -77,12 +73,6 @@ impl DynResourceExt for R { .expect("Resource doesn't have the expected backend type.") } - fn expect_downcast_mut<'a, T: DynResource>(&'a mut self) -> &'a mut T { - self.as_any_mut() - .downcast_mut() - .expect("Resource doesn't have the expected backend type.") - } - unsafe fn unbox(self: Box) -> T { debug_assert!( ::type_id(self.as_ref()) == TypeId::of::(), diff --git a/wgpu-hal/src/dynamic/queue.rs b/wgpu-hal/src/dynamic/queue.rs index 04744a9b4..5425b2d12 100644 --- a/wgpu-hal/src/dynamic/queue.rs +++ b/wgpu-hal/src/dynamic/queue.rs @@ -12,7 +12,7 @@ pub trait DynQueue: DynResource { &self, command_buffers: &[&dyn DynCommandBuffer], surface_textures: &[&dyn DynSurfaceTexture], - signal_fence: (&mut dyn DynFence, FenceValue), + signal_fence: (&dyn DynFence, FenceValue), ) -> Result<(), DeviceError>; unsafe fn present( &self, @@ -28,7 +28,7 @@ impl DynQueue for Q { &self, command_buffers: &[&dyn DynCommandBuffer], surface_textures: &[&dyn DynSurfaceTexture], - signal_fence: (&mut dyn DynFence, FenceValue), + signal_fence: (&dyn DynFence, FenceValue), ) -> Result<(), DeviceError> { let command_buffers = command_buffers .iter() @@ -38,7 +38,7 @@ impl DynQueue for Q { .iter() .map(|surface| (*surface).expect_downcast_ref()) .collect::>(); - let signal_fence = (signal_fence.0.expect_downcast_mut(), signal_fence.1); + let signal_fence = (signal_fence.0.expect_downcast_ref(), signal_fence.1); unsafe { Q::submit(self, &command_buffers, &surface_textures, signal_fence) } } diff --git a/wgpu-hal/src/gles/fence.rs b/wgpu-hal/src/gles/fence.rs index 201f78fcb..0470e5ba1 100644 --- a/wgpu-hal/src/gles/fence.rs +++ b/wgpu-hal/src/gles/fence.rs @@ -1,20 +1,29 @@ -use alloc::vec::Vec; +use alloc::{sync::Arc, vec::Vec}; use core::sync::atomic::Ordering; +use parking_lot::RwLock; use glow::HasContext; use crate::AtomicFenceValue; -#[derive(Debug, Copy, Clone)] +#[derive(Debug)] struct GLFence { - sync: glow::Fence, + // Since a fence can be `Copy`ed, there can exist some + // cases where a fence could be destroyed while something + // else is still using it. Therefore, while a function is + // using this fence (and doesn't keep pending read locked), + // it should clone the `Arc` to show it needs this to + // stay alive. + // + // The arc should not be kept after a function has finished + sync: Arc, value: crate::FenceValue, } #[derive(Debug)] pub struct Fence { last_completed: AtomicFenceValue, - pending: Vec, + pending: RwLock>, fence_behavior: wgt::GlFenceBehavior, } @@ -29,24 +38,27 @@ impl Fence { pub fn new(options: &wgt::GlBackendOptions) -> Self { Self { last_completed: AtomicFenceValue::new(0), - pending: Vec::new(), + pending: RwLock::new(Vec::new()), fence_behavior: options.fence_behavior, } } pub fn signal( - &mut self, + &self, gl: &glow::Context, value: crate::FenceValue, ) -> Result<(), crate::DeviceError> { if self.fence_behavior.is_auto_finish() { - *self.last_completed.get_mut() = value; + self.last_completed.store(value, Ordering::Release); return Ok(()); } let sync = unsafe { gl.fence_sync(glow::SYNC_GPU_COMMANDS_COMPLETE, 0) } .map_err(|_| crate::DeviceError::OutOfMemory)?; - self.pending.push(GLFence { sync, value }); + self.pending.write().push(GLFence { + sync: Arc::new(sync), + value, + }); Ok(()) } @@ -62,12 +74,15 @@ impl Fence { return max_value; } - for gl_fence in self.pending.iter() { + let pending = self.pending.read(); + + for gl_fence in pending.iter() { if gl_fence.value <= max_value { // We already know this was good, no need to check again continue; } - let status = unsafe { gl.get_sync_status(gl_fence.sync) }; + // We have pending `read` locked, so we shouldn't have to clone it. + let status = unsafe { gl.get_sync_status(*gl_fence.sync) }; if status == glow::SIGNALED { max_value = gl_fence.value; } else { @@ -82,20 +97,29 @@ impl Fence { max_value } - pub fn maintain(&mut self, gl: &glow::Context) { + pub fn maintain(&self, gl: &glow::Context) { if self.fence_behavior.is_auto_finish() { return; } let latest = self.get_latest(gl); - for &gl_fence in self.pending.iter() { - if gl_fence.value <= latest { + let mut pending = self.pending.write(); + pending.retain_mut(|gl_fence| { + if gl_fence.value > latest { + true + } else if let Some(fence) = Arc::get_mut(&mut gl_fence.sync) { unsafe { - gl.delete_sync(gl_fence.sync); + gl.delete_sync(*fence); } + false + } else { + // Another function is currently using this value. In general, these should finish + // very quickly (for wait because the fence should already be signaled, an all + // others are just fast), but submit should be very fast, so we shouldn't block on + // this. + true } - } - self.pending.retain(|&gl_fence| gl_fence.value > latest); + }); } pub fn wait( @@ -115,25 +139,32 @@ impl Fence { return Ok(true); } + let pending = self.pending.read(); + // Find a matching fence - let gl_fence = self - .pending - .iter() - .find(|gl_fence| gl_fence.value >= wait_value); + let gl_fence = pending.iter().find(|gl_fence| gl_fence.value >= wait_value); let Some(gl_fence) = gl_fence else { log::warn!("Tried to wait for {wait_value} but that value has not been signalled yet"); return Ok(false); }; + // clone to show we're using the fence + let sync = gl_fence.sync.clone(); + let fence_value = gl_fence.value; + + drop(pending); + let status = unsafe { gl.client_wait_sync( - gl_fence.sync, + *sync, glow::SYNC_FLUSH_COMMANDS_BIT, timeout_ns.min(i32::MAX as u32) as i32, ) }; + drop(sync); + let signalled = match status { glow::ALREADY_SIGNALED | glow::CONDITION_SATISFIED => true, glow::TIMEOUT_EXPIRED | glow::WAIT_FAILED => false, @@ -144,8 +175,7 @@ impl Fence { }; if signalled { - self.last_completed - .fetch_max(gl_fence.value, Ordering::AcqRel); + self.last_completed.fetch_max(fence_value, Ordering::AcqRel); } Ok(signalled) @@ -156,9 +186,12 @@ impl Fence { return; } - for gl_fence in self.pending { + for gl_fence in self.pending.into_inner() { unsafe { - gl.delete_sync(gl_fence.sync); + gl.delete_sync( + Arc::into_inner(gl_fence.sync) + .expect("A function has failed to drop all its references to this"), + ); } } } diff --git a/wgpu-hal/src/gles/queue.rs b/wgpu-hal/src/gles/queue.rs index b84d60e25..816ef8f55 100644 --- a/wgpu-hal/src/gles/queue.rs +++ b/wgpu-hal/src/gles/queue.rs @@ -1904,7 +1904,7 @@ impl crate::Queue for super::Queue { &self, command_buffers: &[&super::CommandBuffer], _surface_textures: &[&super::Texture], - (signal_fence, signal_value): (&mut super::Fence, crate::FenceValue), + (signal_fence, signal_value): (&super::Fence, crate::FenceValue), ) -> Result<(), crate::DeviceError> { let shared = Arc::clone(&self.shared); let gl = &shared.context.lock(); diff --git a/wgpu-hal/src/lib.rs b/wgpu-hal/src/lib.rs index 1ef3c062a..1d5e2c0f9 100644 --- a/wgpu-hal/src/lib.rs +++ b/wgpu-hal/src/lib.rs @@ -634,6 +634,9 @@ pub trait Api: Clone + fmt::Debug + Sized + WasmNotSendSync + 'static { /// before a lower-valued operation, then waiting for the fence to reach the /// lower value could return before the lower-valued operation has actually /// finished. + /// + /// Fences are internally synchronised by the hal, and so should not need to be + /// contained in external synchronisation primitives. type Fence: DynFence; type BindGroupLayout: DynBindGroupLayout; @@ -717,6 +720,8 @@ pub trait Surface: WasmNotSendSync { /// [`SurfaceTexture`] to [`self.discard_texture`], so that it can be reused /// by future acquisitions. /// + /// The fence is internally synchronised by the hal. + /// /// # Portability /// /// Some backends can't support a timeout when acquiring a texture. On these @@ -1245,7 +1250,7 @@ pub trait Queue: WasmNotSendSync { &self, command_buffers: &[&::CommandBuffer], surface_textures: &[&::SurfaceTexture], - signal_fence: (&mut ::Fence, FenceValue), + signal_fence: (&::Fence, FenceValue), ) -> Result<(), DeviceError>; /// Present a surface texture to the screen. /// diff --git a/wgpu-hal/src/metal/device.rs b/wgpu-hal/src/metal/device.rs index 508da11cc..70bf2a7ee 100644 --- a/wgpu-hal/src/metal/device.rs +++ b/wgpu-hal/src/metal/device.rs @@ -1,5 +1,6 @@ use alloc::{borrow::ToOwned as _, sync::Arc, vec::Vec}; use core::{ptr::NonNull, sync::atomic}; +use parking_lot::RwLock; use std::{thread, time}; use bytemuck::TransparentWrapper; @@ -1886,7 +1887,7 @@ impl crate::Device for super::Device { }; Ok(super::Fence { completed_value: Arc::new(atomic::AtomicU64::new(0)), - pending_command_buffers: Vec::new(), + pending_command_buffers: RwLock::new(Vec::new()), shared_event, }) } @@ -1897,7 +1898,8 @@ impl crate::Device for super::Device { unsafe fn get_fence_value(&self, fence: &super::Fence) -> DeviceResult { let mut max_value = fence.completed_value.load(atomic::Ordering::Acquire); - for &(value, ref cmd_buf) in fence.pending_command_buffers.iter() { + let pending_command_buffers = fence.pending_command_buffers.read(); + for &(value, ref cmd_buf) in pending_command_buffers.iter() { if cmd_buf.status() == MTLCommandBufferStatus::Completed { max_value = value; } @@ -1914,18 +1916,22 @@ impl crate::Device for super::Device { return Ok(true); } - let cmd_buf = match fence - .pending_command_buffers + let pending_command_buffers = fence.pending_command_buffers.read(); + + let cmd_buf = match pending_command_buffers .iter() .find(|&&(value, _)| value >= wait_value) { - Some((_, cmd_buf)) => cmd_buf, + Some((_, cmd_buf)) => cmd_buf.clone(), None => { log::error!("No active command buffers for fence value {wait_value}"); return Err(crate::DeviceError::Lost); } }; + // Make sure that nothing is blocked during the actual wait. + drop(pending_command_buffers); + let start = time::Instant::now(); loop { if let MTLCommandBufferStatus::Completed = cmd_buf.status() { diff --git a/wgpu-hal/src/metal/mod.rs b/wgpu-hal/src/metal/mod.rs index 1b55f9820..fa2138cc6 100644 --- a/wgpu-hal/src/metal/mod.rs +++ b/wgpu-hal/src/metal/mod.rs @@ -538,7 +538,7 @@ impl crate::Queue for Queue { &self, command_buffers: &[&CommandBuffer], _surface_textures: &[&SurfaceTexture], - (signal_fence, signal_value): (&mut Fence, crate::FenceValue), + (signal_fence, signal_value): (&Fence, crate::FenceValue), ) -> Result<(), crate::DeviceError> { autoreleasepool(|_| { let extra_command_buffer = { @@ -564,6 +564,7 @@ impl crate::Queue for Queue { signal_fence.maintain(); signal_fence .pending_command_buffers + .write() .push((signal_value, raw.clone())); if let Some(shared_event) = &signal_fence.shared_event { @@ -1029,13 +1030,15 @@ unsafe impl Sync for QuerySet {} pub struct Fence { completed_value: Arc, /// The pending fence values have to be ascending. - pending_command_buffers: Vec<( - crate::FenceValue, - Retained>, - )>, + pending_command_buffers: RwLock>, shared_event: Option>>, } +type PendingCommandBuffer = ( + crate::FenceValue, + Retained>, +); + impl crate::DynFence for Fence {} unsafe impl Send for Fence {} @@ -1044,7 +1047,8 @@ unsafe impl Sync for Fence {} impl Fence { fn get_latest(&self) -> crate::FenceValue { let mut max_value = self.completed_value.load(atomic::Ordering::Acquire); - for &(value, ref cmd_buf) in self.pending_command_buffers.iter() { + let pending_command_buffers = self.pending_command_buffers.read(); + for &(value, ref cmd_buf) in pending_command_buffers.iter() { if cmd_buf.status() == MTLCommandBufferStatus::Completed { max_value = value; } @@ -1052,9 +1056,10 @@ impl Fence { max_value } - fn maintain(&mut self) { + fn maintain(&self) { let latest = self.get_latest(); self.pending_command_buffers + .write() .retain(|&(value, _)| value > latest); } diff --git a/wgpu-hal/src/noop/mod.rs b/wgpu-hal/src/noop/mod.rs index 563b1b27d..c8491ded4 100644 --- a/wgpu-hal/src/noop/mod.rs +++ b/wgpu-hal/src/noop/mod.rs @@ -262,7 +262,7 @@ impl crate::Queue for Context { &self, command_buffers: &[&CommandBuffer], surface_textures: &[&Resource], - (fence, fence_value): (&mut Fence, crate::FenceValue), + (fence, fence_value): (&Fence, crate::FenceValue), ) -> DeviceResult<()> { // All commands are executed synchronously. for cb in command_buffers { diff --git a/wgpu-hal/src/vulkan/device.rs b/wgpu-hal/src/vulkan/device.rs index f5f76de4e..c7c8129ad 100644 --- a/wgpu-hal/src/vulkan/device.rs +++ b/wgpu-hal/src/vulkan/device.rs @@ -10,7 +10,7 @@ use core::{ use arrayvec::ArrayVec; use ash::{ext, vk}; use hashbrown::hash_map::Entry; -use parking_lot::Mutex; +use parking_lot::{Mutex, RwLock}; use super::{conv, descriptor::DescriptorCounts, RawTlasInstance}; use crate::TlasInstance; @@ -2279,11 +2279,11 @@ impl crate::Device for super::Device { super::Fence::TimelineSemaphore(raw) } else { - super::Fence::FencePool { + super::Fence::FencePool(RwLock::new(super::FencePool { last_completed: 0, active: Vec::new(), free: Vec::new(), - } + })) }) } unsafe fn destroy_fence(&self, fence: super::Fence) { @@ -2291,13 +2291,17 @@ impl crate::Device for super::Device { super::Fence::TimelineSemaphore(raw) => { unsafe { self.shared.raw.destroy_semaphore(raw, None) }; } - super::Fence::FencePool { - active, - free, - last_completed: _, - } => { + super::Fence::FencePool(pool) => { + let super::FencePool { + active, + free, + last_completed: _, + } = pool.into_inner(); + for (_, raw) in active { - unsafe { self.shared.raw.destroy_fence(raw, None) }; + unsafe { + self.shared.raw.destroy_fence(Arc::into_inner(raw).expect("Fence should have its reference count be one by the end of each function"), None) + }; } for raw in free { unsafe { self.shared.raw.destroy_fence(raw, None) }; @@ -2829,17 +2833,28 @@ impl super::DeviceShared { Err(other) => Err(super::map_host_device_oom_and_lost_err(other)), } } - super::Fence::FencePool { - last_completed, - ref active, - free: _, - } => { + super::Fence::FencePool(ref pool) => { + let pool = pool.read(); + let super::FencePool { + last_completed, + ref active, + free: _, + } = *pool; if wait_value <= last_completed { Ok(true) } else { match active.iter().find(|&&(value, _)| value >= wait_value) { - Some(&(_, raw)) => { - match unsafe { self.raw.wait_for_fences(&[raw], true, timeout_ns) } { + Some((_, fence)) => { + // clone to show we are using this fence while the pool is unlocked. + let fence = fence.clone(); + drop(pool); + match unsafe { + self.raw.wait_for_fences( + core::slice::from_ref(&fence), + true, + timeout_ns, + ) + } { Ok(()) => Ok(true), Err(vk::Result::TIMEOUT) => Ok(false), Err(other) => Err(super::map_host_device_oom_and_lost_err(other)), diff --git a/wgpu-hal/src/vulkan/mod.rs b/wgpu-hal/src/vulkan/mod.rs index 124df07c8..7d18b6bd4 100644 --- a/wgpu-hal/src/vulkan/mod.rs +++ b/wgpu-hal/src/vulkan/mod.rs @@ -1162,14 +1162,29 @@ pub enum Fence { /// for each queue submission we might want to wait for, and remember which /// [`FenceValue`] each one represents. /// + /// One should keep the fence pool read while there are any references to the + /// fences inside of them. This ensures there are no race conditions when + /// resetting the fences + /// /// [fence]: https://registry.khronos.org/vulkan/specs/1.3-extensions/html/vkspec.html#synchronization-fences /// [`FenceValue`]: crate::FenceValue - FencePool { - last_completed: crate::FenceValue, - /// The pending fence values have to be ascending. - active: Vec<(crate::FenceValue, vk::Fence)>, - free: Vec, - }, + FencePool(RwLock), +} + +/// A shared fence type. The arc is expect to have a ref-count of one once a function has finished being called +/// +/// A fence should have access synchronised as fence resetting might happen at any point. Resetting checks the ref-count +/// of the fence, so instead of copying the fence, it should have its `Arc` container cloned which shows not to reset +/// this fence as it is being used. +pub(super) type SynchronizedFence = Arc; + +#[derive(Debug)] +pub struct FencePool { + last_completed: crate::FenceValue, + /// The pending fence values have to be ascending. + active: Vec<(crate::FenceValue, SynchronizedFence)>, + // Don't need extra synchronisation around the fences here, if they are used they should be put into active. + free: Vec, } impl crate::DynFence for Fence {} @@ -1186,13 +1201,15 @@ impl Fence { fn check_active( device: &ash::Device, mut last_completed: crate::FenceValue, - active: &[(crate::FenceValue, vk::Fence)], + active: &[(crate::FenceValue, SynchronizedFence)], ) -> Result { - for &(value, raw) in active.iter() { + for &(value, ref raw) in active.iter() { unsafe { if value > last_completed && device - .get_fence_status(raw) + // Don't need to clone as active should be from a read or + // write lock which means this is already synchronised. + .get_fence_status(**raw) .map_err(map_host_device_oom_and_lost_err)? { last_completed = value; @@ -1221,11 +1238,14 @@ impl Fence { .map_err(map_host_device_oom_and_lost_err)?, }) }, - Self::FencePool { - last_completed, - ref active, - free: _, - } => Self::check_active(device, last_completed, active), + Self::FencePool(ref pool) => { + let FencePool { + last_completed, + ref active, + free: _, + } = *pool.read(); + Self::check_active(device, last_completed, active) + } } } @@ -1241,23 +1261,35 @@ impl Fence { /// /// [`FencePool`]: Fence::FencePool /// [`Queue::submit`]: crate::Queue::submit - fn maintain(&mut self, device: &ash::Device) -> Result<(), crate::DeviceError> { + fn maintain(&self, device: &ash::Device) -> Result<(), crate::DeviceError> { match *self { Self::TimelineSemaphore(_) => {} - Self::FencePool { - ref mut last_completed, - ref mut active, - ref mut free, - } => { - let latest = Self::check_active(device, *last_completed, active)?; + Self::FencePool(ref pool) => { + let FencePool { + ref mut last_completed, + ref mut active, + ref mut free, + } = *pool.write(); + let base_free = free.len(); - for &(value, raw) in active.iter() { - if value <= latest { - free.push(raw); + let latest = Self::check_active(device, *last_completed, active)?; + + active.retain_mut(|&mut (value, ref mut fence)| { + if value > latest { + true + } else if let Some(fence) = Arc::get_mut(fence) { + // No other references to these, so we have exclusive access. Add them to free and reset them later, + // but drop them from active immediately + free.push(*fence); + false + } else { + // some other function is using it. Although this shouldn't be to long, + // maintain shouldn't block, and it should be cleared up by the next time it happens + true } - } + }); + if free.len() != base_free { - active.retain(|&(value, _)| value > latest); unsafe { device.reset_fences(&free[base_free..]) } .map_err(map_device_oom_err)? } @@ -1275,7 +1307,7 @@ impl crate::Queue for Queue { &self, command_buffers: &[&CommandBuffer], surface_textures: &[&SurfaceTexture], - (signal_fence, signal_value): (&mut Fence, crate::FenceValue), + (signal_fence, signal_value): (&Fence, crate::FenceValue), ) -> Result<(), crate::DeviceError> { let mut fence_raw = vk::Fence::null(); @@ -1343,25 +1375,33 @@ impl crate::Queue for Queue { // We need to signal our wgpu::Fence if we have one, this adds it to the signal list. signal_fence.maintain(&self.device.raw)?; + // Keeping the Arc around is probably unneeded - the fence should never be signaled as it was reset, + // and newer submits should not happen until this submit is done. Therefore, it should be too high + // to be reset. + let shared_fence; match *signal_fence { Fence::TimelineSemaphore(raw) => { signal_semaphores.push_signal(SemaphoreType::Timeline(raw, signal_value)); } - Fence::FencePool { - ref mut active, - ref mut free, - .. - } => { - fence_raw = match free.pop() { - Some(raw) => raw, + Fence::FencePool(ref pool) => { + let FencePool { + ref mut active, + ref mut free, + .. + } = *pool.write(); + shared_fence = match free.pop() { + Some(raw) => Arc::new(raw), None => unsafe { - self.device + let fence = self + .device .raw .create_fence(&vk::FenceCreateInfo::default(), None) - .map_err(map_host_device_oom_err)? + .map_err(map_host_device_oom_err)?; + Arc::new(fence) }, }; - active.push((signal_value, fence_raw)); + fence_raw = *shared_fence; + active.push((signal_value, shared_fence.clone())); } }