Move fence synchronisation into wgpu-hal (#9475)

This commit is contained in:
Vecvec authored and GitHub committed 2026-05-28 14:49:12 -07:00
1 parent 5c4cdc16a4
commit a1fceed071
21 files changed
+260 -271

No files matched your search

+1
View File
@@ -185,6 +185,7 @@ By @beholdnec in [#8505](https://github.com/gfx-rs/wgpu/pull/8505).
- BREAKING: `map_label` helpers have changed slightly. By @beicause and @andyleiserson in [#9480](https://github.com/gfx-rs/wgpu/pull/9480), [#9481](https://github.com/gfx-rs/wgpu/pull/9481), and [#9526](https://github.com/gfx-rs/wgpu/pull/9526).
- `TextureDescriptor::map_label_and_view_formats` and `SurfaceConfiguration::map_view_formats` now take `FnOnce(&V)` instead of `FnOnce(V)`.
- All `map_label` helpers except `CreateShaderModuleDescriptorPassthrough` now have the signature `map_label<'a, K>(&'a self, fun: impl FnOnce(&'a L) -> K)` (previously the lifetimes were implicit and thus could differ).
- Relaxed locking within `wgpu-core` to enable queue submission processing on one thread to proceed while another thread is blocked in a device poll. To facilitate this, `wgpu-hal` fences are now internally synchronized. By @Vecvec in [#9475](https://github.com/gfx-rs/wgpu/pull/9475).
#### Validation
+2 -70
View File
@@ -4,13 +4,12 @@ use alloc::sync::Arc;
use hal::DynResource;
use crate::{
device::Device,
global::Global,
id::{
AdapterId, BlasId, BufferId, CommandEncoderId, DeviceId, QueueId, SurfaceId, TextureId,
TextureViewId, TlasId,
},
lock::{RankData, RwLockReadGuard},
lock::RankData,
resource::RawResourceAccess,
snatch::SnatchGuard,
};
@@ -154,73 +153,6 @@ where
{
}
/// A guard which holds alive a device and the device's fence lock, dereferencing to the Hal type.
struct FenceGuard<Fence> {
device: Arc<Device>,
fence_lock_rank_data: ManuallyDrop<RankData>,
ptr: *const Fence,
}
impl<Fence> FenceGuard<Fence>
where
Fence: 'static,
{
/// Creates a new guard over a device's fence.
///
/// Returns `None` if:
/// - The device's fence is not of the expected Hal type.
pub fn new(device: Arc<Device>) -> Option<Self> {
// Grab the fence lock.
let fence_guard = device.fence.read();
// Get the raw fence and downcast it to the expected Hal type, coercing it to a pointer
// to get rid of the lifetime connecting us to the fence guard.
let ptr: *const Fence = fence_guard.as_any().downcast_ref::<Fence>()?;
// SAFETY: At this point all panicking or divergance has already happened,
// so we can safely forget the fence guard without causing the lock to be left open.
let fence_lock_rank_data = RwLockReadGuard::forget(fence_guard);
// SAFETY: We only construct this guard while the fence lock is held,
// as the `drop` implementation of this guard will unsafely release the lock.
Some(Self {
device,
fence_lock_rank_data: ManuallyDrop::new(fence_lock_rank_data),
ptr,
})
}
}
impl<Fence> Deref for FenceGuard<Fence> {
type Target = Fence;
fn deref(&self) -> &Self::Target {
// SAFETY: The pointer is guaranteed to be valid as the original device's fence
// is still alive and the fence lock is still being held due to the forgotten
// fence guard.
unsafe { &*self.ptr }
}
}
impl<Fence> Drop for FenceGuard<Fence> {
fn drop(&mut self) {
// SAFETY:
// - We are not going to access the rank data anymore.
let data = unsafe { ManuallyDrop::take(&mut self.fence_lock_rank_data) };
// SAFETY:
// - The pointer is no longer going to be accessed.
// - The fence lock is being held because this type was not created
// until after the fence lock was forgotten.
unsafe {
self.device.fence.force_unlock_read(data);
};
}
}
unsafe impl<Fence> Send for FenceGuard<Fence> where Fence: Send {}
unsafe impl<Fence> Sync for FenceGuard<Fence> where Fence: Sync {}
impl Global {
/// # Safety
///
@@ -312,7 +244,7 @@ impl Global {
let device = self.hub.devices.get(id);
FenceGuard::new(device)
SimpleResourceGuard::new(device, move |device| device.fence.as_any().downcast_ref())
}
/// # Safety
-1
View File
@@ -28,7 +28,6 @@ pub mod ray_tracing;
pub mod resource;
#[cfg(any(feature = "trace", feature = "replay"))]
pub mod trace;
pub(crate) use resource::{FenceReadGuard, FenceWriteGuard};
pub use {life::WaitIdleError, resource::Device};
pub const SHADER_STAGE_COUNT: usize = hal::MAX_CONCURRENT_SHADER_STAGES;
+20 -24
View File
@@ -24,7 +24,7 @@ use crate::{
CommandAllocator, CommandBuffer, CommandEncoder, CommandEncoderError, CopySide,
TransferError,
},
device::{DeviceError, FenceReadGuard, FenceWriteGuard, WaitIdleError},
device::{DeviceError, WaitIdleError},
get_lowest_common_denom,
global::Global,
hal_label,
@@ -227,6 +227,13 @@ impl Queue {
Ok(())
}
/// Maintains the queue's list of finished command buffers.
///
/// Command buffers submitted before `submission_index` was submitted
/// stop being tracked and callbacks which are waiting for them are
/// returned. Also returned is whether the queue is empty. This may
/// be stale unless new submissions are prevented by locking
/// [`Device::command_indices`]
pub(crate) fn maintain(
&self,
submission_index: u64,
@@ -575,7 +582,6 @@ impl WebGpuError for QueueSubmitError {
pub(crate) struct PendingSubmission<'a> {
queue: &'a Queue,
snatch_guard: SnatchGuard<'a>,
fence: FenceWriteGuard<'a>,
command_index_guard: RwLockWriteGuard<'a, CommandIndices>,
// Command buffers to be executed, along with trackers for the resources they use.
pub executions: Vec<EncoderInFlight>,
@@ -587,7 +593,6 @@ pub(crate) struct PendingSubmission<'a> {
}
pub(crate) struct SubmissionResult<'a> {
pub fence: FenceReadGuard<'a>,
pub snatch_guard: SnatchGuard<'a>,
}
@@ -1494,10 +1499,7 @@ impl Queue {
let pending_writes = self.pending_writes.lock();
let SubmissionResult {
fence,
snatch_guard,
} = match submission.submit(pending_writes) {
let SubmissionResult { snatch_guard } = match submission.submit(pending_writes) {
Ok(result) => result,
Err(e) => break 'error Err(e.into()),
};
@@ -1507,9 +1509,7 @@ impl Queue {
// This will schedule destruction of all resources that are no longer needed
// by the user but used in the command stream, among other things.
// `device.maintain` consumes and will release the snatch guard.
let (closures, result) = self
.device
.maintain(fence, wgt::PollType::Poll, snatch_guard);
let (closures, result) = self.device.maintain(wgt::PollType::Poll, snatch_guard);
match result {
Ok(status) => {
debug_assert!(matches!(
@@ -1568,10 +1568,6 @@ impl Queue {
&'a self,
snatch_guard: SnatchGuard<'a>,
) -> Result<PendingSubmission<'a>, (SubmissionIndex, DeviceError)> {
// Lock ordering requires that the fence lock be acquired after the snatch lock and
// before the command index lock.
let fence = self.device.fence.write();
let mut command_index_guard = self.device.command_indices.write();
command_index_guard.active_submission_index += 1;
let index = command_index_guard.active_submission_index;
@@ -1583,7 +1579,6 @@ impl Queue {
let submission = PendingSubmission {
queue: self,
snatch_guard,
fence,
command_index_guard,
executions: Vec::new(),
surface_textures: FastHashMap::default(),
@@ -1607,8 +1602,7 @@ impl Queue {
/// [`PendingSubmission`].
/// - Advances `last_successful_submission_index` and registers the
/// submission with the lifetime tracker.
/// - Returns a [`SubmissionResult`], which contains the snatch guard
/// and a downgraded [`FenceReadGuard`].
/// - Returns a [`SubmissionResult`], which contains the snatch guard.
fn submit_pending_submission<'a>(
&self,
mut pending_writes: MutexGuard<'_, PendingWrites>,
@@ -1617,7 +1611,6 @@ impl Queue {
let PendingSubmission {
queue: _,
snatch_guard,
mut fence,
command_index_guard,
mut executions,
mut surface_textures,
@@ -1688,7 +1681,7 @@ impl Queue {
self.raw().submit(
&hal_command_buffers,
&submit_surface_textures,
(fence.as_mut(), submit_index),
(self.device.fence.as_ref(), submit_index),
)
}
.map_err(|e| self.device.handle_hal_error(e))?;
@@ -1697,7 +1690,6 @@ impl Queue {
// command index guard until we have submitted, to prevent another submission
// from claiming the next index and reaching `submit` before we do.
drop(pending_writes);
drop(command_index_guard);
// Advance the successful submission index.
self.device
@@ -1708,10 +1700,14 @@ impl Queue {
// this will register the new submission to the life time tracker
self.lock_life().track_submission(submit_index, executions);
Ok(SubmissionResult {
fence: RwLockWriteGuard::downgrade(fence),
snatch_guard,
})
// `device.maintain` relies on being able to prevent new submissions by
// using `command_index_guard` while also checking whether there are
// no tracked submissions to guarantee no new submissions will happen
// after a device is lost. This requires `command_index_guard` to be
// held over `self.lock_life()`
drop(command_index_guard);
Ok(SubmissionResult { snatch_guard })
}
pub fn get_timestamp_period(&self) -> f32 {
+20 -24
View File
@@ -41,7 +41,7 @@ use crate::{
TextureInitTrackerAction,
},
instance::{Adapter, RequestDeviceError},
lock::{rank, Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard},
lock::{rank, Mutex, RwLock},
pipeline::{self, ColorStateError},
pool::ResourcePool,
present,
@@ -229,9 +229,7 @@ pub struct Device {
/// [`active_submission_index`]: CommandIndices::active_submission_index
pub(crate) last_successful_submission_index: hal::AtomicFenceValue,
// NOTE: if both are needed, the `snatchable_lock` must be consistently acquired before the
// `fence` lock to avoid deadlocks.
pub(crate) fence: RwLock<ManuallyDrop<Box<dyn hal::DynFence>>>,
pub(crate) fence: ManuallyDrop<Box<dyn hal::DynFence>>,
pub(crate) snatchable_lock: SnatchLock,
/// Is this device valid? Valid is closely associated with "lose the device",
@@ -288,9 +286,6 @@ pub struct Device {
pub(crate) trace: Mutex<Option<Box<dyn trace::Trace + Send + Sync + 'static>>>,
}
pub(crate) type FenceReadGuard<'a> = RwLockReadGuard<'a, ManuallyDrop<Box<dyn hal::DynFence>>>;
pub(crate) type FenceWriteGuard<'a> = RwLockWriteGuard<'a, ManuallyDrop<Box<dyn hal::DynFence>>>;
pub(crate) enum DeferredDestroy {
TextureViews(WeakVec<TextureView>),
BindGroups(WeakVec<BindGroup>),
@@ -321,7 +316,7 @@ impl Drop for Device {
let default_external_texture_params_buffer =
unsafe { ManuallyDrop::take(&mut self.default_external_texture_params_buffer) };
// SAFETY: We are in the Drop impl and we don't use self.fence anymore after this point.
let fence = unsafe { ManuallyDrop::take(&mut self.fence.write()) };
let fence = unsafe { ManuallyDrop::take(&mut self.fence) };
if let Some(indirect_validation) = self.indirect_validation.take() {
indirect_validation.dispose(self.raw.as_ref());
}
@@ -541,7 +536,7 @@ impl Device {
},
),
last_successful_submission_index: AtomicU64::new(0),
fence: RwLock::new(rank::DEVICE_FENCE, ManuallyDrop::new(fence)),
fence: ManuallyDrop::new(fence),
snatchable_lock: unsafe { SnatchLock::new(rank::DEVICE_SNATCHABLE_LOCK) },
valid: AtomicBool::new(true),
device_lost_closure: Mutex::new(rank::DEVICE_LOST_CLOSURE, None),
@@ -799,8 +794,7 @@ impl Device {
poll_type: wgt::PollType<crate::SubmissionIndex>,
) -> (UserClosures, Result<wgt::PollStatus, WaitIdleError>) {
let snatch_guard = self.snatchable_lock.read();
let fence = self.fence.read();
let maintain_result = self.maintain(fence, poll_type, snatch_guard);
let maintain_result = self.maintain(poll_type, snatch_guard);
self.lose_if_oom();
@@ -830,7 +824,6 @@ impl Device {
/// if there was a timeout or a validation error.
pub(crate) fn maintain<'this>(
&'this self,
fence: FenceReadGuard<'_>,
poll_type: wgt::PollType<crate::SubmissionIndex>,
snatch_guard: SnatchGuard,
) -> (UserClosures, Result<wgt::PollStatus, WaitIdleError>) {
@@ -882,7 +875,7 @@ impl Device {
let wait_result = unsafe {
self.raw()
.wait(fence.as_ref(), target_submission_index, wait_timeout)
.wait(self.fence.as_ref(), target_submission_index, wait_timeout)
};
// This error match is only about `DeviceErrors`. At this stage we do not care if
@@ -895,7 +888,7 @@ impl Device {
// Get the currently finished submission index. This may be higher than the requested
// wait, or it may be less than the requested wait if the wait failed.
let fence_value_result = unsafe { self.raw().get_fence_value(fence.as_ref()) };
let fence_value_result = unsafe { self.raw().get_fence_value(self.fence.as_ref()) };
let current_finished_submission = match fence_value_result {
Ok(fence_value) => fence_value,
Err(e) => {
@@ -904,6 +897,15 @@ impl Device {
}
};
// Prevent new commands from being submitted as we want to act on `queue_empty`.
let command_indices = self.command_indices.read();
// Check that the device is valid. This is combined with queue empty to decide whether
// to destroy all resources. Queue.submit blocks on command indices being writable
// and rejects if invalid so if the device in now invalid, and all submissions are
// finished, there will be no more submissions.
let device_valid = self.is_valid();
drop(command_indices);
// Maintain all finished submissions on the queue, updating the relevant user closures and
// collecting if the queue is empty.
//
@@ -974,7 +976,7 @@ impl Device {
// our caller. This will complete the steps for both destroy and for
// "lose the device".
let mut should_release_gpu_resource = false;
if !self.is_valid() && queue_empty {
if !device_valid && queue_empty {
// We can release gpu resources associated with this device (but not
// while holding the life_tracker lock).
should_release_gpu_resource = true;
@@ -992,9 +994,6 @@ impl Device {
}
}
// Don't hold the locks while calling release_gpu_resources.
drop(fence);
if should_release_gpu_resource {
self.release_gpu_resources();
}
@@ -4965,13 +4964,11 @@ impl Device {
&self,
submission_index: crate::SubmissionIndex,
) -> Result<(), DeviceError> {
let fence = self.fence.read();
let last_done_index = unsafe { self.raw().get_fence_value(fence.as_ref()) }
let last_done_index = unsafe { self.raw().get_fence_value(self.fence.as_ref()) }
.map_err(|e| self.handle_hal_error(e))?;
if last_done_index < submission_index {
unsafe { self.raw().wait(fence.as_ref(), submission_index, None) }
unsafe { self.raw().wait(self.fence.as_ref(), submission_index, None) }
.map_err(|e| self.handle_hal_error(e))?;
drop(fence);
if let Some(queue) = self.get_queue() {
let closures = queue.lock_life().triage_submissions(submission_index);
assert!(
@@ -5224,11 +5221,10 @@ impl Device {
// Wait for all work to finish before configuring the surface.
let snatch_guard = self.snatchable_lock.read();
let fence = self.fence.read();
let maintain_result;
(user_callbacks, maintain_result) =
self.maintain(fence, wgt::PollType::wait_indefinitely(), snatch_guard);
self.maintain(wgt::PollType::wait_indefinitely(), snatch_guard);
match maintain_result {
// We're happy
+5 -14
View File
@@ -185,15 +185,6 @@ impl<'a, T> RwLockReadGuard<'a, T> {
}
}
impl<'a, T> RwLockWriteGuard<'a, T> {
pub fn downgrade(this: Self) -> RwLockReadGuard<'a, T> {
RwLockReadGuard {
inner: parking_lot::RwLockWriteGuard::downgrade(this.inner),
_state: this._state,
}
}
}
impl<T: core::fmt::Debug> core::fmt::Debug for RwLock<T> {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
self.inner.fmt(f)
@@ -226,11 +217,11 @@ impl<'a, T> core::ops::DerefMut for RwLockWriteGuard<'a, T> {
///
/// This type serves two purposes:
///
/// - Operations like `RwLockWriteGuard::downgrade` would like to be able to
/// destructure lock guards and reassemble their pieces into new guards, but
/// if the guard type itself implements `Drop`, we can't destructure it
/// without unsafe code or pointless `Option`s whose state is almost always
/// statically known.
/// - Operations would like to be able to destructure lock guards and
/// reassemble their pieces into new guards, but if the guard type
/// itself implements `Drop`, we can't destructure it without unsafe
/// code or pointless `Option`s whose state is almost always statically
/// known.
///
/// - We can just implement `Drop` for this type once, and then use it in lock
/// guards, rather than implementing `Drop` separately for each guard type.
+1 -5
View File
@@ -118,7 +118,7 @@ define_lock_ranks! {
}
rank DEVICE_SNATCHABLE_LOCK "Device::snatchable_lock" followed by {
BUFFER_MAP_STATE,
DEVICE_FENCE,
DEVICE_COMMAND_INDICES,
QUEUE_PENDING_WRITES,
TEXTURE_INITIALIZATION_STATUS,
QUEUE_LIFE_TRACKER,
@@ -136,10 +136,6 @@ define_lock_ranks! {
// Uncomment this to see an interesting cycle.
// COMMAND_BUFFER_DATA,
}
rank DEVICE_FENCE "Device::fence" followed by {
DEVICE_COMMAND_INDICES,
QUEUE_LIFE_TRACKER,
}
rank DEVICE_COMMAND_INDICES "Device::command_indices" followed by {
BUFFER_POOL,
QUEUE_PENDING_WRITES,
+5 -14
View File
@@ -110,11 +110,11 @@ impl LockState {
///
/// This type serves two purposes:
///
/// - Operations like `RwLockWriteGuard::downgrade` would like to be able to
/// destructure lock guards and reassemble their pieces into new guards, but
/// if the guard type itself implements `Drop`, we can't destructure it
/// without unsafe code or pointless `Option`s whose state is almost always
/// statically known.
/// - Operations would like to be able to destructure lock guards and
/// reassemble their pieces into new guards, but if the guard type
/// itself implements `Drop`, we can't destructure it without unsafe
/// code or pointless `Option`s whose state is almost always statically
/// known.
///
/// - We can just implement `Drop` for this type once, and then use it in lock
/// guards, rather than implementing `Drop` separately for each guard type.
@@ -299,15 +299,6 @@ impl<'a, T> RwLockReadGuard<'a, T> {
}
}
impl<'a, T> RwLockWriteGuard<'a, T> {
pub fn downgrade(this: Self) -> RwLockReadGuard<'a, T> {
RwLockReadGuard {
inner: parking_lot::RwLockWriteGuard::downgrade(this.inner),
saved: this.saved,
}
}
}
impl<T: fmt::Debug> fmt::Debug for RwLock<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
self.inner.fmt(f)
-6
View File
@@ -116,12 +116,6 @@ impl<'a, T> RwLockReadGuard<'a, T> {
}
}
impl<'a, T> RwLockWriteGuard<'a, T> {
pub fn downgrade(this: Self) -> RwLockReadGuard<'a, T> {
RwLockReadGuard(parking_lot::RwLockWriteGuard::downgrade(this.0))
}
}
impl<T: fmt::Debug> fmt::Debug for RwLock<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
self.0.fmt(f)
+5 -6
View File
@@ -165,18 +165,14 @@ impl Surface {
return Err(SurfaceError::NotConfigured);
};
let fence = device.fence.read();
let suf = self.raw(device.backend()).unwrap();
let (texture, status) = match unsafe {
suf.acquire_texture(
Some(core::time::Duration::from_millis(FRAME_TIMEOUT_MS as u64)),
fence.as_ref(),
device.fence.as_ref(),
)
} {
Ok(ast) => {
drop(fence);
let texture_desc = wgt::TextureDescriptor {
label: hal_label(
Some(alloc::borrow::Cow::Borrowed("<Surface Texture>")),
@@ -343,7 +339,10 @@ impl Queue {
Some(resource::TextureInner::Surface { raw }) => {
let raw_surface = surface.raw(device.backend()).unwrap();
let raw_queue = self.raw();
let _fence_lock = device.fence.write();
// [`wgpu_hal::Queue::present`] requires the queue to be synchronized with submit calls and
// other present calls. Locking command indices prevents submits which must increment the
// submission index, and by `write`ing prevents other present calls.
let _command_indices = device.command_indices.write();
unsafe { raw_queue.present(raw_surface, raw) }
}
_ => unreachable!(),
+1 -1
View File
@@ -1670,7 +1670,7 @@ impl crate::Queue for Queue {
&self,
command_buffers: &[&CommandBuffer],
_surface_textures: &[&Texture],
(signal_fence, signal_value): (&mut Fence, crate::FenceValue),
(signal_fence, signal_value): (&Fence, crate::FenceValue),
) -> Result<(), crate::DeviceError> {
let mut temp_lists = self.temp_lists.lock();
temp_lists.clear();
-10
View File
@@ -57,10 +57,6 @@ trait DynResourceExt {
///
/// - Panics if `self` is not downcastable to `T`.
fn expect_downcast_ref<T: DynResource>(&self) -> &T;
/// # Panics
///
/// - Panics if `self` is not downcastable to `T`.
fn expect_downcast_mut<T: DynResource>(&mut self) -> &mut T;
/// Unboxes a `Box<dyn DynResource>` to a concrete type.
///
@@ -77,12 +73,6 @@ impl<R: DynResource + ?Sized> DynResourceExt for R {
.expect("Resource doesn't have the expected backend type.")
}
fn expect_downcast_mut<'a, T: DynResource>(&'a mut self) -> &'a mut T {
self.as_any_mut()
.downcast_mut()
.expect("Resource doesn't have the expected backend type.")
}
unsafe fn unbox<T: DynResource + 'static>(self: Box<Self>) -> T {
debug_assert!(
<Self as Any>::type_id(self.as_ref()) == TypeId::of::<T>(),
+3 -3
View File
@@ -12,7 +12,7 @@ pub trait DynQueue: DynResource {
&self,
command_buffers: &[&dyn DynCommandBuffer],
surface_textures: &[&dyn DynSurfaceTexture],
signal_fence: (&mut dyn DynFence, FenceValue),
signal_fence: (&dyn DynFence, FenceValue),
) -> Result<(), DeviceError>;
unsafe fn present(
&self,
@@ -28,7 +28,7 @@ impl<Q: Queue + DynResource> DynQueue for Q {
&self,
command_buffers: &[&dyn DynCommandBuffer],
surface_textures: &[&dyn DynSurfaceTexture],
signal_fence: (&mut dyn DynFence, FenceValue),
signal_fence: (&dyn DynFence, FenceValue),
) -> Result<(), DeviceError> {
let command_buffers = command_buffers
.iter()
@@ -38,7 +38,7 @@ impl<Q: Queue + DynResource> DynQueue for Q {
.iter()
.map(|surface| (*surface).expect_downcast_ref())
.collect::<Vec<_>>();
let signal_fence = (signal_fence.0.expect_downcast_mut(), signal_fence.1);
let signal_fence = (signal_fence.0.expect_downcast_ref(), signal_fence.1);
unsafe { Q::submit(self, &command_buffers, &surface_textures, signal_fence) }
}
+58 -25
View File
@@ -1,20 +1,29 @@
use alloc::vec::Vec;
use alloc::{sync::Arc, vec::Vec};
use core::sync::atomic::Ordering;
use parking_lot::RwLock;
use glow::HasContext;
use crate::AtomicFenceValue;
#[derive(Debug, Copy, Clone)]
#[derive(Debug)]
struct GLFence {
sync: glow::Fence,
// Since a fence can be `Copy`ed, there can exist some
// cases where a fence could be destroyed while something
// else is still using it. Therefore, while a function is
// using this fence (and doesn't keep pending read locked),
// it should clone the `Arc` to show it needs this to
// stay alive.
//
// The arc should not be kept after a function has finished
sync: Arc<glow::Fence>,
value: crate::FenceValue,
}
#[derive(Debug)]
pub struct Fence {
last_completed: AtomicFenceValue,
pending: Vec<GLFence>,
pending: RwLock<Vec<GLFence>>,
fence_behavior: wgt::GlFenceBehavior,
}
@@ -29,24 +38,27 @@ impl Fence {
pub fn new(options: &wgt::GlBackendOptions) -> Self {
Self {
last_completed: AtomicFenceValue::new(0),
pending: Vec::new(),
pending: RwLock::new(Vec::new()),
fence_behavior: options.fence_behavior,
}
}
pub fn signal(
&mut self,
&self,
gl: &glow::Context,
value: crate::FenceValue,
) -> Result<(), crate::DeviceError> {
if self.fence_behavior.is_auto_finish() {
*self.last_completed.get_mut() = value;
self.last_completed.store(value, Ordering::Release);
return Ok(());
}
let sync = unsafe { gl.fence_sync(glow::SYNC_GPU_COMMANDS_COMPLETE, 0) }
.map_err(|_| crate::DeviceError::OutOfMemory)?;
self.pending.push(GLFence { sync, value });
self.pending.write().push(GLFence {
sync: Arc::new(sync),
value,
});
Ok(())
}
@@ -62,12 +74,15 @@ impl Fence {
return max_value;
}
for gl_fence in self.pending.iter() {
let pending = self.pending.read();
for gl_fence in pending.iter() {
if gl_fence.value <= max_value {
// We already know this was good, no need to check again
continue;
}
let status = unsafe { gl.get_sync_status(gl_fence.sync) };
// We have pending `read` locked, so we shouldn't have to clone it.
let status = unsafe { gl.get_sync_status(*gl_fence.sync) };
if status == glow::SIGNALED {
max_value = gl_fence.value;
} else {
@@ -82,20 +97,29 @@ impl Fence {
max_value
}
pub fn maintain(&mut self, gl: &glow::Context) {
pub fn maintain(&self, gl: &glow::Context) {
if self.fence_behavior.is_auto_finish() {
return;
}
let latest = self.get_latest(gl);
for &gl_fence in self.pending.iter() {
if gl_fence.value <= latest {
let mut pending = self.pending.write();
pending.retain_mut(|gl_fence| {
if gl_fence.value > latest {
true
} else if let Some(fence) = Arc::get_mut(&mut gl_fence.sync) {
unsafe {
gl.delete_sync(gl_fence.sync);
gl.delete_sync(*fence);
}
false
} else {
// Another function is currently using this value. In general, these should finish
// very quickly (for wait because the fence should already be signaled, an all
// others are just fast), but submit should be very fast, so we shouldn't block on
// this.
true
}
}
self.pending.retain(|&gl_fence| gl_fence.value > latest);
});
}
pub fn wait(
@@ -115,25 +139,32 @@ impl Fence {
return Ok(true);
}
let pending = self.pending.read();
// Find a matching fence
let gl_fence = self
.pending
.iter()
.find(|gl_fence| gl_fence.value >= wait_value);
let gl_fence = pending.iter().find(|gl_fence| gl_fence.value >= wait_value);
let Some(gl_fence) = gl_fence else {
log::warn!("Tried to wait for {wait_value} but that value has not been signalled yet");
return Ok(false);
};
// clone to show we're using the fence
let sync = gl_fence.sync.clone();
let fence_value = gl_fence.value;
drop(pending);
let status = unsafe {
gl.client_wait_sync(
gl_fence.sync,
*sync,
glow::SYNC_FLUSH_COMMANDS_BIT,
timeout_ns.min(i32::MAX as u32) as i32,
)
};
drop(sync);
let signalled = match status {
glow::ALREADY_SIGNALED | glow::CONDITION_SATISFIED => true,
glow::TIMEOUT_EXPIRED | glow::WAIT_FAILED => false,
@@ -144,8 +175,7 @@ impl Fence {
};
if signalled {
self.last_completed
.fetch_max(gl_fence.value, Ordering::AcqRel);
self.last_completed.fetch_max(fence_value, Ordering::AcqRel);
}
Ok(signalled)
@@ -156,9 +186,12 @@ impl Fence {
return;
}
for gl_fence in self.pending {
for gl_fence in self.pending.into_inner() {
unsafe {
gl.delete_sync(gl_fence.sync);
gl.delete_sync(
Arc::into_inner(gl_fence.sync)
.expect("A function has failed to drop all its references to this"),
);
}
}
}
+1 -1
View File
@@ -1904,7 +1904,7 @@ impl crate::Queue for super::Queue {
&self,
command_buffers: &[&super::CommandBuffer],
_surface_textures: &[&super::Texture],
(signal_fence, signal_value): (&mut super::Fence, crate::FenceValue),
(signal_fence, signal_value): (&super::Fence, crate::FenceValue),
) -> Result<(), crate::DeviceError> {
let shared = Arc::clone(&self.shared);
let gl = &shared.context.lock();
+6 -1
View File
@@ -634,6 +634,9 @@ pub trait Api: Clone + fmt::Debug + Sized + WasmNotSendSync + 'static {
/// before a lower-valued operation, then waiting for the fence to reach the
/// lower value could return before the lower-valued operation has actually
/// finished.
///
/// Fences are internally synchronised by the hal, and so should not need to be
/// contained in external synchronisation primitives.
type Fence: DynFence;
type BindGroupLayout: DynBindGroupLayout;
@@ -717,6 +720,8 @@ pub trait Surface: WasmNotSendSync {
/// [`SurfaceTexture`] to [`self.discard_texture`], so that it can be reused
/// by future acquisitions.
///
/// The fence is internally synchronised by the hal.
///
/// # Portability
///
/// Some backends can't support a timeout when acquiring a texture. On these
@@ -1245,7 +1250,7 @@ pub trait Queue: WasmNotSendSync {
&self,
command_buffers: &[&<Self::A as Api>::CommandBuffer],
surface_textures: &[&<Self::A as Api>::SurfaceTexture],
signal_fence: (&mut <Self::A as Api>::Fence, FenceValue),
signal_fence: (&<Self::A as Api>::Fence, FenceValue),
) -> Result<(), DeviceError>;
/// Present a surface texture to the screen.
///
+11 -5
View File
@@ -1,5 +1,6 @@
use alloc::{borrow::ToOwned as _, sync::Arc, vec::Vec};
use core::{ptr::NonNull, sync::atomic};
use parking_lot::RwLock;
use std::{thread, time};
use bytemuck::TransparentWrapper;
@@ -1886,7 +1887,7 @@ impl crate::Device for super::Device {
};
Ok(super::Fence {
completed_value: Arc::new(atomic::AtomicU64::new(0)),
pending_command_buffers: Vec::new(),
pending_command_buffers: RwLock::new(Vec::new()),
shared_event,
})
}
@@ -1897,7 +1898,8 @@ impl crate::Device for super::Device {
unsafe fn get_fence_value(&self, fence: &super::Fence) -> DeviceResult<crate::FenceValue> {
let mut max_value = fence.completed_value.load(atomic::Ordering::Acquire);
for &(value, ref cmd_buf) in fence.pending_command_buffers.iter() {
let pending_command_buffers = fence.pending_command_buffers.read();
for &(value, ref cmd_buf) in pending_command_buffers.iter() {
if cmd_buf.status() == MTLCommandBufferStatus::Completed {
max_value = value;
}
@@ -1914,18 +1916,22 @@ impl crate::Device for super::Device {
return Ok(true);
}
let cmd_buf = match fence
.pending_command_buffers
let pending_command_buffers = fence.pending_command_buffers.read();
let cmd_buf = match pending_command_buffers
.iter()
.find(|&&(value, _)| value >= wait_value)
{
Some((_, cmd_buf)) => cmd_buf,
Some((_, cmd_buf)) => cmd_buf.clone(),
None => {
log::error!("No active command buffers for fence value {wait_value}");
return Err(crate::DeviceError::Lost);
}
};
// Make sure that nothing is blocked during the actual wait.
drop(pending_command_buffers);
let start = time::Instant::now();
loop {
if let MTLCommandBufferStatus::Completed = cmd_buf.status() {
+12 -7
View File
@@ -538,7 +538,7 @@ impl crate::Queue for Queue {
&self,
command_buffers: &[&CommandBuffer],
_surface_textures: &[&SurfaceTexture],
(signal_fence, signal_value): (&mut Fence, crate::FenceValue),
(signal_fence, signal_value): (&Fence, crate::FenceValue),
) -> Result<(), crate::DeviceError> {
autoreleasepool(|_| {
let extra_command_buffer = {
@@ -564,6 +564,7 @@ impl crate::Queue for Queue {
signal_fence.maintain();
signal_fence
.pending_command_buffers
.write()
.push((signal_value, raw.clone()));
if let Some(shared_event) = &signal_fence.shared_event {
@@ -1029,13 +1030,15 @@ unsafe impl Sync for QuerySet {}
pub struct Fence {
completed_value: Arc<atomic::AtomicU64>,
/// The pending fence values have to be ascending.
pending_command_buffers: Vec<(
crate::FenceValue,
Retained<ProtocolObject<dyn MTLCommandBuffer>>,
)>,
pending_command_buffers: RwLock<Vec<PendingCommandBuffer>>,
shared_event: Option<Retained<ProtocolObject<dyn MTLSharedEvent>>>,
}
type PendingCommandBuffer = (
crate::FenceValue,
Retained<ProtocolObject<dyn MTLCommandBuffer>>,
);
impl crate::DynFence for Fence {}
unsafe impl Send for Fence {}
@@ -1044,7 +1047,8 @@ unsafe impl Sync for Fence {}
impl Fence {
fn get_latest(&self) -> crate::FenceValue {
let mut max_value = self.completed_value.load(atomic::Ordering::Acquire);
for &(value, ref cmd_buf) in self.pending_command_buffers.iter() {
let pending_command_buffers = self.pending_command_buffers.read();
for &(value, ref cmd_buf) in pending_command_buffers.iter() {
if cmd_buf.status() == MTLCommandBufferStatus::Completed {
max_value = value;
}
@@ -1052,9 +1056,10 @@ impl Fence {
max_value
}
fn maintain(&mut self) {
fn maintain(&self) {
let latest = self.get_latest();
self.pending_command_buffers
.write()
.retain(|&(value, _)| value > latest);
}
+1 -1
View File
@@ -262,7 +262,7 @@ impl crate::Queue for Context {
&self,
command_buffers: &[&CommandBuffer],
surface_textures: &[&Resource],
(fence, fence_value): (&mut Fence, crate::FenceValue),
(fence, fence_value): (&Fence, crate::FenceValue),
) -> DeviceResult<()> {
// All commands are executed synchronously.
for cb in command_buffers {
+31 -16
View File
@@ -10,7 +10,7 @@ use core::{
use arrayvec::ArrayVec;
use ash::{ext, vk};
use hashbrown::hash_map::Entry;
use parking_lot::Mutex;
use parking_lot::{Mutex, RwLock};
use super::{conv, descriptor::DescriptorCounts, RawTlasInstance};
use crate::TlasInstance;
@@ -2279,11 +2279,11 @@ impl crate::Device for super::Device {
super::Fence::TimelineSemaphore(raw)
} else {
super::Fence::FencePool {
super::Fence::FencePool(RwLock::new(super::FencePool {
last_completed: 0,
active: Vec::new(),
free: Vec::new(),
}
}))
})
}
unsafe fn destroy_fence(&self, fence: super::Fence) {
@@ -2291,13 +2291,17 @@ impl crate::Device for super::Device {
super::Fence::TimelineSemaphore(raw) => {
unsafe { self.shared.raw.destroy_semaphore(raw, None) };
}
super::Fence::FencePool {
active,
free,
last_completed: _,
} => {
super::Fence::FencePool(pool) => {
let super::FencePool {
active,
free,
last_completed: _,
} = pool.into_inner();
for (_, raw) in active {
unsafe { self.shared.raw.destroy_fence(raw, None) };
unsafe {
self.shared.raw.destroy_fence(Arc::into_inner(raw).expect("Fence should have its reference count be one by the end of each function"), None)
};
}
for raw in free {
unsafe { self.shared.raw.destroy_fence(raw, None) };
@@ -2829,17 +2833,28 @@ impl super::DeviceShared {
Err(other) => Err(super::map_host_device_oom_and_lost_err(other)),
}
}
super::Fence::FencePool {
last_completed,
ref active,
free: _,
} => {
super::Fence::FencePool(ref pool) => {
let pool = pool.read();
let super::FencePool {
last_completed,
ref active,
free: _,
} = *pool;
if wait_value <= last_completed {
Ok(true)
} else {
match active.iter().find(|&&(value, _)| value >= wait_value) {
Some(&(_, raw)) => {
match unsafe { self.raw.wait_for_fences(&[raw], true, timeout_ns) } {
Some((_, fence)) => {
// clone to show we are using this fence while the pool is unlocked.
let fence = fence.clone();
drop(pool);
match unsafe {
self.raw.wait_for_fences(
core::slice::from_ref(&fence),
true,
timeout_ns,
)
} {
Ok(()) => Ok(true),
Err(vk::Result::TIMEOUT) => Ok(false),
Err(other) => Err(super::map_host_device_oom_and_lost_err(other)),
+77 -37
View File
@@ -1162,14 +1162,29 @@ pub enum Fence {
/// for each queue submission we might want to wait for, and remember which
/// [`FenceValue`] each one represents.
///
/// One should keep the fence pool read while there are any references to the
/// fences inside of them. This ensures there are no race conditions when
/// resetting the fences
///
/// [fence]: https://registry.khronos.org/vulkan/specs/1.3-extensions/html/vkspec.html#synchronization-fences
/// [`FenceValue`]: crate::FenceValue
FencePool {
last_completed: crate::FenceValue,
/// The pending fence values have to be ascending.
active: Vec<(crate::FenceValue, vk::Fence)>,
free: Vec<vk::Fence>,
},
FencePool(RwLock<FencePool>),
}
/// A shared fence type. The arc is expect to have a ref-count of one once a function has finished being called
///
/// A fence should have access synchronised as fence resetting might happen at any point. Resetting checks the ref-count
/// of the fence, so instead of copying the fence, it should have its `Arc` container cloned which shows not to reset
/// this fence as it is being used.
pub(super) type SynchronizedFence = Arc<vk::Fence>;
#[derive(Debug)]
pub struct FencePool {
last_completed: crate::FenceValue,
/// The pending fence values have to be ascending.
active: Vec<(crate::FenceValue, SynchronizedFence)>,
// Don't need extra synchronisation around the fences here, if they are used they should be put into active.
free: Vec<vk::Fence>,
}
impl crate::DynFence for Fence {}
@@ -1186,13 +1201,15 @@ impl Fence {
fn check_active(
device: &ash::Device,
mut last_completed: crate::FenceValue,
active: &[(crate::FenceValue, vk::Fence)],
active: &[(crate::FenceValue, SynchronizedFence)],
) -> Result<crate::FenceValue, crate::DeviceError> {
for &(value, raw) in active.iter() {
for &(value, ref raw) in active.iter() {
unsafe {
if value > last_completed
&& device
.get_fence_status(raw)
// Don't need to clone as active should be from a read or
// write lock which means this is already synchronised.
.get_fence_status(**raw)
.map_err(map_host_device_oom_and_lost_err)?
{
last_completed = value;
@@ -1221,11 +1238,14 @@ impl Fence {
.map_err(map_host_device_oom_and_lost_err)?,
})
},
Self::FencePool {
last_completed,
ref active,
free: _,
} => Self::check_active(device, last_completed, active),
Self::FencePool(ref pool) => {
let FencePool {
last_completed,
ref active,
free: _,
} = *pool.read();
Self::check_active(device, last_completed, active)
}
}
}
@@ -1241,23 +1261,35 @@ impl Fence {
///
/// [`FencePool`]: Fence::FencePool
/// [`Queue::submit`]: crate::Queue::submit
fn maintain(&mut self, device: &ash::Device) -> Result<(), crate::DeviceError> {
fn maintain(&self, device: &ash::Device) -> Result<(), crate::DeviceError> {
match *self {
Self::TimelineSemaphore(_) => {}
Self::FencePool {
ref mut last_completed,
ref mut active,
ref mut free,
} => {
let latest = Self::check_active(device, *last_completed, active)?;
Self::FencePool(ref pool) => {
let FencePool {
ref mut last_completed,
ref mut active,
ref mut free,
} = *pool.write();
let base_free = free.len();
for &(value, raw) in active.iter() {
if value <= latest {
free.push(raw);
let latest = Self::check_active(device, *last_completed, active)?;
active.retain_mut(|&mut (value, ref mut fence)| {
if value > latest {
true
} else if let Some(fence) = Arc::get_mut(fence) {
// No other references to these, so we have exclusive access. Add them to free and reset them later,
// but drop them from active immediately
free.push(*fence);
false
} else {
// some other function is using it. Although this shouldn't be to long,
// maintain shouldn't block, and it should be cleared up by the next time it happens
true
}
}
});
if free.len() != base_free {
active.retain(|&(value, _)| value > latest);
unsafe { device.reset_fences(&free[base_free..]) }
.map_err(map_device_oom_err)?
}
@@ -1275,7 +1307,7 @@ impl crate::Queue for Queue {
&self,
command_buffers: &[&CommandBuffer],
surface_textures: &[&SurfaceTexture],
(signal_fence, signal_value): (&mut Fence, crate::FenceValue),
(signal_fence, signal_value): (&Fence, crate::FenceValue),
) -> Result<(), crate::DeviceError> {
let mut fence_raw = vk::Fence::null();
@@ -1343,25 +1375,33 @@ impl crate::Queue for Queue {
// We need to signal our wgpu::Fence if we have one, this adds it to the signal list.
signal_fence.maintain(&self.device.raw)?;
// Keeping the Arc around is probably unneeded - the fence should never be signaled as it was reset,
// and newer submits should not happen until this submit is done. Therefore, it should be too high
// to be reset.
let shared_fence;
match *signal_fence {
Fence::TimelineSemaphore(raw) => {
signal_semaphores.push_signal(SemaphoreType::Timeline(raw, signal_value));
}
Fence::FencePool {
ref mut active,
ref mut free,
..
} => {
fence_raw = match free.pop() {
Some(raw) => raw,
Fence::FencePool(ref pool) => {
let FencePool {
ref mut active,
ref mut free,
..
} = *pool.write();
shared_fence = match free.pop() {
Some(raw) => Arc::new(raw),
None => unsafe {
self.device
let fence = self
.device
.raw
.create_fence(&vk::FenceCreateInfo::default(), None)
.map_err(map_host_device_oom_err)?
.map_err(map_host_device_oom_err)?;
Arc::new(fence)
},
};
active.push((signal_value, fence_raw));
fence_raw = *shared_fence;
active.push((signal_value, shared_fence.clone()));
}
}