mirror of
https://github.com/storytold/wgpu.git
synced 2026-11-20 14:31:08 +01:00
metal: prevent command buffer exhaustion deadlocks (#8574)
- Track outstanding Metal command buffers per queue and gate begin_encoding against the hard MAX_COMMAND_BUFFERS, returning device-lost with an actionable warning instead of letting new_command_buffer hang when encoders are leaked. - Share the counter across queue/encoders and decrement on submit or discard after clearing the raw command buffer so drop happens before the bookkeeping update. Fixes #3084, #8047. --------- Co-authored-by: Andy Leiserson <aleiserson@mozilla.com>
This commit is contained in:
1 parent
384b7bb027
commit
28bda54ae0
3 files changed
+94
-13
No files matched your search
@@ -8,6 +8,7 @@ use objc2_metal::{
|
||||
use wgt::{AstcBlock, AstcChannel};
|
||||
|
||||
use alloc::{string::ToString as _, sync::Arc, vec::Vec};
|
||||
use core::sync::atomic;
|
||||
|
||||
use crate::metal::QueueShared;
|
||||
|
||||
@@ -30,7 +31,7 @@ use super::{OsFeatures, TimestampQuerySupport};
|
||||
/// <https://bugzilla.mozilla.org/show_bug.cgi?id=1971452>.
|
||||
///
|
||||
/// [new command buffer]: https://developer.apple.com/documentation/metal/mtlcommandqueue/makecommandbuffer()?language=objc
|
||||
const MAX_COMMAND_BUFFERS: usize = 4096;
|
||||
pub(super) const MAX_COMMAND_BUFFERS: usize = 4096;
|
||||
|
||||
// Metal has a single buffer limit that we must split across 3 WebGPU limits:
|
||||
// The Metal limit is: 31 "Maximum number of entries in the buffer argument table, per graphics or kernel function".
|
||||
@@ -106,7 +107,10 @@ impl crate::Adapter for super::Adapter {
|
||||
counters: Default::default(),
|
||||
},
|
||||
queue: super::Queue {
|
||||
shared: Arc::new(QueueShared { raw: queue }),
|
||||
shared: Arc::new(QueueShared {
|
||||
raw: queue,
|
||||
command_buffer_created_not_submitted: atomic::AtomicUsize::new(0),
|
||||
}),
|
||||
timestamp_period,
|
||||
},
|
||||
})
|
||||
|
||||
@@ -5,21 +5,25 @@ use objc2::{
|
||||
use objc2_foundation::{NSRange, NSString, NSUInteger};
|
||||
use objc2_metal::{
|
||||
MTLAccelerationStructure, MTLAccelerationStructureCommandEncoder, MTLBlitCommandEncoder,
|
||||
MTLBlitPassDescriptor, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue,
|
||||
MTLComputeCommandEncoder, MTLComputePassDescriptor, MTLCounterDontSample, MTLDevice,
|
||||
MTLLoadAction, MTLPrimitiveType, MTLRenderCommandEncoder, MTLRenderPassDescriptor,
|
||||
MTLBlitPassDescriptor, MTLBuffer, MTLCommandBuffer, MTLCommandBufferStatus, MTLCommandEncoder,
|
||||
MTLCommandQueue, MTLComputeCommandEncoder, MTLComputePassDescriptor, MTLCounterDontSample,
|
||||
MTLDevice, MTLLoadAction, MTLPrimitiveType, MTLRenderCommandEncoder, MTLRenderPassDescriptor,
|
||||
MTLResidencySet, MTLResidencySetDescriptor, MTLSamplerState, MTLScissorRect, MTLSize,
|
||||
MTLStoreAction, MTLTexture, MTLVertexAmplificationViewMapping, MTLViewport,
|
||||
MTLVisibilityResultMode,
|
||||
};
|
||||
|
||||
use super::{adapter::VERTEX_BUFFER_SLOT_START, conv, TimestampQuerySupport};
|
||||
use super::{
|
||||
adapter::{self, VERTEX_BUFFER_SLOT_START},
|
||||
conv, TimestampQuerySupport,
|
||||
};
|
||||
use crate::CommandEncoder as _;
|
||||
use alloc::{
|
||||
borrow::{Cow, ToOwned as _},
|
||||
sync::Arc,
|
||||
vec::Vec,
|
||||
};
|
||||
use core::{ops::Range, ptr::NonNull};
|
||||
use core::{ops::Range, ptr::NonNull, sync::atomic};
|
||||
use smallvec::SmallVec;
|
||||
|
||||
// has to match `Temp::binding_sizes`
|
||||
@@ -454,6 +458,26 @@ impl crate::CommandEncoder for super::CommandEncoder {
|
||||
unsafe fn begin_encoding(&mut self, label: crate::Label) -> Result<(), crate::DeviceError> {
|
||||
let queue = &self.queue_shared.raw;
|
||||
let retain_references = self.shared.settings.retain_command_buffer_references;
|
||||
|
||||
// Guard against exhausting Metal's command buffer budget. Use the hard
|
||||
// limit (`MAX_COMMAND_BUFFERS`) so we fail before Metal can hang inside
|
||||
// `new_command_buffer`.
|
||||
let previous = self
|
||||
.queue_shared
|
||||
.command_buffer_created_not_submitted
|
||||
.fetch_add(1, atomic::Ordering::AcqRel);
|
||||
if previous >= adapter::MAX_COMMAND_BUFFERS {
|
||||
let current = previous + 1;
|
||||
log::warn!(
|
||||
"metal: refusing to create new command buffer; {current} outstanding command \
|
||||
buffers exceeds the limit of {}. Treating this as device lost. \
|
||||
Ensure command encoders are submitted or dropped rather than kept alive \
|
||||
to avoid exhausting Metal's command buffer budget.",
|
||||
adapter::MAX_COMMAND_BUFFERS
|
||||
);
|
||||
return Err(crate::DeviceError::Lost);
|
||||
}
|
||||
|
||||
let raw = autoreleasepool(move |_| {
|
||||
let cmd_buf_ref = if retain_references {
|
||||
queue.commandBuffer()
|
||||
@@ -483,7 +507,15 @@ impl crate::CommandEncoder for super::CommandEncoder {
|
||||
if let Some(encoder) = self.state.compute.take() {
|
||||
encoder.endEncoding();
|
||||
}
|
||||
let had_command_buffer = self.raw_cmd_buf.is_some();
|
||||
// Clear the Option first so the underlying `metal::CommandBuffer` is
|
||||
// dropped before we update the counter.
|
||||
self.raw_cmd_buf = None;
|
||||
if had_command_buffer {
|
||||
self.queue_shared
|
||||
.command_buffer_created_not_submitted
|
||||
.fetch_sub(1, atomic::Ordering::AcqRel);
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn end_encoding(&mut self) -> Result<super::CommandBuffer, crate::DeviceError> {
|
||||
@@ -501,6 +533,7 @@ impl crate::CommandEncoder for super::CommandEncoder {
|
||||
|
||||
Ok(super::CommandBuffer {
|
||||
raw: self.raw_cmd_buf.take().unwrap(),
|
||||
queue_shared: Arc::clone(&self.queue_shared),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1866,3 +1899,21 @@ impl Drop for super::CommandEncoder {
|
||||
self.counters.command_encoders.sub(1);
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for super::CommandBuffer {
|
||||
fn drop(&mut self) {
|
||||
// `command_buffer_created_not_submitted` is usually decremented when the command
|
||||
// buffer is submitted. But if we're dropping a command buffer that was never
|
||||
// submitted, we need to decrement the count here.
|
||||
let status = self.raw.status();
|
||||
if status == MTLCommandBufferStatus::NotEnqueued
|
||||
|| status == MTLCommandBufferStatus::Enqueued
|
||||
{
|
||||
let previous = self
|
||||
.queue_shared
|
||||
.command_buffer_created_not_submitted
|
||||
.fetch_sub(1, atomic::Ordering::AcqRel);
|
||||
debug_assert!(previous > 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -459,14 +459,26 @@ impl Queue {
|
||||
timestamp_period: f32,
|
||||
) -> Self {
|
||||
Self {
|
||||
shared: Arc::new(QueueShared { raw }),
|
||||
shared: Arc::new(QueueShared {
|
||||
raw,
|
||||
command_buffer_created_not_submitted: atomic::AtomicUsize::new(0),
|
||||
}),
|
||||
timestamp_period,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct QueueShared {
|
||||
raw: Retained<ProtocolObject<dyn MTLCommandQueue>>,
|
||||
// Tracks command buffers created via `CommandEncoder::begin_encoding` that
|
||||
// have not yet been submitted or discarded. Used to proactively fail
|
||||
// before hitting Metal's `maxCommandBufferCount`.
|
||||
//
|
||||
// (In a few places we call `.commandBuffer{,WithUnretainedReferences}` directly
|
||||
// to create command buffers for internal purposes. In those cases we always
|
||||
// commit the buffer immediately, so we don't adjust the counter for them.)
|
||||
command_buffer_created_not_submitted: atomic::AtomicUsize,
|
||||
}
|
||||
|
||||
pub struct Device {
|
||||
@@ -528,11 +540,14 @@ impl crate::Queue for Queue {
|
||||
|
||||
let raw = match command_buffers.last() {
|
||||
Some(&cmd_buf) => cmd_buf.raw.clone(),
|
||||
None => self
|
||||
.shared
|
||||
.raw
|
||||
.commandBufferWithUnretainedReferences()
|
||||
.unwrap(),
|
||||
None => {
|
||||
// We do not bother adjusting `command_buffer_created_not_submitted`
|
||||
// because we immediately commit this buffer.
|
||||
self.shared
|
||||
.raw
|
||||
.commandBufferWithUnretainedReferences()
|
||||
.unwrap()
|
||||
}
|
||||
};
|
||||
raw.setLabel(Some(ns_string!("(wgpu internal) Signal")));
|
||||
unsafe { raw.addCompletedHandler(block2::RcBlock::as_ptr(&block)) };
|
||||
@@ -554,6 +569,14 @@ impl crate::Queue for Queue {
|
||||
|
||||
for cmd_buffer in command_buffers {
|
||||
cmd_buffer.raw.commit();
|
||||
// One command buffer per `end_encoding` call moves from the
|
||||
// "created but not yet submitted" bucket into the submitted
|
||||
// set, so update the counter.
|
||||
let previous = self
|
||||
.shared
|
||||
.command_buffer_created_not_submitted
|
||||
.fetch_sub(1, atomic::Ordering::AcqRel);
|
||||
debug_assert!(previous > 0);
|
||||
}
|
||||
|
||||
if let Some(raw) = extra_command_buffer {
|
||||
@@ -568,6 +591,8 @@ impl crate::Queue for Queue {
|
||||
texture: SurfaceTexture,
|
||||
) -> Result<(), crate::SurfaceError> {
|
||||
autoreleasepool(|_| {
|
||||
// We do not bother adjusting `command_buffer_created_not_submitted`
|
||||
// because we immediately commit this buffer.
|
||||
let command_buffer = self.shared.raw.commandBuffer().unwrap();
|
||||
command_buffer.setLabel(Some(ns_string!("(wgpu internal) Present")));
|
||||
|
||||
@@ -1093,6 +1118,7 @@ unsafe impl Sync for CommandEncoder {}
|
||||
#[derive(Debug)]
|
||||
pub struct CommandBuffer {
|
||||
raw: Retained<ProtocolObject<dyn MTLCommandBuffer>>,
|
||||
queue_shared: Arc<QueueShared>,
|
||||
}
|
||||
|
||||
impl crate::DynCommandBuffer for CommandBuffer {}
|
||||
|
||||
Reference in new issue
Block a user